riley/Beam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance async capabilities and refactor project structure

Browse Source 
Updated project files for `Beam.Dynamic`, `Beam.Exports`, `Beam.Puppeteer`, `Beam.Temporary.Cli`, and `Beam` to include new XML headers, reorganize property groups, and add project references.

Modified `PuppetedUnitDownloader` to support additional parameters for async transformers. Changed return types in `CommonTransformers` to `AsyncTransformer` for asynchronous processing.

Significant refactoring in `DownloadBuilder`, `DownloadContext`, and `DownloadContextBuilder` to introduce generic parameters and improve context management. Updated `SequentialDownloader`, `SequentialFragmentDownloader`, and `UnitDownloader` to accommodate new async transformer types.

Introduced `TypeExtensions` for unique type name generation and added `UnitFragmentDownloaderBinary` for handling binary downloads. Updated solution file to include the new `aeqw89.Beam` project, ensuring proper references across the solution.

These changes enhance the asynchronous capabilities of the Beam library, improve type safety, and streamline the downloading process.
This commit is contained in:
qwsdcvghyu89
2025-06-23 20:30:09 +03:00
parent 482a46b568
commit 056e426572
23 changed files with 531 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Beam.Temporary.Cli/Program.cs
+4 -27
View File
@@ -53,34 +53,11 @@ namespace Beam.Temporary.Cli {
CancellationTokenSource cts = new();
HtmlTransformer<IDocumentMetaData> ArticleDataTransformer(DataBindings? binding) => (x) => {
return new ArticleData() {
Authors = [OnlineCleaner.Clean(binding?.Authors?.Resolve(x) ?? "")],
Name = OnlineCleaner.Clean(binding?.Title?.ResolveString(x) ?? ""),
Categories = OnlineCleaner.Clean(binding?.Tags?.ResolveString(x) ?? "").Split(';') ?? [],
Description = OnlineCleaner.Clean(binding?.Description?.ResolveString(x) ?? "")
};
};
HtmlTransformer<IDocument> DocumentTransformer(DataBindings? binding, IDocumentMetaData? metaData = null) => (x) => {
var resolved = binding?.Resolve(x);
var articleData = new ArticleData() {
Name = OnlineCleaner.Clean(resolved?.Title),
};
Dictionary<DataKey<IDocumentMetaData>, IDocumentMetaData> meta = [];
meta.Add(IArchitecture.Default.ChapterKey, articleData);
if (metaData is not null)
meta.Add(IArchitecture.Default.BookKey, metaData);
return new StringDocument(Path.GetRandomFileName(), OnlineCleaner.Clean(resolved?.Content)) {
MetaData = meta
};
};
var novel = new DataKey<TextResource>("novels:house_of_horrors");
var metadata2 = await DownloadBuilder<IDocumentMetaData>.FromMeta(novel, BeamData)
var metadata2 = await DownloadBuilder<HtmlDocument, IDocumentMetaData>.FromMeta(novel, BeamData)
.WithLink()
.WithTransformer(ArticleDataTransformer)
.WithTransformer(CommonTransformers.ArticleDataTransformer)
.Configure((x) => x
.WithDownloadLogger(logger)
.WithRetryReporter(new Progress<RetryReport>())
@@ -88,10 +65,10 @@ namespace Beam.Temporary.Cli {
.Build()
.FirstAsync();
var downloader2 = DownloadBuilder<IDocument>.FromText(novel, BeamData)
var downloader2 = DownloadBuilder<HtmlDocument, IDocument>.FromText(novel, BeamData)
.WithRange(1..5)
.WithLinkGenerator()
.WithTransformer((x) => DocumentTransformer(x, metadata2.Data))
.WithTransformer((x) => CommonTransformers.DocumentTransformer(x, metadata2.Data))
.Configure((x) => x
.WithDownloadLogger(logger)
.WithDownloadReporter(new Progress<DownloadReport>((x) => logger.LogInformation(x.ToString())))