riley/Beam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance project metadata and refactor core classes

Browse Source 
Updated project files for `Beam.Dynamic`, `Beam.Exports`, `Beam.Temporary.Cli`, and `Beam` to include additional metadata and specific package versions. Refactored `DataBindings` and `ResolvedBindings` to records, added a new `Text` property in `Binding.cs`, and introduced `ParseNumbers` in `OnlineCleaner`. New classes `PuppetContext` and `PuppetUnitDownloader` added for Playwright integration. Introduced `ImmutableState` struct and `UnitDownloaderBinary` class for improved download management. Updated tests in `UnitTest1.cs` for number localization. Added `Beam.Puppeteer` project to the solution.
This commit is contained in:
qwsdcvghyu89
2025-06-23 02:11:19 +03:00
parent a9a22ea23d
commit 482a46b568
27 changed files with 354 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Beam.Temporary.Cli/Program.cs
+10 -18
View File
@@ -49,6 +49,8 @@ namespace Beam.Temporary.Cli {
NovelStatics.Define_WoDuShu_HouseOfHorrors(BeamData);
ClassicTemplates.Register(BeamData);
await sharedContext.ForceSave();
CancellationTokenSource cts = new();
HtmlTransformer<IDocumentMetaData> ArticleDataTransformer(DataBindings? binding) => (x) => {
@@ -75,38 +77,23 @@ namespace Beam.Temporary.Cli {
};
var novel = new DataKey<TextResource>("novels:house_of_horrors");
var context_aux = Architecture.GetMeta(web, novel, BeamData, cts.Token);
context_aux.RetryReporter = new Progress<RetryReport>((x) => Console.WriteLine($"Failed. Trying again. {x.TryNumber}"));
var metaDownloader = new DownloadEnumerable<IDocumentMetaData>(
new SequentialFragmentDownloader<IDocumentMetaData>(
context_aux,
(c) => new UnitFragmentDownloader<IDocumentMetaData>(c.Web, c.AsyncTranformer, c.AsyncFailurePredicates, 4, logger),
logger)
.UnwrapFragmented());
var metadata = (await metaDownloader.FirstAsync());
var metadata2 = await DownloadBuilder<IDocumentMetaData>.FromMeta(novel, BeamData)
.WithLink()
.WithTransformer(ArticleDataTransformer)
.Configure((x) => x
.WithDownloadLogger(logger)
.WithRetryReporter(new Progress<RetryReport>())
.WithTimeOut(TimeSpan.FromSeconds(15)))
.Build()
.FirstAsync();
var context = Architecture.GetTextRecord(web, novel, BeamData, metadata.Data, cts.Token);
context.DownloadReporter = new Progress<DownloadReport>((x) => Console.WriteLine(x));
var downloader = new DownloadEnumerable<IDocument>(
new SequentialFragmentDownloader<IDocument>(
context,
(c) => new UnitFragmentDownloader<IDocument>(c.Web, c.AsyncTranformer, c.AsyncFailurePredicates, 4, logger),
logger)
.UnwrapFragmented());
var downloader2 = DownloadBuilder<IDocument>.FromText(novel, BeamData)
.WithRange(1..5)
.WithLinkGenerator()
.WithTransformer((x) => DocumentTransformer(x, metadata2.Data))
.Configure((x) => x
.WithDownloadLogger(logger)
.WithDownloadReporter(new Progress<DownloadReport>((x) => logger.LogInformation(x.ToString())))
.WithTimeOut(TimeSpan.FromSeconds(15))
)
@@ -122,11 +109,16 @@ namespace Beam.Temporary.Cli {
continue;
if (meta is not ArticleData articleMetaData)
continue;
if (!download.Data.MetaData.TryGetValue(Architecture.BookKey, out var bookmeta))
continue;
if (meta is not ArticleData bookMetaData)
continue;
//Console.WriteLine($"Title: {data.Name}");
//Console.WriteLine($"Description: {data.Description}");
//Console.WriteLine($"Categories: {data.Categories.Aggregate((x, y) => $"{x}; {y}")}");
//Console.WriteLine($"Authors: {data.Authors.Aggregate((x,y) => $"{x}; {y}")}");
Console.WriteLine($"Chapter title: {articleMetaData.Name}");
Console.WriteLine($"Book title: {bookMetaData.Name}");
//Console.WriteLine($"Content: {download}");
//translationTasks.Add(Task.Run(async () => {