riley/Beam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce Beam.Fluent and Beam.Models projects

Browse Source 
Added new Beam.Fluent and Beam.Models projects with staged download builder and data context models. Refactored and moved model classes from Beam.Temporary.Cli to Beam.Models. Added new data providers and extended DataBindings in Beam.Dynamic. Renamed Beam.Puppeteer to Beam.Playwright and updated related classes. Updated project references and package versions. Removed obsolete and unused files from Beam.Temporary.Cli.
This commit is contained in:
qwsdcvghyu89
2025-09-18 18:32:25 +10:00
parent 849bdcd089
commit a7d148a96f
72 changed files with 2100 additions and 721 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Beam.Temporary.Cli/MainArchitecture.cs
+35 -35
View File
@@ -11,49 +11,49 @@ namespace Beam.Temporary.Cli {
public DataKey<IDocumentMetaData> ChapterKey { get; set; } = new("ma:chapter");
public DataKey<IDocumentMetaData> BookKey { get; set; } = new("ma:book");
public DownloadContext<IDocumentMetaData>? GetMeta(HtmlWeb web, DataKey<TextResource> pieceKey, BeamDataDictionary sdd, CancellationToken ct = default, ILogger? logger = null) {
var piece = sdd.Novels[pieceKey].ToRecord(sdd); // retrieves novel data from the sdd
var auxiliary = piece.AssociatedMetaSource?.ToRecord(sdd); // retrieves novel aux data from the sdd
//public DownloadContext<IDocumentMetaData>? GetMeta(HtmlWeb web, DataKey<ResourceDictionary> pieceKey, BeamDataDictionary sdd, CancellationToken ct = default, ILogger? logger = null) {
// var piece = sdd.ResourceDictionaries[pieceKey].ToRecord(sdd); // retrieves novel data from the sdd
// var auxiliary = piece.AssociatedMetaSource?.ToRecord(sdd); // retrieves novel aux data from the sdd
// null checks
if (auxiliary is null) // aux is required to get metadata
return null;
if (piece?.Resource?.MetaTemplateInitialData is null) // sanity check to avoid null warnings
return null;
// // null checks
// if (auxiliary is null) // aux is required to get metadata
// return null;
// if (piece?.Resource?.MetaTemplateInitialData is null) // sanity check to avoid null warnings
// return null;
// gets the link for the novel's metadata using the auxillary data retrieved from the sdd
var link = sdd.Templates[auxiliary.Resource.Key].Builder.Build(piece?.Resource?.MetaTemplateInitialData);
var binding = auxiliary.Bindings;
// // gets the link for the novel's metadata using the auxillary data retrieved from the sdd
// var link = sdd.Templates[auxiliary.Resource.Key].Builder.Build(piece?.Resource?.MetaTemplateInitialData);
// var binding = auxiliary.Bindings;
return new DownloadContext<IDocumentMetaData>(web, new(), [link], downloadLogger: logger);
}
// return new DownloadContext<IDocumentMetaData>(web, new(), [link], downloadLogger: logger);
//}
public DownloadContext<IDocument>? GetTextRecord(HtmlWeb web, DataKey<TextResource> resKey, BeamDataDictionary sdd, IDocumentMetaData? metaData = null, CancellationToken ct = default, ILogger? logger = null) {
var res = sdd.Novels[resKey].ToRecord(sdd); // retrieves the novel data from the sdd
var aggregator = res.AssociatedSource?.ToRecord(sdd); // retrieves the aggregator (novel web source) from the sdd
//public DownloadContext<IDocument>? GetTextRecord(HtmlWeb web, DataKey<ResourceDictionary> resKey, BeamDataDictionary sdd, IDocumentMetaData? metaData = null, CancellationToken ct = default, ILogger? logger = null) {
// var res = sdd.ResourceDictionaries[resKey].ToRecord(sdd); // retrieves the novel data from the sdd
// var aggregator = res.AssociatedSource?.ToRecord(sdd); // retrieves the aggregator (novel web source) from the sdd
if (aggregator is null) // ensure aggergator data was retrieved successfully
return null;
if (res is null) // ensure novel data was retrieved successfully
return null;
// if (aggregator is null) // ensure aggergator data was retrieved successfully
// return null;
// if (res is null) // ensure novel data was retrieved successfully
// return null;
var template = sdd.Templates[aggregator.Resource.Key]; // gets the link generator for the specified aggregator
// var template = sdd.Templates[aggregator.Resource.Key]; // gets the link generator for the specified aggregator
// creates a generative enumerable of type link from 'template'
var sle = SourceLinkEnumerable.FromGenerator(new OrderedSourceLinkGenerator(
template.Builder, new NumberedStateChanger(template.Factory.Behavior),
res.Resource.TemplateInitialData));
// // creates a generative enumerable of type link from 'template'
// var sle = SourceLinkEnumerable.FromGenerator(new OrderedSourceLinkGenerator(
// template.Builder, new NumberedStateChanger(template.Factory.Behavior),
// res.Resource.TemplateInitialData));
return new DownloadContext<IDocument>(web, new(), sle,
retryReporter: new Progress<RetryReport>((x) => Console.WriteLine($"Retrying download of '{x.Link}' ({x.TryNumber}x)")),
//downloadReporter: new Progress<DownloadReport>((x) => Console.WriteLine($"Downloaded ({x})")),
asyncFailurePredicates: [
//(x) => Task.FromResult(!x.DocumentNode.InnerHtml.Contains("<div id=\"chapter-container\" class=\"chapter-content\" itemprop=\"description\">"))
],
timeOut: TimeSpan.FromSeconds(15),
downloadLogger: logger
);
}
// return new DownloadContext<IDocument>(web, new(), sle,
// retryReporter: new Progress<RetryReport>((x) => Console.WriteLine($"Retrying download of '{x.Link}' ({x.TryNumber}x)")),
// //downloadReporter: new Progress<DownloadReport>((x) => Console.WriteLine($"Downloaded ({x})")),
// asyncFailurePredicates: [
// //(x) => Task.FromResult(!x.DocumentNode.InnerHtml.Contains("<div id=\"chapter-container\" class=\"chapter-content\" itemprop=\"description\">"))
// ],
// timeOut: TimeSpan.FromSeconds(15),
// downloadLogger: logger
// );
//}
}
}