qwsdcvghyu89
056e426572
Updated project files for `Beam.Dynamic`, `Beam.Exports`, `Beam.Puppeteer`, `Beam.Temporary.Cli`, and `Beam` to include new XML headers, reorganize property groups, and add project references. Modified `PuppetedUnitDownloader` to support additional parameters for async transformers. Changed return types in `CommonTransformers` to `AsyncTransformer` for asynchronous processing. Significant refactoring in `DownloadBuilder`, `DownloadContext`, and `DownloadContextBuilder` to introduce generic parameters and improve context management. Updated `SequentialDownloader`, `SequentialFragmentDownloader`, and `UnitDownloader` to accommodate new async transformer types. Introduced `TypeExtensions` for unique type name generation and added `UnitFragmentDownloaderBinary` for handling binary downloads. Updated solution file to include the new `aeqw89.Beam` project, ensuring proper references across the solution. These changes enhance the asynchronous capabilities of the Beam library, improve type safety, and streamline the downloading process.
61 lines
3.4 KiB
C#
61 lines
3.4 KiB
C#
using aeqw89.DataKeys;
|
|
using Beam.Dynamic;
|
|
using HtmlAgilityPack;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace Beam.Temporary.Cli {
|
|
public partial interface IArchitecture {
|
|
private class MainArchitecture : IArchitecture {
|
|
public MainArchitecture() { }
|
|
|
|
public DataKey<IDocumentMetaData> ChapterKey { get; set; } = new("ma:chapter");
|
|
public DataKey<IDocumentMetaData> BookKey { get; set; } = new("ma:book");
|
|
|
|
public DownloadContext<IDocumentMetaData>? GetMeta(HtmlWeb web, DataKey<TextResource> pieceKey, BeamDataDictionary sdd, CancellationToken ct = default, ILogger? logger = null) {
|
|
var piece = sdd.Novels[pieceKey].ToRecord(sdd); // retrieves novel data from the sdd
|
|
var auxiliary = piece.AssociatedMetaSource?.ToRecord(sdd); // retrieves novel aux data from the sdd
|
|
|
|
// null checks
|
|
if (auxiliary is null) // aux is required to get metadata
|
|
return null;
|
|
if (piece?.Resource?.MetaTemplateInitialData is null) // sanity check to avoid null warnings
|
|
return null;
|
|
|
|
// gets the link for the novel's metadata using the auxillary data retrieved from the sdd
|
|
var link = sdd.Templates[auxiliary.Resource.Key].Builder.Build(piece?.Resource?.MetaTemplateInitialData);
|
|
var binding = auxiliary.Bindings;
|
|
|
|
return new DownloadContext<IDocumentMetaData>(web, new(), [link], downloadLogger: logger);
|
|
}
|
|
|
|
public DownloadContext<IDocument>? GetTextRecord(HtmlWeb web, DataKey<TextResource> resKey, BeamDataDictionary sdd, IDocumentMetaData? metaData = null, CancellationToken ct = default, ILogger? logger = null) {
|
|
var res = sdd.Novels[resKey].ToRecord(sdd); // retrieves the novel data from the sdd
|
|
var aggregator = res.AssociatedSource?.ToRecord(sdd); // retrieves the aggregator (novel web source) from the sdd
|
|
|
|
if (aggregator is null) // ensure aggergator data was retrieved successfully
|
|
return null;
|
|
if (res is null) // ensure novel data was retrieved successfully
|
|
return null;
|
|
|
|
var template = sdd.Templates[aggregator.Resource.Key]; // gets the link generator for the specified aggregator
|
|
|
|
// creates a generative enumerable of type link from 'template'
|
|
var sle = SourceLinkEnumerable.FromGenerator(new OrderedSourceLinkGenerator(
|
|
template.Builder, new NumberedStateChanger(template.Factory.Behavior),
|
|
res.Resource.TemplateInitialData));
|
|
|
|
return new DownloadContext<IDocument>(web, new(), sle,
|
|
retryReporter: new Progress<RetryReport>((x) => Console.WriteLine($"Retrying download of '{x.Link}' ({x.TryNumber}x)")),
|
|
//downloadReporter: new Progress<DownloadReport>((x) => Console.WriteLine($"Downloaded ({x})")),
|
|
asyncFailurePredicates: [
|
|
//(x) => Task.FromResult(!x.DocumentNode.InnerHtml.Contains("<div id=\"chapter-container\" class=\"chapter-content\" itemprop=\"description\">"))
|
|
],
|
|
timeOut: TimeSpan.FromSeconds(15),
|
|
downloadLogger: logger
|
|
);
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|