using aeqw89.DataKeys; using Beam.Dynamic; using HtmlAgilityPack; using Microsoft.Extensions.Logging; namespace Beam.Temporary.Cli { partial interface IArchitecture { private class MainArchitecture : IArchitecture { public MainArchitecture() { } public DataKey ChapterKey { get; set; } = new("ma:chapter"); public DataKey BookKey { get; set; } = new("ma:book"); public DownloadContext? GetMeta(HtmlWeb web, DataKey pieceKey, SharedDataDictionary sdd, ILogger? logger = null) { var piece = sdd.Novels[pieceKey].ToRecord(sdd); // retrieves novel data from the sdd var auxiliary = piece.AssociatedMetaSource?.ToRecord(sdd); // retrieves novel aux data from the sdd // null checks if (auxiliary is null) // aux is required to get metadata return null; if (piece?.Resource?.MetaTemplateInitialData is null) // sanity check to avoid null warnings return null; // gets the link for the novel's metadata using the auxillary data retrieved from the sdd var link = sdd.Templates[auxiliary.Resource.Key].GenerateLink(piece?.Resource?.MetaTemplateInitialData!); var binding = auxiliary.Bindings; return new DownloadContext(web, [link], downloadLogger: logger, transformer: (x) => { return new ArticleData() { Authors = [OnlineCleaner.Clean(binding?.Authors?.Resolve(x) ?? "")], Name = OnlineCleaner.Clean(binding?.Title?.ResolveString(x) ?? ""), Categories = OnlineCleaner.Clean(binding?.Tags?.ResolveString(x) ?? "").Split(';') ?? [], Description = OnlineCleaner.Clean(binding?.Description?.ResolveString(x) ?? "") }; }); } public DownloadContext? GetTextRecord(HtmlWeb web, DataKey resKey, SharedDataDictionary sdd, IDocumentMetaData? metaData = null, ILogger? logger = null) { var res = sdd.Novels[resKey].ToRecord(sdd); // retrieves the novel data from the sdd var aggregator = res.AssociatedSource?.ToRecord(sdd); // retrieves the aggregator (novel web source) from the sdd if (aggregator is null) // ensure aggergator data was retrieved successfully return null; if (res is null) // ensure novel data was retrieved successfully return null; var template = sdd.Templates[aggregator.Resource.Key]; // gets the link generator for the specified aggregator // creates a generative enumerable of type link from 'template' var sle = SourceLinkEnumerable.FromGenerator(new DataBackedSourceLinkGenerator( template, res.Resource.TemplateInitialData)); return new DownloadContext(web, sle, transformer: (x) => { var resolved = aggregator.Bindings.Resolve(x); var articleData = new ArticleData() { Name = OnlineCleaner.Clean(resolved.Title), }; Dictionary, IDocumentMetaData> meta = []; meta.Add(ChapterKey, articleData); if (metaData is not null) meta.Add(BookKey, metaData); return new StringDocument(Path.GetRandomFileName(), OnlineCleaner.Clean(resolved.Content)) { MetaData = meta }; }, retryReporter: new Progress((x) => Console.WriteLine($"Retrying download ({x})")), downloadReporter: new Progress((x) => Console.WriteLine($"Downloaded ({x.Filename})")), asyncFailurePredicates: [ (x) => Task.FromResult(!x.DocumentNode.InnerHtml.Contains("
")) ], timeOut: TimeSpan.FromSeconds(15), downloadLogger: logger ); } } } }