80 lines
4.4 KiB
C#
80 lines
4.4 KiB
C#
using aeqw89.DataKeys;
|
|
using Beam.Dynamic;
|
|
using HtmlAgilityPack;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace Beam.Temporary.Cli {
|
|
partial interface IArchitecture {
|
|
private class MainArchitecture : IArchitecture {
|
|
public MainArchitecture() { }
|
|
|
|
public DataKey<IDocumentMetaData> ChapterKey { get; set; } = new("ma:chapter");
|
|
public DataKey<IDocumentMetaData> BookKey { get; set; } = new("ma:book");
|
|
|
|
public DownloadContext<IDocumentMetaData>? GetMeta(HtmlWeb web, DataKey<TextResource> pieceKey, SharedDataDictionary sdd, ILogger? logger = null) {
|
|
var piece = sdd.Novels[pieceKey].ToRecord(sdd); // retrieves novel data from the sdd
|
|
var auxiliary = piece.AssociatedMetaSource?.ToRecord(sdd); // retrieves novel aux data from the sdd
|
|
|
|
// null checks
|
|
if (auxiliary is null) // aux is required to get metadata
|
|
return null;
|
|
if (piece?.Resource?.MetaTemplateInitialData is null) // sanity check to avoid null warnings
|
|
return null;
|
|
|
|
// gets the link for the novel's metadata using the auxillary data retrieved from the sdd
|
|
var link = sdd.Templates[auxiliary.Resource.Key].GenerateLink(piece?.Resource?.MetaTemplateInitialData!);
|
|
var binding = auxiliary.Bindings;
|
|
|
|
return new DownloadContext<IDocumentMetaData>(web, [link], downloadLogger: logger, transformer: (x) => {
|
|
return new ArticleData() {
|
|
Authors = [OnlineCleaner.Clean(binding?.Authors?.Resolve(x) ?? "")],
|
|
Name = OnlineCleaner.Clean(binding?.Title?.ResolveString(x) ?? ""),
|
|
Categories = OnlineCleaner.Clean(binding?.Tags?.ResolveString(x) ?? "").Split(';') ?? [],
|
|
Description = OnlineCleaner.Clean(binding?.Description?.ResolveString(x) ?? "")
|
|
};
|
|
});
|
|
}
|
|
|
|
public DownloadContext<IDocument>? GetTextRecord(HtmlWeb web, DataKey<TextResource> resKey, SharedDataDictionary sdd, IDocumentMetaData? metaData = null, ILogger? logger = null) {
|
|
var res = sdd.Novels[resKey].ToRecord(sdd); // retrieves the novel data from the sdd
|
|
var aggregator = res.AssociatedSource?.ToRecord(sdd); // retrieves the aggregator (novel web source) from the sdd
|
|
|
|
if (aggregator is null) // ensure aggergator data was retrieved successfully
|
|
return null;
|
|
if (res is null) // ensure novel data was retrieved successfully
|
|
return null;
|
|
|
|
var template = sdd.Templates[aggregator.Resource.Key]; // gets the link generator for the specified aggregator
|
|
|
|
// creates a generative enumerable of type link from 'template'
|
|
var sle = SourceLinkEnumerable.FromGenerator(new DataBackedSourceLinkGenerator(
|
|
template, res.Resource.TemplateInitialData));
|
|
|
|
return new DownloadContext<IDocument>(web, sle,
|
|
transformer: (x) => {
|
|
var resolved = aggregator.Bindings.Resolve(x);
|
|
var articleData = new ArticleData() {
|
|
Name = OnlineCleaner.Clean(resolved.Title),
|
|
};
|
|
Dictionary<DataKey<IDocumentMetaData>, IDocumentMetaData> meta = [];
|
|
meta.Add(ChapterKey, articleData);
|
|
if (metaData is not null)
|
|
meta.Add(BookKey, metaData);
|
|
return new StringDocument(Path.GetRandomFileName(), OnlineCleaner.Clean(resolved.Content)) {
|
|
MetaData = meta
|
|
};
|
|
},
|
|
retryReporter: new Progress<int>((x) => Console.WriteLine($"Retrying download ({x})")),
|
|
downloadReporter: new Progress<IDocument>((x) => Console.WriteLine($"Downloaded ({x.Filename})")),
|
|
asyncFailurePredicates: [
|
|
(x) => Task.FromResult(!x.DocumentNode.InnerHtml.Contains("<div id=\"chapter-container\" class=\"chapter-content\" itemprop=\"description\">"))
|
|
],
|
|
timeOut: TimeSpan.FromSeconds(15),
|
|
downloadLogger: logger
|
|
);
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|