Add project files.

2025-04-19 20:47:58 +03:00
parent 9e14d137ae
commit bfdcdb1f3b
66 changed files with 2394 additions and 0 deletions
@@ -0,0 +1,79 @@
+using aeqw89.DataKeys;
+using Beam.Dynamic;
+using HtmlAgilityPack;
+using Microsoft.Extensions.Logging;
+
+namespace Beam.Temporary.Cli {
+    partial interface IArchitecture {
+        private class MainArchitecture : IArchitecture {
+            public MainArchitecture() { }
+
+            public DataKey<IDocumentMetaData> ChapterKey { get; set; } = new("ma:chapter");
+            public DataKey<IDocumentMetaData> BookKey { get; set; } = new("ma:book");
+
+            public DownloadContext<IDocumentMetaData>? GetMeta(HtmlWeb web, DataKey<TextResource> pieceKey, SharedDataDictionary sdd, ILogger? logger = null) {
+                var piece = sdd.Novels[pieceKey].ToRecord(sdd); // retrieves novel data from the sdd
+                var auxiliary = piece.AssociatedMetaSource?.ToRecord(sdd); // retrieves novel aux data from the sdd
+                
+                // null checks
+                if (auxiliary is null) // aux is required to get metadata
+                    return null;
+                if (piece?.Resource?.MetaTemplateInitialData is null) // sanity check to avoid null warnings
+                    return null;
+
+                // gets the link for the novel's metadata using the auxillary data retrieved from the sdd
+                var link = sdd.Templates[auxiliary.Resource.Key].GenerateLink(piece?.Resource?.MetaTemplateInitialData!);
+                var binding = auxiliary.Bindings; 
+
+                return new DownloadContext<IDocumentMetaData>(web, [link], downloadLogger: logger, transformer: (x) => {
+                    return new ArticleData() {
+                        Authors = [OnlineCleaner.Clean(binding?.Authors?.Resolve(x) ?? "")],
+                        Name = OnlineCleaner.Clean(binding?.Title?.ResolveString(x) ?? ""),
+                        Categories = OnlineCleaner.Clean(binding?.Tags?.ResolveString(x) ?? "").Split(';') ?? [],
+                        Description = OnlineCleaner.Clean(binding?.Description?.ResolveString(x) ?? "")
+                    };
+                });
+            }
+
+            public DownloadContext<IDocument>? GetTextRecord(HtmlWeb web, DataKey<TextResource> resKey, SharedDataDictionary sdd, IDocumentMetaData? metaData = null, ILogger? logger = null) {
+                var res = sdd.Novels[resKey].ToRecord(sdd); // retrieves the novel data from the sdd
+                var aggregator = res.AssociatedSource?.ToRecord(sdd); // retrieves the aggregator (novel web source) from the sdd
+
+                if (aggregator is null) // ensure aggergator data was retrieved successfully
+                    return null;
+                if (res is null) // ensure novel data was retrieved successfully
+                    return null;
+
+                var template = sdd.Templates[aggregator.Resource.Key]; // gets the link generator for the specified aggregator
+
+                // creates a generative enumerable of type link from 'template'
+                var sle = SourceLinkEnumerable.FromGenerator(new DataBackedSourceLinkGenerator(
+                    template, res.Resource.TemplateInitialData));
+
+                return new DownloadContext<IDocument>(web, sle,
+                    transformer: (x) => {
+                        var resolved = aggregator.Bindings.Resolve(x);
+                        var articleData = new ArticleData() {
+                            Name = OnlineCleaner.Clean(resolved.Title),
+                        };
+                        Dictionary<DataKey<IDocumentMetaData>, IDocumentMetaData> meta = [];
+                        meta.Add(ChapterKey, articleData);
+                        if (metaData is not null)
+                            meta.Add(BookKey, metaData);
+                        return new StringDocument(Path.GetRandomFileName(), OnlineCleaner.Clean(resolved.Content)) {
+                            MetaData = meta
+                        };
+                    },
+                    retryReporter: new Progress<int>((x) => Console.WriteLine($"Retrying download ({x})")),
+                    downloadReporter: new Progress<IDocument>((x) => Console.WriteLine($"Downloaded ({x.Filename})")),
+                    asyncFailurePredicates: [
+                        (x) => Task.FromResult(!x.DocumentNode.InnerHtml.Contains("<div id=\"chapter-container\" class=\"chapter-content\" itemprop=\"description\">"))
+                    ],
+                    timeOut: TimeSpan.FromSeconds(15),
+                    downloadLogger: logger
+                );
+            }
+
+        }
+    }
+}