using aeqw89.PersistentData; using aeqw89.DataKeys; using Beam.Dynamic; using HtmlAgilityPack; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; using Beam.Temporary.Cli.Templates.Classic; using Beam.Exports; using System.Diagnostics; using Beam.Models; using Beam.Stealth; namespace Beam.Temporary.Cli { internal class Program { public static JsonSerializerOptions ConversionOptions { get; internal set; } = new(); public static BeamDataContext BeamData { get; set; } = []; public static IArchitecture Architecture = IArchitecture.Default; const string BeamDataPath = "data/.dat"; static async Task Main(string[] args) { ConversionOptions.Converters.AddPersistentDataRequiredConverters(); ConversionOptions.WriteIndented = true; var web = new HtmlWeb(); var lf = LoggerFactory.Create((x) => x .AddConsole() .SetMinimumLevel(LogLevel.Trace) ); ILogger logger = lf .CreateLogger("Program"); await using var sharedContext = await DataDictionaryContext.Create( BeamDataPath, false, DataKind.Shared, logger, ConversionOptions ); BeamData = sharedContext.Data; BeamData.Clear(); NovelStatics.Define_YeBiQuge(BeamData); NovelStatics.Define_YeBiQuge_LordOfMysteries(BeamData); NovelStatics.Define_KuaiShu5(BeamData); NovelStatics.Define_KuaiShu5_LordOfMysteries(BeamData); ClassicTemplates.Register(BeamData); await sharedContext.ForceSave(); BeamData = sharedContext.Data; // need to refresh instance after forced save! CancellationTokenSource cts = new(); using var config = StealthConfig.Create(true, null, TimeSpan.FromMinutes(2), Browser.Chrome, lf.CreateLogger()); var unit = new StealthUnitPageDownloader(new(), config, (x) => { return Task.CompletedTask; }, x => Task.FromResult(x)); var (success, result) = await unit.TryDownload([new("https://duckduckgo.com/?t=ffab&q=C%23+stealth+headless+browser&ia=web", 0)], default); if (success) logger?.LogInformation("Success! Downloaded '{}'", result?.DocumentNode.Name); else logger?.LogError("Failed to download!"); Console.WriteLine(result?.DocumentNode.OuterHtml); //var novelResDict = new DataKey("kuaishu5:novels:lord_of_the_mysteries"); //var metadata2 = await DownloadBuilder.FromResource(novelResDict, nameof(TableOfContentsData), BeamData) // .WithLink() // .WithTransformer(CommonTransformers.TableOfContentsTransformer) // .Configure((x) => x // .WithDownloadLogger(logger) // .WithRetryReporter(new Progress()) // .WithTimeOut(TimeSpan.FromSeconds(15))) // .Build() // .FirstAsync(); //if (metadata2.Data.PagesLinks is null || metadata2.Data.PagesLinks.Length == 0) // Debugger.Break(); //var pageLinks = DownloadBuilder.FromScratch() // .WithLinks(metadata2.Data.PagesLinks) // .WithTransformer(CommonTransformers.TableOfContentsTransformer(BeamData.Bindings[BeamData.Resources[BeamData.ResourceDictionaries[novelResDict].Resources[nameof(TableOfContentsData)]].Bindings])) // .Configure(x => x // .WithDownloadLogger(logger) // .WithRetryReporter(new Progress()) // .WithTimeOut(TimeSpan.FromSeconds(15))) // .Build(); //var links = (await pageLinks // .ToListAsync()) // .Where(x => x?.Data?.ContentLinks is not null) // .SelectMany(x => x.Data.ContentLinks!) // .DistinctBy(x => x.Link.AbsoluteUri); //var downloader = DownloadBuilder.FromScratch() // .WithLinks(links) // .WithTransformer(CommonTransformers.DocumentTransformer(BeamData.Bindings[BeamData.Resources[BeamData.ResourceDictionaries[novelResDict].Resources[nameof(IDocument)]].Bindings])) // .Configure(x => x // .WithDownloadLogger(logger) // .WithRetryReporter(new Progress(x => logger?.LogWarning("Retrying download {} for the {} time", x.Link, x.TryNumber))) // .WithTimeOut(TimeSpan.FromSeconds(15))) // .WithParallelism(4) // .UseFragments() // .Build(); //HashSet> downloaded = []; //try { // await foreach (var download in downloader) { // logger?.LogInformation("Downloaded chapter with order={}", download.Order); // try { // downloaded.Add(download); // } catch (Exception e) { // logger?.LogError(e, "Unknown error occurred"); // } // } //} catch (Exception e) { // logger?.LogError(e, "Uncaught error detected!"); //} finally { // logger?.LogInformation("Done with loop, downloaded {}", downloaded.Count); // try { // string serialized = JsonSerializer.Serialize(downloaded.Select(x => new { x.Order, x.Data.Content, x.Data.MetaData }).ToArray(), ConversionOptions); // System.IO.File.WriteAllText("lordOfTheMysteries.json", serialized); // } catch (Exception e) { // logger?.LogInformation(e, "Failed to serialize chapters"); // } //} //var downloader2 = DownloadBuilder.FromText(novel, BeamData) // .WithRange(1..5) // .WithLinkGenerator() // .WithTransformer((x) => CommonTransformers.DocumentTransformer(x, metadata2.Data)) // .Configure((x) => x // .WithDownloadLogger(logger) // .WithDownloadReporter(new Progress((x) => logger.LogInformation(x.ToString()))) // .WithTimeOut(TimeSpan.FromSeconds(15)) // ) // .Build(); //List>> translationTasks = []; //List> documents = []; //await foreach (var download in downloader2.Take(10)) { // if (!download.Data.MetaData.TryGetValue(Architecture.ChapterKey, out var meta)) // continue; // if (meta is not ArticleData articleMetaData) // continue; // if (!download.Data.MetaData.TryGetValue(Architecture.BookKey, out var bookmeta)) // continue; // if (meta is not ArticleData bookMetaData) // continue; // //Console.WriteLine($"Title: {data.Name}"); // //Console.WriteLine($"Description: {data.Description}"); // //Console.WriteLine($"Categories: {data.Categories.Aggregate((x, y) => $"{x}; {y}")}"); // //Console.WriteLine($"Authors: {data.Authors.Aggregate((x,y) => $"{x}; {y}")}"); // Console.WriteLine($"Chapter title: {articleMetaData.Name}"); // Console.WriteLine($"Book title: {bookMetaData.Name}"); // //Console.WriteLine($"Content: {download}"); // //translationTasks.Add(Task.Run(async () => { // // logger.LogInformation("Beginning translation {} task for {}", download.Order, articleMetaData.Name); // // var ret = new Ordered(await QuickAndDirtyJanitor.TranslateAsync(download.Data), download.Order); // // logger.LogInformation("Finished translation {} task for {}", download.Order, articleMetaData.Name); // // return ret; // //})); //} //documents = (await Task.WhenAll(translationTasks)).ToList(); //string testDir = Path.Combine("txt", Path.GetRandomFileName()); //Directory.CreateDirectory(testDir); //int len = documents.MaxBy((x) => x.Order)?.Order ?? -1; //foreach (var document in documents.OrderBy((x) => x.Order)) { // document.Data.MetaData.TryGetValue(Architecture.ChapterKey, out var chapterMetaData); // Dictionary linkButtons = new(); // if (document.Order != 0) // linkButtons.Add("Previous", $"{document.Order - 1}.html"); // if (document.Order != len) // linkButtons.Add("Next", $"{document.Order + 1}.html"); // new HtmlExporter(document.Data, chapterMetaData as ArticleData, linkButtons).Write(Path.Combine(testDir, $"{document.Order}.html")); //} Console.ReadKey(); //foreach (var download in documents.OrderBy((x) => x.Order)) { // if (download.Data.TryGetTaggedMetaData(Architecture.ChapterKey, out var meta)) // Console.WriteLine($"{download.Order}:{meta.Name}"); //} //string[] templates = new DataKey[] { // HtmlBook.Keys.ContentPage, // HtmlBook.Keys.NoContentPage, // HtmlBook.Keys.TitlePage, // HtmlBook.Keys.StylesPage, //}.Select( // (x) => BeamData.Files.ReadToString(x.WithNamespace("aeqw89:files:templates:classic")) //).ToArray(); //HtmlBook book = new( // bookname: Path.Combine(Path.GetRandomFileName(), "I Alone Level Up"), // new CssData(), // new ArticleData(), // new HtmlBookTemplates() { // ContentPageTemplate = templates[0], // NoContentTemplate = templates[1], // TitlePageTemplate = templates[2], // CssTemplate = templates[3], // }, // documents: documents.Select((x) => x.Data).ToList() //); //book.Update(); //Console.WriteLine("One variable!"); } } }