Enhance project metadata and refactor core classes

Updated project files for `Beam.Dynamic`, `Beam.Exports`, `Beam.Temporary.Cli`, and `Beam` to include additional metadata and specific package versions. Refactored `DataBindings` and `ResolvedBindings` to records, added a new `Text` property in `Binding.cs`, and introduced `ParseNumbers` in `OnlineCleaner`. New classes `PuppetContext` and `PuppetUnitDownloader` added for Playwright integration. Introduced `ImmutableState` struct and `UnitDownloaderBinary` class for improved download management. Updated tests in `UnitTest1.cs` for number localization. Added `Beam.Puppeteer` project to the solution.

Beam.Dynamic/Beam.Dynamic.csproj

@@ -4,19 +4,21 @@
     <TargetFramework>net9.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <Title>Beam Dynamic</Title>
+    <Authors>aeqw89</Authors>
+    <Company>qwsdcvghyu</Company>
+    <Product />
+    <Description>Beam utilities facilitating dynamic fetching of elements of webpages</Description>
+    <RepositoryUrl>https://github.com/qwsdcvghyu89/Beam</RepositoryUrl>
+	<PackageId>aeqw89.Beam.Dynamic</PackageId>
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\Beam\Beam.csproj" />
+    <PackageReference Include="aeqw89.DataKeys" Version="1.0.1" />
-  </ItemGroup>
+    <PackageReference Include="aeqw89.PersistentData" Version="1.0.0" />
-
+	<PackageReference Include="aeqw89.Beam" Version="1.0.0" />
-  <ItemGroup>
+	<PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
-    <Reference Include="aeqw89.DataKeys">
+    <PackageReference Include="Microsoft.Recognizers.Text.Number" Version="1.8.13" />
-      <HintPath>..\..\aeqw89.DataKeys\aeqw89.DataKeys\bin\Debug\net9.0\aeqw89.DataKeys.dll</HintPath>
-    </Reference>
-    <Reference Include="aeqw89.PersistentData">
-      <HintPath>..\..\aeqw89.PersistentData\aeqw89.PersistentData\bin\Release\net9.0\aeqw89.PersistentData.dll</HintPath>
-    </Reference>
   </ItemGroup>
 
 </Project>

Beam.Dynamic/Binding.cs

@@ -16,6 +16,7 @@ namespace Beam.Dynamic {
         public string? ArrayDelimiters { get; set; }
         public string? XPath { get; set; }
         public string? CssPath { get; set; }
+        public string? Text { get; set; }
         private IDataProvider? Provider_;
         public IDataProvider? Provider {
             get => Provider_;

Beam.Dynamic/DataBindings.cs

@@ -1,7 +1,7 @@
 using HtmlAgilityPack;
 
 namespace Beam.Dynamic {
-    public class DataBindings {
+    public record class DataBindings {
         public Binding? Title { get; set; }
         public Binding? Authors { get; set; }
         public Binding? Description { get; set; }
@@ -9,7 +9,7 @@ namespace Beam.Dynamic {
         public Binding? Language { get; set; }
         public Binding? Tags { get; set; }
 
-        public ResolvedBindings Resolve(HtmlDocument doc) {
+        public virtual ResolvedBindings Resolve(HtmlDocument doc) {
             return new ResolvedBindings() {
                 Title = Title?.Resolve(doc),
                 Authors = Authors?.Resolve(doc) ?? Array.Empty<string>(),
@@ -21,7 +21,7 @@ namespace Beam.Dynamic {
         }
     }
 
-    public class ResolvedBindings {
+    public record class ResolvedBindings {
         public string? Title { get; set; }
         public string[]? Authors { get; set; }
         public string? Description { get; set; }

Beam.Dynamic/IDataProvider.cs

@@ -1,8 +1,8 @@
 using HtmlAgilityPack;
 
 namespace Beam.Dynamic {
-    [System.Text.Json.Serialization.JsonDerivedType(typeof(ParagraphedContentDataProvider), 20)]
+    [System.Text.Json.Serialization.JsonDerivedType(typeof(ParagraphedContentDataProvider), "paragraphed-data-provider")]
-    [System.Text.Json.Serialization.JsonDerivedType(typeof(ListContentDataProvider), 21)]
+    [System.Text.Json.Serialization.JsonDerivedType(typeof(ListContentDataProvider), "list-data-provider")]
     public interface IDataProvider {
         public string Get(HtmlDocument document);
         public HtmlNode? GetNode(HtmlDocument document);

Beam.Dynamic/OnlineCleaner.cs

@@ -1,6 +1,8 @@
 using HtmlAgilityPack;
+using Microsoft.Recognizers.Text.Number;
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -25,6 +27,19 @@ namespace Beam.Dynamic {
             });
         }
 
+        public static List<int> ParseNumbers(string text, string from) {
+            var results = NumberRecognizer.RecognizeNumber(text, from, NumberOptions.None, false);
+            var resolved = results.Select((x) => {
+                if (x.Resolution.TryGetValue("value", out var value) && double.TryParse(value.ToString(), out var number))
+                    return (int?)number;
+                return null;
+            })
+                .Where((x) => x.HasValue).ToList();
+            if (resolved.Count == 0)
+                return [];
+            return resolved.Select((x) => x!.Value).ToList();
+        }
+
         public static string Clean(string? onlineText) {
             if (string.IsNullOrWhiteSpace(onlineText))
                 return "";

Beam.Exports/Beam.Exports.csproj

@@ -4,10 +4,17 @@
     <TargetFramework>net9.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+	  <Title>Beam.Exports</Title>
+	  <Authors>aeqw89</Authors>
+	  <Company>qwsdcvghyu</Company>
+	  <Description>Beam library that facilitates exporting different kinds of views for IDocuments</Description>
+	  <PackageProjectUrl>https://github.com/qwsdcvghyu89/Beam</PackageProjectUrl>
+	  <RepositoryUrl>https://github.com/qwsdcvghyu89/Beam</RepositoryUrl>
+	  <PackageId>aeqw89.Beam.Exports</PackageId>
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\Beam\Beam.csproj" />
+    <PackageReference Include="aeqw89.Beam" Version="1.0.0" />
   </ItemGroup>
 
 </Project>

Beam.Puppeteer/Beam.Puppeteer.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Playwright" Version="1.52.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Beam\Beam.csproj" />
+  </ItemGroup>
+
+</Project>

Beam.Puppeteer/PuppetedUnitDownloader.cs

@@ -0,0 +1,39 @@
+
+using HtmlAgilityPack;
+using Microsoft.Playwright;
+
+namespace Beam.Puppeteer {
+    public class PuppetContext(IPlaywright playwright, IBrowser browser) {
+        public IPlaywright Playwright { get; set; } = playwright;
+        public IBrowser Browser { get; set; } = browser;
+    }
+
+    public class PuppetUnitDownloader<T> : UnitDownloader<T> {
+        public PuppetContext Context { get; }
+
+        public PuppetUnitDownloader(PuppetContext pc, DownloadContext<T> context) 
+            : base(context.Web, context.AsyncTranformer, context.AsyncFailurePredicates) {
+            Context = pc;
+        }
+
+        protected override async Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
+            var page = await Context.Browser.NewPageAsync();
+            try {
+                var content = await page.ContentAsync();
+                await page.CloseAsync();
+
+                HtmlDocument doc = new();
+                doc.LoadHtml(content);
+                var transformed = await Transformer(doc);
+                if (FailurePredicates is null || !(await IsFailure(doc)))
+                    return (true, transformed);
+                return (false, default);
+            } catch (Exception) {
+                return (false, default);
+            } finally {
+                if (!page.IsClosed)
+                    await page.CloseAsync();
+            }
+        }
+    }
+}

Beam.Temporary.Cli/Beam.Temporary.Cli.csproj

@@ -5,9 +5,22 @@
     <TargetFramework>net9.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+
+	  <Title>Beam.Temporary.Cli</Title>
+	  <Authors>aeqw89</Authors>
+	  <Company>qwsdcvghyu</Company>
+	  <Description>A temporary CLI for Beam providing several useful mechanisms</Description>
+	  <PackageProjectUrl>https://github.com/qwsdcvghyu89/Beam</PackageProjectUrl>
+	  <RepositoryUrl>https://github.com/qwsdcvghyu89/Beam</RepositoryUrl>
+	  <PackageId>aeqw89.Beam.Temporary.Cli</PackageId>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="aeqw89.Beam.Exports" Version="1.0.0" />
+    <PackageReference Include="aeqw89.DataKeys" Version="1.0.1" />
+    <PackageReference Include="aeqw89.PersistentData" Version="1.0.0" />
+    <PackageReference Include="aeqw89.Beam" Version="1.0.0" />
+    <PackageReference Include="aeqw89.Beam.Dynamic" Version="1.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.1" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.1" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
@@ -16,19 +29,4 @@
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\Beam.Dynamic\Beam.Dynamic.csproj" />
-    <ProjectReference Include="..\Beam.Exports\Beam.Exports.csproj" />
-    <ProjectReference Include="..\Beam\Beam.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Reference Include="aeqw89.DataKeys">
-      <HintPath>..\..\aeqw89.DataKeys\aeqw89.DataKeys\bin\Debug\net9.0\aeqw89.DataKeys.dll</HintPath>
-    </Reference>
-    <Reference Include="aeqw89.PersistentData">
-      <HintPath>..\..\aeqw89.PersistentData\aeqw89.PersistentData\bin\Release\net9.0\aeqw89.PersistentData.dll</HintPath>
-    </Reference>
-  </ItemGroup>
-
 </Project>

Beam.Temporary.Cli/DownloadBuilder.cs

@@ -1,5 +1,6 @@
 using aeqw89.DataKeys;
 using Beam.Dynamic;
+using Beam;
 using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Generic;
@@ -24,17 +25,18 @@ namespace Beam.Temporary.Cli {
         public interface ILinkStage {
             ITransformStage WithLink();
             ITransformStage WithLinkGenerator();
+            ILinkStage WithRange(Range range);
         }
 
         public interface ITransformStage {
-            IContextStage WithTransformer(Func<DataBindings, HtmlTransformer<T>> factory);
+            IContextStage<U> WithTransformer<U>(Func<DataBindings, AsyncTransformer<T, U>> factory);
         }
 
-        public interface IContextStage {
+        public interface IContextStage<U> {
-            IContextStage Configure(Action<DownloadContextBuilder<T>> configure);
+            IContextStage<U> Configure(Action<DownloadContextBuilder<T>> configure);
-            IContextStage WithParallelism(int degree);
+            IContextStage<U> WithParallelism(int degree);
-            IContextStage WithTimeout(TimeSpan timeout);
+            IContextStage<U> WithTimeout(TimeSpan timeout);
-            IContextStage WithRetryReporter(IProgress<RetryReport> reporter);
+            IContextStage<U> WithRetryReporter(IProgress<RetryReport> reporter);
             DownloadEnumerable<T> Build();
         }
 
@@ -61,7 +63,7 @@ namespace Beam.Temporary.Cli {
                 initial = textRecord.Resource.MetaTemplateInitialData ?? throw new InvalidOperationException("Meta template data missing.");
             } else {
                 source = textRecord.AssociatedSource ?? throw new InvalidOperationException($"Text source missing for '{novelKey}'.");
-                initial = textRecord.Resource.TemplateInitialData ?? throw new InvalidOperationException("Template initial data missing.");
+                initial = textRecord.Resource.TemplateInitialData;
             }
 
             return (source, initial);
@@ -74,6 +76,10 @@ namespace Beam.Temporary.Cli {
             State Initial,
             BeamDataDictionary Data,
             DownloadContextBuilder<T> CtxBuilder) : ILinkStage {
+
+            private State? endState;
+            private bool linksFrozen = false;
+
             public ITransformStage WithLink() {
                 var link = Data.Templates[Source.Key].Builder.Build(Initial);
                 CtxBuilder.WithLinks(new[] { link });
@@ -85,28 +91,45 @@ namespace Beam.Temporary.Cli {
                 var generator = SourceLinkEnumerable.FromGenerator(new OrderedSourceLinkGenerator(
                     template.Builder,
                     new NumberedStateChanger(template.Factory.Behavior),
-                    Initial));
+                    Initial, endState));
                 CtxBuilder.WithLinks(generator);
+                linksFrozen = true;
                 return new TransformStage(Source, Data, CtxBuilder);
             }
+
+            public ILinkStage WithRange(Range range) {
+                if (linksFrozen)
+                    throw new InvalidOperationException($"WithRange must be called before WithLinkGenerator");
+                if (range.End.Value < range.Start.Value)
+                    throw new ArgumentOutOfRangeException(nameof(range), $" start must be < end");
+                var template = Data.Templates[Source.Key];
+                var stateChanger = new NumberedStateChanger(template.Factory.Behavior);
+                endState = Initial.Copy();
+                stateChanger.Apply(Initial, range.Start.Value - 1);
+                stateChanger.Apply(endState, range.End.Value - 1);
+                return this;
+            }
         }
 
         private sealed record TransformStage(
             WebResource Source,
             BeamDataDictionary Data,
             DownloadContextBuilder<T> CtxBuilder) : ITransformStage {
-            public IContextStage WithTransformer(Func<DataBindings, HtmlTransformer<T>> factory) {
+            public IContextStage WithTransformer<U>(Func<DataBindings, Func<object, T>> factory) {
                 var transformer = factory(Data.Bindings[Source.Bindings]);
-                CtxBuilder.WithTransformer(transformer);
+                return new ContextStage<U>(CtxBuilder, transformer);
-                return new ContextStage(CtxBuilder);
             }
         }
 
-        private sealed class ContextStage : IContextStage {
+        private sealed class ContextStage<U> : IContextStage {
             private readonly DownloadContextBuilder<T> _ctxBuilder;
+            private readonly Func<object, T> _transformer;
             private int _parallelism = 4;
 
-            public ContextStage(DownloadContextBuilder<T> ctxBuilder) => _ctxBuilder = ctxBuilder;
+            public ContextStage(DownloadContextBuilder<T> ctxBuilder, Func<object, T> transformer) {
+                _ctxBuilder = ctxBuilder;
+                _transformer = transformer;
+            }
 
             public IContextStage Configure(Action<DownloadContextBuilder<T>> configure) {
                 configure(_ctxBuilder);
@@ -134,7 +157,7 @@ namespace Beam.Temporary.Cli {
                         context,
                         ctx => new UnitFragmentDownloader<T>(
                             context.Web,
-                            context.AsyncTranformer,
+                            _transformer,
                             context.AsyncFailurePredicates,
                             _parallelism,
                             context.DownloadLogger),

Beam.Temporary.Cli/LinkCollection.cs

@@ -0,0 +1,13 @@
+using aeqw89.DataKeys;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Beam.Temporary.Cli {
+    internal class LinkCollection(DataKey<string> key, List<SourceLink> links) {
+        public DataKey<string> Key { get; set; } = key;
+        public List<SourceLink> Links { get; set; } = links;
+    }
+}

Beam.Temporary.Cli/NovelStatics.cs

@@ -50,8 +50,8 @@ namespace Beam.Temporary.Cli {
                 FriendlyName = "My House Of Horrors",
                 AssociatedSource = wdsAgg,
                 AssociatedMetaSource = wdsAux,
-                TemplateInitialData = new State(["24349", "2896325"]),
+                TemplateInitialData = new ImmutableState(["24349", "2896325"]),
-                MetaTemplateInitialData = new State(["24349"])
+                MetaTemplateInitialData = new ImmutableState(["24349"])
             };
 
             sdd.Novels.TryAdd(novel.Key, novel);

Beam.Temporary.Cli/Program.cs

@@ -49,6 +49,8 @@ namespace Beam.Temporary.Cli {
             NovelStatics.Define_WoDuShu_HouseOfHorrors(BeamData);
             ClassicTemplates.Register(BeamData);
 
+            await sharedContext.ForceSave();
+
             CancellationTokenSource cts = new();
 
             HtmlTransformer<IDocumentMetaData> ArticleDataTransformer(DataBindings? binding) => (x) => {
@@ -75,38 +77,23 @@ namespace Beam.Temporary.Cli {
             };
 
             var novel = new DataKey<TextResource>("novels:house_of_horrors");
-            var context_aux = Architecture.GetMeta(web, novel, BeamData, cts.Token);
-            context_aux.RetryReporter = new Progress<RetryReport>((x) => Console.WriteLine($"Failed. Trying again. {x.TryNumber}"));
-            var metaDownloader = new DownloadEnumerable<IDocumentMetaData>(
-                new SequentialFragmentDownloader<IDocumentMetaData>(
-                    context_aux,
-                    (c) => new UnitFragmentDownloader<IDocumentMetaData>(c.Web, c.AsyncTranformer, c.AsyncFailurePredicates, 4, logger),
-                    logger)
-                .UnwrapFragmented());
-            var metadata = (await metaDownloader.FirstAsync());
 
             var metadata2 = await DownloadBuilder<IDocumentMetaData>.FromMeta(novel, BeamData)
                 .WithLink()
                 .WithTransformer(ArticleDataTransformer)
                 .Configure((x) => x
+                    .WithDownloadLogger(logger)
                     .WithRetryReporter(new Progress<RetryReport>())
                     .WithTimeOut(TimeSpan.FromSeconds(15)))
                 .Build()
                 .FirstAsync();
 
-            var context = Architecture.GetTextRecord(web, novel, BeamData, metadata.Data, cts.Token);
-            context.DownloadReporter = new Progress<DownloadReport>((x) => Console.WriteLine(x));
-            var downloader = new DownloadEnumerable<IDocument>(
-                new SequentialFragmentDownloader<IDocument>(
-                    context, 
-                    (c) => new UnitFragmentDownloader<IDocument>(c.Web, c.AsyncTranformer, c.AsyncFailurePredicates, 4, logger),
-                    logger)
-                .UnwrapFragmented());
-
             var downloader2 = DownloadBuilder<IDocument>.FromText(novel, BeamData)
+                .WithRange(1..5)
                 .WithLinkGenerator()
                 .WithTransformer((x) => DocumentTransformer(x, metadata2.Data))
                 .Configure((x) => x
+                    .WithDownloadLogger(logger)
                     .WithDownloadReporter(new Progress<DownloadReport>((x) => logger.LogInformation(x.ToString())))
                     .WithTimeOut(TimeSpan.FromSeconds(15))
                 )
@@ -122,11 +109,16 @@ namespace Beam.Temporary.Cli {
                     continue;
                 if (meta is not ArticleData articleMetaData)
                     continue;
+                if (!download.Data.MetaData.TryGetValue(Architecture.BookKey, out var bookmeta))
+                    continue;
+                if (meta is not ArticleData bookMetaData)
+                    continue;
                 //Console.WriteLine($"Title: {data.Name}");
                 //Console.WriteLine($"Description: {data.Description}");
                 //Console.WriteLine($"Categories: {data.Categories.Aggregate((x, y) => $"{x}; {y}")}");
                 //Console.WriteLine($"Authors: {data.Authors.Aggregate((x,y) => $"{x}; {y}")}");
                 Console.WriteLine($"Chapter title: {articleMetaData.Name}");
+                Console.WriteLine($"Book title: {bookMetaData.Name}");
                 //Console.WriteLine($"Content: {download}");
 
                 //translationTasks.Add(Task.Run(async () => {

Beam.Temporary.Cli/StateChangerFactory.cs

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
 
 namespace Beam.Temporary.Cli {
     public class StateChangerFactory {
+        [JsonIgnore]
         public IStateChangeBehaviour Behavior => FactoryTable[StateChangerKey]();
         
         [JsonInclude]

Beam.Temporary.Cli/TextResource.cs

@@ -13,8 +13,8 @@ namespace Beam.Temporary.Cli {
         public string? FriendlyName { get; set; }
         public DataKey<WebResource>? AssociatedSource { get; set; }
         public DataKey<WebResource>? AssociatedMetaSource { get; set; }
-        public required State TemplateInitialData { get; set; }
+        public required ImmutableState TemplateInitialData { get; set; }
-        public State? MetaTemplateInitialData { get; set; }
+        public ImmutableState? MetaTemplateInitialData { get; set; }
 
         public TextResourceRecord ToRecord(BeamDataDictionary sdd) {
             return new(this,

Beam.Tests/OnlineCleaner.Tests.cs

@@ -0,0 +1,34 @@
+using Beam.Dynamic;
+using Microsoft.Recognizers.Text;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Beam.Tests {
+    public class OnlineCleanerTests {
+        [Fact]
+        public void Should_LocalizeArabic() {
+            const string test = "1234";
+
+            List<int> localized = OnlineCleaner.ParseNumbers(test,  Culture.English);
+
+            Assert.Single(localized);
+            Assert.Equal(1234, localized[0]);
+        }
+
+        [Fact]
+        public void Should_LocalizeIndian() {
+            const string test = "九一五";
+
+            List<int> localized = OnlineCleaner.ParseNumbers(test, Culture.Chinese);
+
+            Assert.Single(localized);
+            Assert.Equal(915, localized[0]);
+        }
+
+
+    }
+}

Beam.Tests/UnitTest1.cs

@@ -1,7 +1,12 @@
-namespace Beam.Tests {
+using System.Globalization;
+
+namespace Beam.Tests {
     public class UnitTest1 {
         [Fact]
         public void Test1() {
+            const string test = "九一五";
+
+            Assert.True(int.TryParse(test, CultureInfo.GetCultureInfo("zh-Hans").NumberFormat, out var number));
         }
     }
 }

Beam.sln

@@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Beam.Exports", "Beam.Export
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Beam.Tests", "Beam.Tests\Beam.Tests.csproj", "{E26800C2-0518-49E8-88DF-A0B6ED97D4AB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Beam.Puppeteer", "Beam.Puppeteer\Beam.Puppeteer.csproj", "{1A967563-D643-401D-A031-68DD43FACE8D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -39,8 +41,15 @@ Global
 		{E26800C2-0518-49E8-88DF-A0B6ED97D4AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E26800C2-0518-49E8-88DF-A0B6ED97D4AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{E26800C2-0518-49E8-88DF-A0B6ED97D4AB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1A967563-D643-401D-A031-68DD43FACE8D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1A967563-D643-401D-A031-68DD43FACE8D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1A967563-D643-401D-A031-68DD43FACE8D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1A967563-D643-401D-A031-68DD43FACE8D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {363CAF17-9E48-45B9-AA3F-78BB5E95DB0E}
+	EndGlobalSection
 EndGlobal

Beam/Beam.csproj

@@ -4,22 +4,26 @@
     <TargetFramework>net9.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+	  
+    <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
+    <Title>Beam</Title>
+    <Authors>aeqw89</Authors>
+    <Company>qwsdcvghyu</Company>
+    <Description>A library for downloading internet resources</Description>
+    <PackageProjectUrl>https://github.com/qwsdcvghyu89/Beam</PackageProjectUrl>
+    <RepositoryUrl>https://github.com/qwsdcvghyu89/Beam</RepositoryUrl>
+    <PackageId>aeqw89.Beam</PackageId>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentBuilder" Version="0.10.0">
+    <PackageReference Include="aeqw89.DataKeys" Version="1.0.1" />
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.1" />
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />
   </ItemGroup>
 
   <ItemGroup>
-    <Reference Include="aeqw89.DataKeys">
+
-      <HintPath>..\..\aeqw89.DataKeys\aeqw89.DataKeys\bin\Debug\net9.0\aeqw89.DataKeys.dll</HintPath>
-    </Reference>
   </ItemGroup>
 
 </Project>

Beam/DownloadContext.cs

@@ -5,12 +5,13 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-using FluentBuilder;
 using Microsoft.Extensions.Logging;
 
 namespace Beam {
     public delegate T HtmlTransformer<out T>(HtmlDocument doc);
+    public delegate Task<U> AsyncTransformer<in T, U>(T elem);
     public delegate Task<T> AsyncHtmlTransformer<T>(HtmlDocument doc);
+    public delegate Task<T> AsyncBinaryTransformer<T>(byte[] bin);
 
     public class DownloadContext<T> : IDisposable {
         private bool disposedValue;
@@ -19,8 +20,6 @@ namespace Beam {
             => DownloadContextBuilder<T>.FromContext(this);
 
         public HtmlWeb Web { get; }
-        public HtmlTransformer<T> Transformer { get; }
-        public AsyncHtmlTransformer<T> AsyncTranformer { get; }
         public IProgress<DownloadReport>? DownloadReporter { get; set; }
         public IProgress<RetryReport>? RetryReporter { get; set; }
         public AsyncDownloadFailurePredicate<HtmlDocument>?[]? AsyncFailurePredicates { get;  }
@@ -33,8 +32,6 @@ namespace Beam {
         public DownloadContext(HtmlWeb web,
                                IEnumerable<SourceLink> links,
                                CancellationToken cancellationToken = default,
-                               HtmlTransformer<T>? transformer = null,
-                               AsyncHtmlTransformer<T>? asyncTransformer = null,
                                IProgress<DownloadReport>? downloadReporter = null,
                                IProgress<RetryReport>? retryReporter = null,
                                AsyncDownloadFailurePredicate<HtmlDocument>?[]? asyncFailurePredicates = null,
@@ -46,16 +43,6 @@ namespace Beam {
             Web = web;
             Links = links;
             CancellationToken = cancellationToken;
-            if (transformer is null && asyncTransformer is null)
-                throw new ArgumentException($"Either {nameof(transformer)} or {nameof(asyncTransformer)} must be not null.");
-
-            Transformer = transformer!;
-            AsyncTranformer = asyncTransformer!;
-            if (transformer is null && asyncTransformer is not null)
-                Transformer = (x) => asyncTransformer(x).Result;
-            if (asyncTransformer is null && transformer is not null)
-                AsyncTranformer = (x) => Task.FromResult(transformer(x)); 
-
             DownloadReporter = downloadReporter;
             RetryReporter = retryReporter;
             AsyncFailurePredicates = asyncFailurePredicates;

Beam/DownloadContextBuilder.cs

@@ -10,8 +10,6 @@ namespace Beam {
 
     public class DownloadContextBuilder<T> {
         private HtmlWeb _web;
-        private HtmlTransformer<T> _transformer;
-        private AsyncHtmlTransformer<T> _asyncTransformer;
         private IProgress<DownloadReport>? _downloadReporter;
         private IProgress<RetryReport>? _retryReporter;
         private AsyncDownloadFailurePredicate<HtmlDocument>?[]? _asyncFailurePredicates;
@@ -34,16 +32,6 @@ namespace Beam {
             return this;
         }
 
-        public DownloadContextBuilder<T> WithTransformer(HtmlTransformer<T> transformer) {
-            _transformer = transformer;
-            return this;
-        }
-
-        public DownloadContextBuilder<T> WithAsyncTransformer(AsyncHtmlTransformer<T> asyncTransformer) {
-            _asyncTransformer = asyncTransformer;
-            return this;
-        }
-
         public DownloadContextBuilder<T> WithDownloadReporter(IProgress<DownloadReport> downloadReporter) {
             _downloadReporter = downloadReporter;
             return this;
@@ -91,8 +79,6 @@ namespace Beam {
                 web: _web,
                 links: _links,
                 cancellationToken: _cancellationToken,
-                transformer: _transformer,
-                asyncTransformer: _asyncTransformer,
                 downloadReporter: _downloadReporter,
                 retryReporter: _retryReporter,
                 asyncFailurePredicates: _asyncFailurePredicates,
@@ -115,8 +101,6 @@ namespace Beam {
             return new DownloadContextBuilder<T>(existing.Web)
                 .WithLinks(existing.Links)
                 .WithCancellationToken(existing.CancellationToken)
-                .WithTransformer(existing.Transformer)
-                .WithAsyncTransformer(existing.AsyncTranformer)
                 .WithDownloadReporter(existing.DownloadReporter!)
                 .WithRetryReporter(existing.RetryReporter!)
                 .WithAsyncFailurePredicates(existing.AsyncFailurePredicates ?? Array.Empty<AsyncDownloadFailurePredicate<HtmlDocument>>())

Beam/ImmutableState.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Beam {
+    public readonly struct ImmutableState(object[] state) {
+        readonly object[] state = state;
+
+        public readonly Span<object> GetState() => state;
+
+        public readonly State Copy()
+            => new((object[])state.Clone());
+
+        public readonly object this[Index i] {
+            get => state[i];
+        }
+
+        public static implicit operator State(ImmutableState state)
+            => state.Copy();
+    }
+}

Beam/OrderedSourceLinkGenerator.cs

@@ -13,15 +13,17 @@ namespace Beam {
     public class OrderedSourceLinkGenerator : IEnumerator<SourceLink> {
         public SourceLinkBuilder Builder { get; set; }
         public NumberedStateChanger Behaviour { get; }
-        private State InitialState;
         
+        private State? EndState;
+        private State InitialState;
 
         public OrderedSourceLinkGenerator(SourceLinkBuilder builder, NumberedStateChanger behaviour, params object[] initialState) 
             : this(builder, behaviour, new State(initialState)) { }
-        public OrderedSourceLinkGenerator(SourceLinkBuilder builder, NumberedStateChanger behaviour, State initialState) {
+        public OrderedSourceLinkGenerator(SourceLinkBuilder builder, NumberedStateChanger behaviour, State initialState, State? endState = null) {
             Builder = builder;
             Behaviour = behaviour;
-            InitialState = initialState;
+            InitialState = initialState.Copy();
+            EndState = endState?.Copy();
             State = InitialState.Copy();
 
             Reset();
@@ -37,8 +39,15 @@ namespace Beam {
         }
 
         public bool MoveNext() {
+            if (!Current.HasValue || (EndState is not null && State.GetState().SequenceEqual(EndState.GetState()))) {
+                Current = SourceLink.InvalidLink;
+                return false;
+            }
             Behaviour.Apply(State, 1);
             Current = Builder.Build(State);
+            if (!Current.HasValue || (EndState is not null && State.GetState().SequenceEqual(EndState.GetState()))) {
+                return false;
+            }
             return Current.HasValue;
         }
 

Beam/SequentialDownloader.cs

@@ -35,13 +35,16 @@ namespace Beam {
             
             links.Add(new Ordered<string>(LinksEnumerator.Current.Link.ToString(), LastOrder++));
 
-            while (LinksEnumerator.MoveNext() && links.Count < idealLinkCount)
+            while (LinksEnumerator.MoveNext() && LinksEnumerator.Current != SourceLink.InvalidLink && links.Count < idealLinkCount)
                 links.Add(new Ordered<string>(LinksEnumerator.Current.Link.ToString(), LastOrder++));
             if (links.Count == 0) {
                 Logger?.LogInformation("Out of links!");
                 return false;
             }
 
+            if (links.Any((x) => x.Data == SourceLink.InvalidLink.Link.ToString()))
+                return false;
+
             var (result, downloadedT) = await unit.TryDownload(
                 links.ToArray(),
                 Context.CancellationToken, 

Beam/UnitDownloaderBinary.cs

@@ -0,0 +1,73 @@
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Beam {
+    /// <summary>
+    /// A download-managing class that retrieves binary data through <see cref="HttpClient"/>,
+    /// applies an <see cref="AsyncBinaryTransformer{T}"/>, and supports failure detection
+    /// plus exponential-back-off retries. Safe to instantiate per request.
+    /// </summary>
+    public class UnitDownloaderBinary<T>(
+        HttpClient client,
+        AsyncBinaryTransformer<T> transformer,
+        AsyncDownloadFailurePredicate<HttpResponseMessage>?[]? failurePredicates = null)
+        : IUnitDownloader<T> {
+        public HttpClient Client { get; } = client;
+        public virtual AsyncBinaryTransformer<T> Transformer { get; } = transformer;
+        public virtual AsyncDownloadFailurePredicate<HttpResponseMessage>?[]? FailurePredicates { get; } = failurePredicates;
+
+        public int LinksPerDownload { get; } = 1;
+
+        /// <summary>Runs all configured failure predicates in parallel on the raw HTTP response.</summary>
+        protected virtual async Task<bool> IsFailure(HttpResponseMessage response) {
+            if (FailurePredicates is null) return false;
+
+            var failed = false;
+            await Parallel.ForEachAsync(FailurePredicates, async (pred, _) => {
+                if (failed || pred is null) return;
+                if (await pred(response)) failed = true;
+            });
+            return failed;
+        }
+
+        /// <summary>One attempt without retries or back-off.</summary>
+        protected virtual async Task<(bool Success, T? Result)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
+            try {
+                using var response = await Client.GetAsync(link, HttpCompletionOption.ResponseHeadersRead, ct);
+                if (!response.IsSuccessStatusCode) return (false, default);
+
+                if (await IsFailure(response)) return (false, default);
+
+                var bytes = await response.Content.ReadAsByteArrayAsync(ct);
+                return (true, await Transformer(bytes));
+            } catch {
+                return (false, default);
+            }
+        }
+
+        public async Task<(bool, T?)> TryDownload(
+            Ordered<string>[] link,
+            CancellationToken ct,
+            int maximumRetryCount = 7,
+            IProgress<RetryReport>? tryProgress = null) {
+            if (link.Length == 0) return (false, default);
+
+            T? result = default;
+            var attempt = 0;
+
+            while (attempt < maximumRetryCount) {
+                ct.ThrowIfCancellationRequested();
+
+                (var success, result) = await TryDownloadWithNoRetries(link[0].Data, ct);
+                if (success && result is not null) return (true, result);
+
+                ++attempt;
+                tryProgress?.Report(new RetryReport(attempt, link[0].Data));
+                await Task.Delay((int)Math.Pow(2, attempt) * 1000, ct);
+            }
+
+            return (false, result);
+        }
+    }
+}

Beam/UnitFragmentDownloader.cs

@@ -13,11 +13,12 @@ namespace Beam {
                                       AsyncHtmlTransformer<T> transformer,
                                       AsyncDownloadFailurePredicate<HtmlDocument>?[]? failurePredicate = null,
                                       int fragmentSize = 4,
-                                      ILogger? logger = null) {
+                                      ILogger? logger = null,
+                                      IUnitDownloader<T>? internalDownloader = null) {
             Web = web;
             Transformer = transformer;
             FailurePredicate = failurePredicate;
-            UnitDownloader = new UnitDownloader<T>(Web, Transformer, FailurePredicate);
+            UnitDownloader = internalDownloader ?? new UnitDownloader<T>(Web, Transformer, FailurePredicate);
             LinksPerDownload = fragmentSize;
             Logger = logger;
         }
@@ -28,7 +29,7 @@ namespace Beam {
         public int LinksPerDownload { get; set; }
         public ILogger? Logger { get; set;  }
 
-        private readonly UnitDownloader<T> UnitDownloader;
+        private readonly IUnitDownloader<T> UnitDownloader;
 
         async Task<(bool, Fragment<Ordered<T>>?)> IUnitDownloader<Fragment<Ordered<T>>>.TryDownload(Ordered<string>[] link, CancellationToken ct, int maximumRetryCount, IProgress<RetryReport>? tryProgress) {
             Fragment<Ordered<T>> fragment = new Fragment<Ordered<T>>(link.Length);