Refactor downloaders to use generic options and unify logic

Replaces specialized binary and HTML downloaders with a generic, options-driven UnitDownloader and UnitFragmentDownloader pattern. Introduces UnitDownloaderOptions and builder classes for flexible configuration, updates interfaces and method signatures to support progress reporting, and removes redundant binary-specific classes. Updates Playwright and Stealth downloaders to use the new generic base, and adds improved error handling and reporting. Also updates dependency versions and makes minor API consistency improvements across the Fluent and Models layers.
2025-09-29 21:27:56 +10:00
parent 8e60109f5e
commit 2958a26e4f
30 changed files with 621 additions and 422 deletions
@@ -8,7 +8,7 @@
    </PropertyGroup>

    <ItemGroup>
-      <PackageReference Include="aeqw89.DataKeys" Version="2.1.0" />
+      <PackageReference Include="aeqw89.DataKeys" Version="2.1.1" />
      <PackageReference Include="aeqw89.PersistentData" Version="1.4.5" />
      <PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
    </ItemGroup>
@@ -1,3 +1,6 @@
 namespace Beam.Abstractions;

-public interface IDownloadReport { }
+public interface IDownloadReport {
+    long BytesDownloaded { get; init; }
+    long? BytesRemaining { get; init; }
+}
@@ -3,5 +3,5 @@ namespace Beam.Abstractions;

 public interface IUnitDownloader<T> {
    public int LinksPerDownload { get; }
-    public Task<(bool, T?)> TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount = 7, IProgress<IRetryReport>? tryProgress = null);
+    public Task<(bool, T?)> TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount = 7, IProgress<IDownloadReport>? downProgress = null, IProgress<IRetryReport>? tryProgress = null);
 }
@@ -14,6 +14,7 @@

    <ItemGroup>
      <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.9" />
+      <PackageReference Include="System.Linq.Async" Version="6.0.3" />
    </ItemGroup>

 </Project>
@@ -58,6 +58,7 @@ namespace Beam.Downloaders {
            var (result, downloadedT) = await unit.TryDownload(
                links.ToArray(),
                Context.CancellationToken, 
+                downProgress: Context.DownloadReporter,
                tryProgress: Context.RetryReporter);

            if (!result) {
@@ -1,6 +1,7 @@
 using Beam.Abstractions;
 using Beam.Models;
 using HtmlAgilityPack;
+using File = System.IO.File;

 namespace Beam.Downloaders {
    /// <summary>
@@ -10,34 +11,104 @@ namespace Beam.Downloaders {
    /// <param name="web"></param>
    /// <param name="transformer"></param>
    /// <param name="failurePredicate"></param>
-    public class UnitDownloader<T>(HtmlWeb web, AsyncTransformer<HtmlDocument, T> transformer, AsyncDownloadFailurePredicate<HtmlDocument>?[]? failurePredicate = null) : IUnitDownloader<T> {
-        public HtmlWeb Web { get; } = web;
-        public virtual AsyncTransformer<HtmlDocument, T> Transformer { get; } = transformer;
-        public virtual AsyncDownloadFailurePredicate<HtmlDocument>?[]? FailurePredicates { get; } = failurePredicate;
+    public class UnitDownloader<RawType, OutType>(UnitDownloaderOptions<RawType, OutType> options) : IUnitDownloader<OutType> where RawType : IDocument {
+        public UnitDownloaderOptions<RawType, OutType> Options { get; } = options;
+        public HttpClient Client => Options.Client;
+        public virtual AsyncTransformer<RawType, OutType> Transformer => Options.AsyncTransformer;
+
+        public virtual AsyncDownloadFailurePredicate<RawType>?[]? FailurePredicates =>
+            Options?.FailurePredicateOptions?.AsyncDownloadFailurePredicates;

        public int LinksPerDownload { get; } = 1;

-        protected virtual async Task<bool> IsFailure(HtmlDocument doc) {
-            if (FailurePredicates is null)
-                return false;
-            var failed = false;
-            await Parallel.ForEachAsync(FailurePredicates, async (x, ct) => {
-                if (failed == true)
-                    return;
-                if (x is null)
-                    return;
-                if (await x(doc))
-                    failed = true;
-            });
+        protected virtual async Task DownloadToStream(string url, int bufferSize, Stream destinationStream, IProgress<IDownloadReport> progress,
+            CancellationToken ct) {

-            return failed;
+            var stream = await Client.GetStreamAsync(url, ct);
+            byte[] buffer = new byte[bufferSize];
+            int inBuffer = 0;
+            long downloaded = 0;
+            while ((inBuffer = stream.Read(buffer)) > 0) {
+                downloaded += inBuffer;
+                await destinationStream.WriteAsync(buffer.AsMemory(0, inBuffer), ct);
+                progress?.Report(new DownloadReport() {
+                    BytesDownloaded = inBuffer,
+                    BytesRemaining = stream.Length - downloaded 
+                });
+                
+                ct.ThrowIfCancellationRequested();
+            }
        }

-        protected virtual async Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
+        protected virtual async Task DownloadToFile(string url, int bufferSize, string path,
+            IProgress<IDownloadReport> progress, CancellationToken ct) {
+
+            if (!Directory.Exists(Path.GetDirectoryName(path)))
+                throw new InvalidOperationException(
+                    string.Format(Exceptions.Exceptions.unit_download_directory_nonexistant, path));
+            await using var file = File.OpenWrite(path);
+            await DownloadToStream(url, bufferSize, file, progress, ct);
+        }
+
+        protected virtual async Task<ByteDocument> DownloadToMemory(string url, int bufferSize,
+            IProgress<IDownloadReport> progress, CancellationToken ct) {
+
+            await using var ms = new MemoryStream();
+            await DownloadToStream(url, bufferSize, ms, progress, ct);
+            if (!ms.TryGetBuffer(out var bytes))
+                throw new Exception(Exceptions.Exceptions.unit_download_invalid_memory_stream);
+            return new ByteDocument(url, bytes);
+        }
+        
+        protected virtual async Task<bool> IsFailure(RawType doc, CancellationToken ct) {
+            if (FailurePredicates is null)
+                return false;
+            if (!(Options?.FailurePredicateOptions?.ProcessInParallel ?? false))
+                foreach (var pred in FailurePredicates) {
+                    if (pred is null)
+                        continue;
+                    if (await pred(doc))
+                        return true;
+                }
+            else {
+                var failed = false;
+                await Parallel.ForEachAsync(FailurePredicates, new ParallelOptions() {
+                        MaxDegreeOfParallelism = Options?.FailurePredicateOptions?.ParallelThreads ?? 4,
+                        CancellationToken = ct
+                    },
+                    async (predicate, token) => {
+                        if (token.IsCancellationRequested)
+                            return;
+                        if (failed)
+                            return;
+                        if (predicate == null)
+                            return;
+                        if (await predicate(doc))
+                            Interlocked.CompareExchange(ref failed, true, false);
+                    }
+                );
+                return failed;
+            }
+
+            return false;
+        }
+
+        protected virtual async Task<RawType> _Download(string link, IProgress<IDownloadReport> progress, CancellationToken ct) {
+            if (Options.DownloadFolder is not null && this is UnitDownloader<StringDocument, OutType>) {
+                var path = Path.Combine(Options.DownloadFolder, Path.GetRandomFileName());
+                await DownloadToFile(link, Options.BufferSize, path, progress, ct);
+                return (RawType)(object)new StringDocument(link, path);
+            }
+            if (this is UnitDownloader<ByteDocument, OutType>) {
+                return (RawType)(object)(await DownloadToMemory(link, Options.BufferSize, progress, ct));
+            }
+            throw new NotSupportedException(Exceptions.Exceptions.unit_downloader_limited_support);
+        }
+        
+        protected virtual async Task<(bool, OutType?)> Transform(RawType download, CancellationToken ct) {
            try {
-                var html = await Web.LoadFromWebAsync(link, ct);
-                if (FailurePredicates is null || !(await IsFailure(html)))
-                    return (true, await Transformer(html));
+                if (FailurePredicates is null || !(await IsFailure(download, ct)))
+                    return (true, await Transformer(download));
                else
                    return (false, default);
            } catch(Exception) {
@@ -45,23 +116,26 @@ namespace Beam.Downloaders {
            }
        }

-        public async Task<(bool, T?)> TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount = 7, IProgress<IRetryReport>? tryProgress = null) {
+        public async Task<(bool, OutType?)> TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount = 7, IProgress<IDownloadReport>? downProgress = null, IProgress<IRetryReport>? tryProgress = null) {
            if (link.Length == 0)
                return (false, default);

-            T? doc = default;
+            downProgress ??= new Progress<IDownloadReport>();
+            
+            OutType? ot = default;
            int tryCount = 0;
            while (tryCount < maximumRetryCount) {
                ct.ThrowIfCancellationRequested();
-                (var success, doc) = await TryDownloadWithNoRetries(link[0].Data, ct);
-                if (success && doc != null)
-                    return (true, doc);
+                var rt = await _Download(link[0].Data, downProgress, ct);
+                (var success, ot) = await Transform(rt, ct);
+                if (success && ot != null)
+                    return (true, ot);
                ++tryCount;
                tryProgress?.Report(new RetryReport(tryCount, link[0].Data));
                await Task.Delay((int)Math.Pow(2, tryCount) * 1000);
            }

-            return (false, doc);
+            return (false, ot);
        }
    }
 }
@@ -1,74 +0,0 @@
-using Beam.Abstractions;
-using Beam.Models;
-
-namespace Beam.Downloaders {
-    /// <summary>
-    /// A download-managing class that retrieves binary data through <see cref="HttpClient"/>,
-    /// applies an <see cref="AsyncBinaryTransformer{T}"/>, and supports failure detection
-    /// plus exponential-back-off retries. Safe to instantiate per request.
-    /// </summary>
-    public class UnitDownloaderBinary<T>(
-        HttpClient client,
-        AsyncTransformer<ByteDocument, T> transformer,
-        AsyncDownloadFailurePredicate<ByteDocument>?[]? failurePredicates = null)
-        : IUnitDownloader<T> {
-        public HttpClient Client { get; } = client;
-        public virtual AsyncTransformer<ByteDocument, T> Transformer { get; } = transformer;
-        public virtual AsyncDownloadFailurePredicate<ByteDocument>?[]? FailurePredicates { get; } = failurePredicates;
-
-        public int LinksPerDownload { get; } = 1;
-
-        /// <summary>Runs all configured failure predicates in parallel on the raw HTTP response.</summary>
-        protected virtual async Task<bool> IsFailure(ByteDocument response) {
-            if (FailurePredicates is null) return false;
-
-            var failed = false;
-            await Parallel.ForEachAsync(FailurePredicates, async (pred, ct) => {
-                if (failed || pred is null) return;
-                if (await pred(response))
-                    failed = true;
-            });
-            return failed;
-        }
-
-        /// <summary>One attempt without retries or back-off.</summary>
-        protected virtual async Task<(bool Success, T? Result)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
-            try {
-                using var response = await Client.GetAsync(link, HttpCompletionOption.ResponseHeadersRead, ct);
-                if (!response.IsSuccessStatusCode) return (false, default);
-
-                var bytes = await response.Content.ReadAsByteArrayAsync(ct);
-                var doc = new ByteDocument(link, bytes);
-                if (await IsFailure(doc)) return (false, default);
-
-                return (true, await Transformer(doc));
-            } catch {
-                return (false, default);
-            }
-        }
-
-        public async Task<(bool, T?)> TryDownload(
-            IOrdered<string>[] link,
-            CancellationToken ct,
-            int maximumRetryCount = 7,
-            IProgress<IRetryReport>? tryProgress = null) {
-            if (link.Length == 0) return (false, default);
-
-            T? result = default;
-            var attempt = 0;
-
-            while (attempt < maximumRetryCount) {
-                ct.ThrowIfCancellationRequested();
-
-                (var success, result) = await TryDownloadWithNoRetries(link[0].Data, ct);
-                if (success && result is not null) return (true, result);
-
-                ++attempt;
-                tryProgress?.Report(new RetryReport(attempt, link[0].Data));
-                await Task.Delay((int)Math.Pow(2, attempt) * 1000, ct);
-            }
-
-            return (false, result);
-        }
-    }
-}
@@ -0,0 +1,198 @@
+using Beam.Models;
+
+namespace Beam.Downloaders;
+
+public record class UnitDownloaderOptions<RawType, OutType> {
+    public HttpClient Client { get; init; } = new();
+    
+    public FailurePredicateOptions<RawType>? FailurePredicateOptions { get; init; }
+    public FragmentOptions? FragmentOptions { get; init; }
+    public required AsyncTransformer<RawType, OutType> AsyncTransformer { get; init; }
+    public string? DownloadFolder { get; init; } = null;
+    public int BufferSize { get; init; } = 80 * 1024; // 80kb
+}
+
+public record class FailurePredicateOptions<RawType> {
+    public required AsyncDownloadFailurePredicate<RawType>?[]? AsyncDownloadFailurePredicates { get; init; }
+    public bool ProcessInParallel { get; init; } = false;
+    public int? ParallelThreads { get; init; }
+}
+
+public record class FragmentOptions {
+    public required int FragmentSize { get; init; }
+    public bool DownloadInParallel { get; init; } = false;
+    public int? ParallelThreads { get; init; }
+}
+
+
+    // ---------- UnitDownloaderOptions Builder ----------
+    public sealed class UnitDownloaderOptionsBuilder<TRaw, TOut>
+    {
+        private HttpClient _client = new HttpClient();
+        private FailurePredicateOptions<TRaw>? _failureOptions;
+        private FragmentOptions? _fragmentOptions;
+        private AsyncTransformer<TRaw, TOut>? _asyncTransformer;
+        private string? _downloadFolder = null;
+        private int _bufferSize = 80 * 1024;
+
+        public UnitDownloaderOptionsBuilder<TRaw, TOut> WithClient(HttpClient client)
+        {
+            _client = client ?? throw new System.ArgumentNullException(nameof(client));
+            return this;
+        }
+
+        public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFailurePredicateOptions(FailurePredicateOptions<TRaw>? options)
+        {
+            _failureOptions = options;
+            return this;
+        }
+
+        public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFailurePredicates(System.Action<FailurePredicateOptionsBuilder<TRaw>> configure)
+        {
+            if (configure == null) throw new System.ArgumentNullException(nameof(configure));
+            var b = new FailurePredicateOptionsBuilder<TRaw>();
+            configure(b);
+            _failureOptions = b.Build();
+            return this;
+        }
+
+        public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFragmentOptions(FragmentOptions? options)
+        {
+            _fragmentOptions = options;
+            return this;
+        }
+
+        public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFragments(System.Action<FragmentOptionsBuilder> configure)
+        {
+            if (configure == null) throw new System.ArgumentNullException(nameof(configure));
+            var b = new FragmentOptionsBuilder();
+            configure(b);
+            _fragmentOptions = b.Build();
+            return this;
+        }
+
+        public UnitDownloaderOptionsBuilder<TRaw, TOut> WithAsyncTransformer(AsyncTransformer<TRaw, TOut> transformer)
+        {
+            _asyncTransformer = transformer ?? throw new System.ArgumentNullException(nameof(transformer));
+            return this;
+        }
+
+        public UnitDownloaderOptionsBuilder<TRaw, TOut> WithDownloadFolder(string? downloadFolder)
+        {
+            _downloadFolder = downloadFolder;
+            return this;
+        }
+
+        public UnitDownloaderOptionsBuilder<TRaw, TOut> WithBufferSize(int bytes)
+        {
+            if (bytes <= 0) throw new System.ArgumentOutOfRangeException(nameof(bytes));
+            _bufferSize = bytes;
+            return this;
+        }
+
+        public UnitDownloaderOptions<TRaw, TOut> Build()
+        {
+            if (_asyncTransformer == null)
+                throw new System.InvalidOperationException("AsyncTransformer must be provided.");
+
+            return new UnitDownloaderOptions<TRaw, TOut>
+            {
+                Client = _client,
+                FailurePredicateOptions = _failureOptions,
+                FragmentOptions = _fragmentOptions,
+                AsyncTransformer = _asyncTransformer,
+                DownloadFolder = _downloadFolder,
+                BufferSize = _bufferSize
+            };
+        }
+    }
+
+    // ---------- FailurePredicateOptions Builder ----------
+    public sealed class FailurePredicateOptionsBuilder<TRaw>
+    {
+        private readonly System.Collections.Generic.List<AsyncDownloadFailurePredicate<TRaw>?> _predicates =
+            new System.Collections.Generic.List<AsyncDownloadFailurePredicate<TRaw>?>();
+        private bool _processInParallel = false;
+        private int? _parallelThreads = null;
+
+        public FailurePredicateOptionsBuilder<TRaw> WithPredicate(AsyncDownloadFailurePredicate<TRaw>? predicate)
+        {
+            _predicates.Add(predicate);
+            return this;
+        }
+
+        public FailurePredicateOptionsBuilder<TRaw> WithPredicates(System.Collections.Generic.IEnumerable<AsyncDownloadFailurePredicate<TRaw>?> predicates)
+        {
+            if (predicates == null) throw new System.ArgumentNullException(nameof(predicates));
+            _predicates.AddRange(predicates);
+            return this;
+        }
+
+        public FailurePredicateOptionsBuilder<TRaw> WithPredicates(params AsyncDownloadFailurePredicate<TRaw>?[] predicates)
+        {
+            _predicates.Clear();
+            if (predicates != null) _predicates.AddRange(predicates);
+            return this;
+        }
+
+        public FailurePredicateOptionsBuilder<TRaw> WithProcessInParallel(bool value = true)
+        {
+            _processInParallel = value;
+            return this;
+        }
+
+        public FailurePredicateOptionsBuilder<TRaw> WithParallelThreads(int? threads)
+        {
+            if (threads.HasValue && threads.Value <= 0)
+                throw new System.ArgumentOutOfRangeException(nameof(threads));
+            _parallelThreads = threads;
+            return this;
+        }
+
+        public FailurePredicateOptions<TRaw> Build()
+        {
+            var arr = _predicates.Count == 0 ? [] : _predicates.ToArray();
+            return new FailurePredicateOptions<TRaw>
+            {
+                AsyncDownloadFailurePredicates = arr,
+                ProcessInParallel = _processInParallel,
+                ParallelThreads = _parallelThreads
+            };
+        }
+    }
+
+    // ---------- FragmentOptions Builder ----------
+    public sealed class FragmentOptionsBuilder {
+        private int? _fragmentSize;
+        private bool _downloadInParallel = false;
+        private int? _parallelThreads = null;
+
+        public FragmentOptionsBuilder WithFragmentSize(int bytes) {
+            if (bytes <= 0) throw new System.ArgumentOutOfRangeException(nameof(bytes));
+            _fragmentSize = bytes;
+            return this;
+        }
+
+        public FragmentOptionsBuilder WithDownloadInParallel(bool value = true) {
+            _downloadInParallel = value;
+            return this;
+        }
+
+        public FragmentOptionsBuilder WithParallelThreads(int? threads) {
+            if (threads.HasValue && threads.Value <= 0)
+                throw new System.ArgumentOutOfRangeException(nameof(threads));
+            _parallelThreads = threads;
+            return this;
+        }
+
+        public FragmentOptions Build() {
+            if (!_fragmentSize.HasValue)
+                throw new System.InvalidOperationException("FragmentSize must be provided.");
+
+            return new FragmentOptions {
+                FragmentSize = _fragmentSize.Value,
+                DownloadInParallel = _downloadInParallel,
+                ParallelThreads = _parallelThreads
+            };
+        }
+    }
@@ -5,55 +5,39 @@ using HtmlAgilityPack;
 using Microsoft.Extensions.Logging;

 namespace Beam.Downloaders {
-    public class UnitFragmentDownloader<T> : IUnitDownloader<Fragment<Ordered<T>>> {
-        public UnitFragmentDownloader(HtmlWeb web,
-                                      AsyncTransformer<HtmlDocument, T> transformer,
-                                      AsyncDownloadFailurePredicate<HtmlDocument>?[]? failurePredicate = null,
-                                      int fragmentSize = 4,
-                                      ILogger? logger = null,
-                                      IUnitDownloader<T>? internalDownloader = null) {
-            Web = web;
-            Transformer = transformer;
-            FailurePredicate = failurePredicate;
-            UnitDownloader = internalDownloader ?? new UnitDownloader<T>(Web, Transformer, FailurePredicate);
-            LinksPerDownload = fragmentSize;
-            Logger = logger;
-        }
+    public class UnitFragmentDownloader<RawType, OutType>(UnitDownloaderOptions<RawType, OutType> options,
+        IUnitDownloader<OutType>? internalDownloader = null) : IUnitDownloader<Fragment<Ordered<OutType>>> where RawType : IDocument {

-        public HtmlWeb Web { get; }
-        public AsyncTransformer<HtmlDocument, T> Transformer { get; }
-        public AsyncDownloadFailurePredicate<HtmlDocument>?[]? FailurePredicate { get; }
+        public UnitDownloaderOptions<RawType, OutType> Options { get; } = options;
        public int LinksPerDownload { get; set; }
-        public ILogger? Logger { get; set;  }
+        private IUnitDownloader<OutType> UnitDownloader { get;  } = internalDownloader ?? new UnitDownloader<RawType, OutType>(options);

-        private readonly IUnitDownloader<T> UnitDownloader;
-
-        async Task<(bool, Fragment<Ordered<T>>?)> IUnitDownloader<Fragment<Ordered<T>>>.TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount, IProgress<IRetryReport>? tryProgress) {
-            Fragment<Ordered<T>> fragment = new Fragment<Ordered<T>>(link.Length);
-            if (!Fragment<Ordered<T>>.TryAcquireUpdater(fragment, out var updater))
+        async Task<(bool, Fragment<Ordered<OutType>>?)> IUnitDownloader<Fragment<Ordered<OutType>>>.TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount, IProgress<IDownloadReport>? downProgress, IProgress<IRetryReport>? tryProgress) {
+            Fragment<Ordered<OutType>> fragment = new Fragment<Ordered<OutType>>(link.Length);
+            if (!Fragment<Ordered<OutType>>.TryAcquireUpdater(fragment, out var updater))
                throw new AssertionException(Exceptions.Exceptions.fragment_locked);
            bool isFailure = false;
            await Parallel.ForEachAsync(link, async (x, pct) => {
                pct.ThrowIfCancellationRequested();
                ct.ThrowIfCancellationRequested();
-                var (result, downloadedT) = await UnitDownloader.TryDownload([x], ct, maximumRetryCount, tryProgress);
+                if (isFailure)
+                    return;
+                var (result, downloadedT) = await UnitDownloader.TryDownload([x], ct, maximumRetryCount, downProgress, tryProgress);
                if (!result) {
                    Interlocked.Exchange(ref isFailure, true);
-                    Logger?.LogError("Failed to retrieve {0} order={1}", x.Data, x.Order);
                    return;
                }
                if (downloadedT == null) {
                    Interlocked.Exchange(ref isFailure, true);
-                    Logger?.LogCritical("Failed to retrieve {0} order={1}", x.Data, x.Order);
                    return;
                }
-                updater(new Ordered<T>(downloadedT, x.Order));
+                updater(new Ordered<OutType>(downloadedT, x.Order));
            });

            if (!isFailure)
-                Fragment<Ordered<T>>.SetComplete(fragment, true);
+                Fragment<Ordered<OutType>>.SetComplete(fragment, true);

-            Fragment<Ordered<T>>.TryReleaseUpdater(fragment, updater);
+            Fragment<Ordered<OutType>>.TryReleaseUpdater(fragment, updater);
            
            return (!isFailure, fragment);

@@ -1,74 +0,0 @@
-using Beam.Abstractions;
-using Beam.Exceptions;
-using Beam.Models;
-using Microsoft.Extensions.Logging;
-
-namespace Beam.Downloaders {
-    /// <summary>
-    /// Groups multiple binary downloads into a single Fragment, applying
-    /// failure detection and exponential-back-off retries for each link.
-    /// </summary>
-    public class UnitFragmentDownloaderBinary<T>
-        : IUnitDownloader<Fragment<Ordered<T>>> {
-        public UnitFragmentDownloaderBinary(HttpClient client,
-                                            AsyncTransformer<ByteDocument, T> transformer,
-                                            AsyncDownloadFailurePredicate<ByteDocument>?[]? failurePredicate = null,
-                                            int fragmentSize = 4,
-                                            ILogger? logger = null,
-                                            IUnitDownloader<T>? internalDownloader = null) {
-            Client = client;
-            Transformer = transformer;
-            FailurePredicate = failurePredicate;
-            UnitDownloader = internalDownloader
-                              ?? new UnitDownloaderBinary<T>(Client, Transformer, FailurePredicate);
-            LinksPerDownload = fragmentSize;
-            Logger = logger;
-        }
-
-        public HttpClient Client { get; }
-        public AsyncTransformer<ByteDocument, T> Transformer { get; }
-        public AsyncDownloadFailurePredicate<ByteDocument>?[]? FailurePredicate { get; }
-        public int LinksPerDownload { get; set; }
-        public ILogger? Logger { get; set; }
-
-        private readonly IUnitDownloader<T> UnitDownloader;
-
-        async Task<(bool, Fragment<Ordered<T>>?)> IUnitDownloader<Fragment<Ordered<T>>>.TryDownload(
-            IOrdered<string>[] link,
-            CancellationToken ct,
-            int maximumRetryCount,
-            IProgress<IRetryReport>? tryProgress) {
-            var fragment = new Fragment<Ordered<T>>(link.Length);
-            if (!Fragment<Ordered<T>>.TryAcquireUpdater(fragment, out var updater))
-                throw new AssertionException(Exceptions.Exceptions.fragment_locked);
-
-            var isFailure = false;
-
-            await Parallel.ForEachAsync(link, async (orderedLink, pct) => {
-                pct.ThrowIfCancellationRequested();
-                ct.ThrowIfCancellationRequested();
-
-                var (success, downloaded) =
-                    await UnitDownloader.TryDownload([orderedLink],
-                                                     ct,
-                                                     maximumRetryCount,
-                                                     tryProgress);
-
-                if (!success || downloaded is null) {
-                    Interlocked.Exchange(ref isFailure, true);
-                    Logger?.LogError("Failed to retrieve {Link} order={Order}",
-                                     orderedLink.Data, orderedLink.Order);
-                    return;
-                }
-
-                updater(new Ordered<T>(downloaded, orderedLink.Order));
-            });
-
-            if (!isFailure)
-                Fragment<Ordered<T>>.SetComplete(fragment, true);
-
-            Fragment<Ordered<T>>.TryReleaseUpdater(fragment, updater);
-            return (!isFailure, fragment);
-        }
-    }
-}
@@ -6,7 +6,7 @@
        <Nullable>enable</Nullable>
    </PropertyGroup>
    <ItemGroup>
-        <PackageReference Include="aeqw89.DataKeys" Version="2.0.1" />
+        <PackageReference Include="aeqw89.DataKeys" Version="2.1.1" />
        <PackageReference Include="aeqw89.PersistentData" Version="1.4.5" />
        <PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
        <PackageReference Include="Microsoft.Recognizers.Text.Number" Version="1.8.13" />
@@ -157,5 +157,32 @@ namespace Beam.Exceptions {
                return ResourceManager.GetString("state_change_error", resourceCulture);
            }
        }
+        
+        /// <summary>
+        ///   Looks up a localized string similar to Could not open a filestream to a non-existant directory &apos;{0}&apos;..
+        /// </summary>
+        public static string unit_download_directory_nonexistant {
+            get {
+                return ResourceManager.GetString("unit_download_directory_nonexistant", resourceCulture);
+            }
+        }
+        
+        /// <summary>
+        ///   Looks up a localized string similar to The memory stream was created with an invisible inner byte array..
+        /// </summary>
+        public static string unit_download_invalid_memory_stream {
+            get {
+                return ResourceManager.GetString("unit_download_invalid_memory_stream", resourceCulture);
+            }
+        }
+        
+        /// <summary>
+        ///   Looks up a localized string similar to The base unit downloader class only supports RawType&apos;s of string and ByteDocument..
+        /// </summary>
+        public static string unit_downloader_limited_support {
+            get {
+                return ResourceManager.GetString("unit_downloader_limited_support", resourceCulture);
+            }
+        }
    }
 }
@@ -51,4 +51,13 @@
    <data name="resource_definition_invalid_states_count" xml:space="preserve">
        <value>There must be at least one state in resource definition.</value>
    </data>
+    <data name="unit_download_directory_nonexistant" xml:space="preserve">
+        <value>Could not open a filestream to a non-existant directory '{0}'.</value>
+    </data>
+    <data name="unit_download_invalid_memory_stream" xml:space="preserve">
+        <value>The memory stream was created with an invisible inner byte array.</value>
+    </data>
+    <data name="unit_downloader_limited_support" xml:space="preserve">
+        <value>The base unit downloader class only supports RawType's of string and ByteDocument.</value>
+    </data>
 </root>
@@ -6,12 +6,11 @@
        <Nullable>enable</Nullable>
    </PropertyGroup>
    <ItemGroup>
-        <PackageReference Include="aeqw89.DataKeys" Version="2.0.1" />
+        <PackageReference Include="aeqw89.DataKeys" Version="2.1.1" />
        <PackageReference Include="aeqw89.PersistentData" Version="1.4.5" />
        <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.9" />
        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.9" />
        <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.9" />
-        <PackageReference Include="System.Linq.Async" Version="6.0.1" />
    </ItemGroup>
    <ItemGroup>
        <ProjectReference Include="..\Beam.Data\Beam.Data.csproj" />
@@ -8,13 +8,14 @@ using Beam.Downloaders;

 namespace Beam.Fluent;

-internal sealed class ContextStage<RawType, OutType> : IContextStage<RawType, OutType> {
+internal sealed class ContextStage<RawType, OutType> : IContextStage<RawType, OutType> where RawType : IDocument {
    private readonly DownloadContextBuilder<RawType> _ctxBuilder;
    private readonly AsyncTransformer<RawType, OutType> _transformer;
    private FragmentMode _fragmentMode = FragmentMode.Single;
    private Channel _channel = Channel.Plain;
    private readonly ContentKind _contentKind;
    private int _parallelism = 4;
+    private UnitDownloaderOptionsBuilder<RawType, OutType> _optionsBuilder = new();

    // ──────────────── playwright ────────────────
    private PlaywrightAsyncManipulator? _playwrightManipulator = null;
@@ -31,12 +32,15 @@ internal sealed class ContextStage<RawType, OutType> : IContextStage<RawType, Ou
        _ctxBuilder = ctxBuilder;
        _transformer = transformer;
        _contentKind = transformer switch {
-            AsyncTransformer<HtmlDocument, OutType> => ContentKind.Html,
+            AsyncTransformer<StringDocument, OutType> => ContentKind.File,
            AsyncTransformer<ByteDocument, OutType> => ContentKind.Binary,
            _ => throw new ArgumentException(string.Format(Exceptions.Exceptions.fluent_unsupported_transformer,
                transformer.GetType()
                    .AsUniqueName()))
        };
+
+        _optionsBuilder
+            .WithAsyncTransformer(_transformer);
    }

    public IContextStage<RawType, OutType> Configure(Action<DownloadContextBuilder<RawType>> configure) {
@@ -44,6 +48,12 @@ internal sealed class ContextStage<RawType, OutType> : IContextStage<RawType, Ou
        return this;
    }

+    public IContextStage<RawType, OutType> ConfigureUnitDownloaderOptions(
+        Action<UnitDownloaderOptionsBuilder<RawType, OutType>> configure) {
+        configure(_optionsBuilder);
+        return this;
+    }
+
    public IContextStage<RawType, OutType> WithParallelism(int degree) {
        _parallelism = Math.Max(1, degree);
        return this;
@@ -108,14 +118,14 @@ internal sealed class ContextStage<RawType, OutType> : IContextStage<RawType, Ou
                   string.Format(Exceptions.Exceptions.fluent_type_conversion_failure,
                       o?.GetType().AsUniqueName() ?? "null", typeof(T).AsUniqueName()));

-        AsyncTransformer<HtmlDocument, OutType> HtmlTransformer()
-            => To<AsyncTransformer<HtmlDocument, OutType>>(_transformer);
+        AsyncTransformer<StringDocument, OutType> FileTransformer()
+            => To<AsyncTransformer<StringDocument, OutType>>(_transformer);

        AsyncTransformer<ByteDocument, OutType> ByteTransformer()
            => To<AsyncTransformer<ByteDocument, OutType>>(_transformer);

-        AsyncDownloadFailurePredicate<HtmlDocument>[] HtmlFailurePredicates()
-            => To<AsyncDownloadFailurePredicate<HtmlDocument>[]>(context.AsyncFailurePredicates);
+        AsyncDownloadFailurePredicate<StringDocument>[] FileFailurePredicates()
+            => To<AsyncDownloadFailurePredicate<StringDocument>[]>(context.AsyncFailurePredicates);

        AsyncDownloadFailurePredicate<ByteDocument>[] ByteFailurePredicates()
            => To<AsyncDownloadFailurePredicate<ByteDocument>[]>(context.AsyncFailurePredicates);
@@ -125,82 +135,39 @@ internal sealed class ContextStage<RawType, OutType> : IContextStage<RawType, Ou

        #endregion

+        if (context.AsyncFailurePredicates is not null)
+            _optionsBuilder
+                .WithFailurePredicates(x => x.WithPredicates(context.AsyncFailurePredicates));
+        var options = _optionsBuilder
+            .WithClient(context.Client)
+            .Build();
+
        return (_channel, _fragmentMode, _contentKind) switch {
-            // ────────────────  fragmented HTML  ────────────────
-            (Channel.Plain, FragmentMode.Fragmented, ContentKind.Html)
-                => new UnitFragmentDownloader<OutType>(
-                    context.Web,
-                    HtmlTransformer(),
-                    HtmlFailurePredicates(),
-                    _parallelism,
-                    context.DownloadLogger),
-            // ────────────────  fragmented binary  ────────────────
-            (Channel.Plain, FragmentMode.Fragmented, ContentKind.Binary)
-                => new UnitFragmentDownloaderBinary<OutType>(
-                    context.Client,
-                    ByteTransformer(),
-                    ByteFailurePredicates(),
-                    _parallelism,
-                    context.DownloadLogger),
-            // ────────────────  single HTML  ────────────────
-            (Channel.Plain, FragmentMode.Single, ContentKind.Html)
-                => new UnitDownloader<OutType>(
-                    context.Web,
-                    HtmlTransformer(),
-                    HtmlFailurePredicates()),
-            // ────────────────  single binary  ────────────────
-            (Channel.Plain, FragmentMode.Single, ContentKind.Binary)
-                => new UnitDownloaderBinary<OutType>(
-                    context.Client,
-                    ByteTransformer(),
-                    ByteFailurePredicates()),
-            // ────────────────  single playwright binary  ────────────────
-            (Channel.Playwright, FragmentMode.Single, ContentKind.Binary)
-                => new PlaywrightUnitDownloader<OutType>(
-                    context.Client,
-                    EnsureExists(_playwrightManipulator),
-                    ByteTransformer(),
-                    ByteFailurePredicates()
-                ),
-            // ────────────────  single playwrigt HTML  ────────────────
-            (Channel.Playwright, FragmentMode.Single, ContentKind.Html)
-                => new PlaywrightUnitPageDownloader<OutType>(
-                    context.Web,
-                    EnsureExists(_playwrightManipulator),
-                    HtmlTransformer(),
-                    HtmlFailurePredicates()),
-            // ────────────────  single stealth HTML  ────────────────
-            (Channel.Stealth, FragmentMode.Single, ContentKind.Html)
-                => new StealthUnitPageDownloader<OutType>(
-                    context.Web,
-                    EnsureExists(_stealthConfig),
-                    EnsureExists(_stealthManipulator),
-                    HtmlTransformer(),
-                    HtmlFailurePredicates()),
-            // ────────────────  single stealth binary  ────────────────
+            // ────────────────  fragmented  ────────────────
+            (Channel.Plain, FragmentMode.Fragmented, _)
+                => new UnitFragmentDownloader<RawType, OutType>(options),
+            // ────────────────  single      ────────────────
+            (Channel.Plain, FragmentMode.Single, _)
+                => new UnitDownloader<RawType, OutType>(options),
+            // ────────────────  single playwright ────────────────
+            (Channel.Playwright, FragmentMode.Single, _)
+                => new PlaywrightUnitDownloader<RawType, OutType>(options, EnsureExists(_playwrightManipulator)),
+            // ────────────────  single stealth file ────────────────
+            (Channel.Stealth, FragmentMode.Single, ContentKind.File)
+                => new StealthUnitPageDownloader<RawType, OutType>(options, EnsureExists(_stealthConfig), EnsureExists(_stealthManipulator)),
+            // ────────────────  single stealth binary ────────────────
            (Channel.Stealth, FragmentMode.Single, ContentKind.Binary)
-                => new StealthUnitDownloader<OutType>(
-                    context.Client,
+                => new StealthUnitDownloader<RawType, OutType>(options, EnsureExists(_stealthConfig), EnsureExists(_stealthManipulator)),
+            // ────────────────  fragment stealth file  ────────────────
+            (Channel.Stealth, FragmentMode.Fragmented, ContentKind.File)
+                => new StealthFragmentPageDownloader<RawType, OutType>(options,
                    EnsureExists(_stealthConfig),
-                    EnsureExists(_stealthManipulator),
-                    ByteTransformer(),
-                    ByteFailurePredicates()),
-            // ────────────────  fragment stealth HTML  ────────────────
-            (Channel.Stealth, FragmentMode.Fragmented, ContentKind.Html)
-                => new StealthFragmentPageDownloader<OutType>(
-                    context.Web,
-                    EnsureExists(_stealthConfig),
-                    EnsureExists(_stealthManipulator),
-                    HtmlTransformer(),
-                    HtmlFailurePredicates()),
+                    EnsureExists(_stealthManipulator)),
            // ────────────────  fragment stealth binary  ────────────────
            (Channel.Stealth, FragmentMode.Fragmented, ContentKind.Binary)
-                => new StealthFragmentDownloader<OutType>(
-                    context.Client,
+                => new StealthFragmentDownloader<RawType, OutType>(options,
                    EnsureExists(_stealthConfig),
-                    EnsureExists(_stealthManipulator),
-                    ByteTransformer(),
-                    ByteFailurePredicates()),
+                    EnsureExists(_stealthManipulator)),
            _ => throw new Exception(string.Format(Exceptions.Exceptions.fluent_unsupported_pattern,
                $"({_channel}, {_fragmentMode}, {_contentKind})")),
        };
@@ -12,6 +12,6 @@ public enum Channel {
 }

 public enum ContentKind {
-    Html,
+    File,
    Binary
 }
@@ -1,10 +1,11 @@
 using System.Collections.Concurrent;
 using System.Text.Json;
+using Beam.Abstractions;
 using Beam.Models;

 namespace Beam.Fluent;

-internal sealed class DownloadStage<RawType, OutType>(DownloadEnumerable<OutType> download) : IDownloadStage<RawType, OutType> {
+internal sealed class DownloadStage<RawType, OutType>(DownloadEnumerable<OutType> download) : IDownloadStage<RawType, OutType> where RawType : IDocument {
    private IAsyncEnumerable<Ordered<OutType>> _download = download;

    public DownloadEnumerable<OutType> AsAsyncEnumerable() {
@@ -1,4 +1,5 @@
 using aeqw89.DataKeys;
+using Beam.Abstractions;
 using Beam.Data;
 using Beam.Downloaders;
 using Beam.Dynamic;
@@ -7,13 +8,13 @@ using Beam.Models;
 namespace Beam.Fluent;

 public static class FluentDownload {
-    public static ITransformStage<RawType, OutType> Links<RawType, OutType>(params IEnumerable<string> links) {
+    public static ITransformStage<RawType, OutType> Links<RawType, OutType>(params IEnumerable<string> links) where RawType : IDocument {
        return new TransformStage<RawType, OutType>(new DownloadContextBuilder<RawType>()
            .WithLinks(links));
    }

    public static ITransformStage<RawType, OutType>
-        ResourceDefinition<RawType, OutType>(ResourceDefinition definition) {
+        ResourceDefinition<RawType, OutType>(ResourceDefinition definition) where RawType : IDocument {
        if (definition.Location.States.Count == 0)
            throw new ArgumentException(Exceptions.Exceptions.resource_definition_invalid_states_count, nameof(definition));
        var linkGenerator = new OrderedLinkGenerator(definition.Location.Segments, (NumberedStateChanger)definition.Location.StateChanger.Behavior,
@@ -22,7 +23,7 @@ public static class FluentDownload {
            .WithLinks(StringEnumerable.FromGenerator(linkGenerator!)));
    }
    
-    public static ITransformStage<RawType, OutType> FromContext<RawType, OutType>(DownloadContext<RawType> existing) {
+    public static ITransformStage<RawType, OutType> FromContext<RawType, OutType>(DownloadContext<RawType> existing) where RawType : IDocument {
        return new TransformStage<RawType, OutType>(DownloadContextBuilder<RawType>.FromContext(existing));
    }
 }
@@ -1,11 +1,12 @@
-using Beam.Data;
+using Beam.Abstractions;
+using Beam.Data;
 using Beam.Downloaders;
 using Beam.Dynamic;
 using Beam.Models;

 namespace Beam.Fluent;

-internal sealed class TransformStage<RawType, OutType>(DownloadContextBuilder<RawType> CtxBuilder) : ITransformStage<RawType, OutType> {
+internal sealed class TransformStage<RawType, OutType>(DownloadContextBuilder<RawType> CtxBuilder) : ITransformStage<RawType, OutType> where RawType : IDocument {
    public IContextStage<RawType, OutType> WithTransformer(AsyncTransformer<RawType, OutType> transformer) {
        return new ContextStage<RawType, OutType>(CtxBuilder, transformer);
    }
@@ -1,15 +1,24 @@
 using System.Text;

 namespace Beam.Models {
-    public class ByteDocument(string filename, byte[] content, Encoding? encoding = null) : Document(filename, encoding) {
-        public byte[] Content { get; set; } = content;
+    public class ByteDocument : Document {
+        public ByteDocument(string filename, byte[] content, Encoding? encoding = null) : base(filename, encoding) {
+            Content = content;
+        }

+        public ByteDocument(string filename, Memory<byte> content, Encoding? encoding = null) :
+            base(filename, encoding) {
+            Content = content;
+        }
+
+        public Memory<byte> Content { get; set; }
+        
        public override byte[] ToBytes() {
-            return Content;
+            return Content.ToArray();
        }

        public override string ToString() {
-            return Encoding.GetString(Content);
+            return Encoding.GetString(Content.ToArray());
        }
    }
 }
@@ -2,7 +2,9 @@

 namespace Beam.Models {
    public struct DownloadReport : IDownloadReport {
-        // TODO implement download report
+        public long BytesDownloaded { get; init; }
+        public long? BytesRemaining { get; init; }
+        
    }

 }
@@ -1,36 +1,36 @@
-using Beam.Downloaders;
+using Beam.Abstractions;
+using Beam.Downloaders;
 using Beam.Models;
 using Microsoft.Playwright;

 namespace Beam.Playwright {
-    public class PlaywrightUnitDownloader<T> : UnitDownloaderBinary<T> {
-        public PlaywrightAsyncManipulator PuppetManipulator { get; }
+    public class PlaywrightUnitDownloader<RawType, OutType>(
+        UnitDownloaderOptions<RawType, OutType> options,
+        PlaywrightAsyncManipulator puppetManipulator)
+        : UnitDownloader<RawType, OutType>(options)
+        where RawType : IDocument {
+        public PlaywrightAsyncManipulator PuppetManipulator { get; } = puppetManipulator;

-        public PlaywrightUnitDownloader(HttpClient client, PlaywrightAsyncManipulator puppetManipulator, AsyncTransformer<ByteDocument, T> asyncHtmlTransformer, AsyncDownloadFailurePredicate<ByteDocument>[] asyncDownloadFailurePredicates)
-            : base(client, asyncHtmlTransformer, asyncDownloadFailurePredicates) {
-            PuppetManipulator = puppetManipulator;
-        }
-
-        protected override async Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
+        protected override async Task DownloadToStream(string url, int bufferSize, Stream destinationStream, IProgress<IDownloadReport> progress, CancellationToken ct) {
            var page = await PlaywrightContext.Browser.Value.NewPageAsync();
            try {
-                await page.GotoAsync(link);
+                await page.GotoAsync(url);
                await PuppetManipulator(page);
                var download = await page.WaitForDownloadAsync();

-                using var stream = await download.CreateReadStreamAsync();
-                byte[] content = new byte[stream.Length];
-
-                await stream.ReadExactlyAsync(content, ct);
-
-                ByteDocument doc = new ByteDocument(download.SuggestedFilename, content);
-                if (FailurePredicates is not null && await IsFailure(doc))
-                    return (false, default);
-
-                var transformed = await Transformer(doc);
-                return (true, transformed);
-            } catch (Exception) {
-                return (false, default);
+                await using var stream = await download.CreateReadStreamAsync();
+                var buffer = new byte[bufferSize];
+                var inBuffer = 0;
+                var downloaded = 0;
+                while ((inBuffer = stream.Read(buffer)) > 0) {
+                    downloaded += inBuffer;
+                    progress?.Report(new DownloadReport() {
+                        BytesDownloaded = downloaded,
+                        BytesRemaining = stream.Length - downloaded
+                    });
+                    await destinationStream.WriteAsync(buffer.AsMemory(0, inBuffer), ct);
+                }
+                
            } finally {
                if (!page.IsClosed)
                    await page.CloseAsync();
@@ -1,39 +0,0 @@
-
-using Beam.Downloaders;
-using Beam.Models;
-using HtmlAgilityPack;
-using Microsoft.Playwright;
-
-namespace Beam.Playwright {
-    public class PlaywrightUnitPageDownloader<T> : UnitDownloader<T> {
-        public PlaywrightAsyncManipulator PuppetManipulator { get; }
-
-        public PlaywrightUnitPageDownloader(HtmlWeb web, PlaywrightAsyncManipulator puppetManipulator, AsyncTransformer<HtmlDocument, T> asyncHtmlTransformer, AsyncDownloadFailurePredicate<HtmlDocument>[] asyncDownloadFailurePredicates)
-            : base(web, asyncHtmlTransformer, asyncDownloadFailurePredicates) {
-            PuppetManipulator = puppetManipulator;
-        }
-
-        protected override async Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
-            var page = await PlaywrightContext.Browser.Value.NewPageAsync();
-            try {
-                await page.GotoAsync(link);
-                await PuppetManipulator(page);
-                var content = await page.ContentAsync();
-                await page.CloseAsync();
-
-                HtmlDocument doc = new();
-                doc.LoadHtml(content);
-                var transformed = await Transformer(doc);
-                if (FailurePredicates is null || !(await IsFailure(doc)))
-                    return (true, transformed);
-                return (false, default);
-            } catch (Exception) {
-                return (false, default);
-            } finally {
-                if (!page.IsClosed)
-                    await page.CloseAsync();
-            }
-        }
-    }
-
-}
@@ -4,11 +4,12 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Beam.Abstractions;
 using Beam.Downloaders;
 using Beam.Models;

 namespace Beam.Stealth {
-    public class StealthFragmentDownloader<T> : UnitFragmentDownloaderBinary<T> {
-        public StealthFragmentDownloader(HttpClient client, StealthConfig config, StealthAsyncManipulator manipulator, AsyncTransformer<ByteDocument, T> transformer, AsyncDownloadFailurePredicate<ByteDocument>?[]? failurePredicate = null, int fragmentSize = 4, ILogger? logger = null) : base(client, transformer, failurePredicate, fragmentSize, logger, new StealthUnitDownloader<T>(client, config, manipulator, transformer, failurePredicate)) {}
+    public class StealthFragmentDownloader<RawType, OutType> : UnitFragmentDownloader<RawType, OutType> where RawType : IDocument {
+        public StealthFragmentDownloader(UnitDownloaderOptions<RawType, OutType> options, StealthConfig config, StealthAsyncManipulator manipulator) : base(options, new StealthUnitDownloader<RawType, OutType>(options, config, manipulator)) {}
    }
 }
@@ -5,11 +5,12 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Beam.Abstractions;
 using Beam.Downloaders;
 using Beam.Models;

 namespace Beam.Stealth {
-    public class StealthFragmentPageDownloader<T> : UnitFragmentDownloader<T> {
-        public StealthFragmentPageDownloader(HtmlWeb web, StealthConfig config, StealthAsyncManipulator manipulator, AsyncTransformer<HtmlDocument, T> transformer, AsyncDownloadFailurePredicate<HtmlDocument>?[]? failurePredicate = null, int fragmentSize = 4, ILogger? logger = null) : base(web, transformer, failurePredicate, fragmentSize, logger, new StealthUnitPageDownloader<T>(web, config, manipulator, transformer, failurePredicate)) {}
+    public class StealthFragmentPageDownloader<RawType, OutType> : UnitFragmentDownloader<RawType, OutType> where RawType : IDocument {
+        public StealthFragmentPageDownloader(UnitDownloaderOptions<RawType, OutType> options, StealthConfig config, StealthAsyncManipulator manipulator) : base(options, new StealthUnitPageDownloader<RawType, OutType>(options, config, manipulator)) {}
    }
 }
@@ -6,50 +6,37 @@ using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Beam.Abstractions;
 using Beam.Downloaders;
 using Beam.Models;

 namespace Beam.Stealth {
    using File = System.IO.File;
    
-    public class StealthUnitDownloader<T> : UnitDownloaderBinary<T> {
+    public class StealthUnitDownloader<RawType, OutType> : UnitDownloader<RawType, OutType> where RawType : IDocument {
        public StealthConfig Config { get; }
        public StealthAsyncManipulator Manipulator { get; }
-
        private ILogger? Logger => Config.Logger;

-        public StealthUnitDownloader(HttpClient client, StealthConfig config, StealthAsyncManipulator manipulator, AsyncTransformer<ByteDocument, T> transformer, AsyncDownloadFailurePredicate<ByteDocument>?[]? failurePredicates = null) : base(client, transformer, failurePredicates) {
+        public StealthUnitDownloader(UnitDownloaderOptions<RawType, OutType> options, StealthConfig config, StealthAsyncManipulator manipulator) : base(options) {
            Config = config;
            Manipulator = manipulator;
        }

-        protected override async Task<(bool Success, T? Result)> TryDownloadWithNoRetries(
-            string link, CancellationToken ct) {
-            try {
-                Logger?.LogInformation("Navigating to {Link}", link);
+        protected override async Task DownloadToStream(string url, int bufferSize, Stream destinationStream,
+            IProgress<IDownloadReport> progress, CancellationToken ct) {
+            var driver = Config.Driver;
+            await driver.Navigate().GoToUrlAsync(url);
+            await Manipulator(driver);

-                var driver = Config.Driver;
-                await driver.Navigate().GoToUrlAsync(link);
-                await Manipulator(driver);
-
-                var sw = Stopwatch.StartNew();
-                ByteDocument? doc = await WaitForDownloadAsync(link, sw, ct);
-
-                if (doc is null || await IsFailure(doc))
-                    return (false, default);
-
-                Logger?.LogInformation("Download finished in {Elapsed}", sw.Elapsed);
-                return (true, await Transformer(doc));
-            } catch (Exception ex) {
-                Logger?.LogError(ex, "Error occurred downloading {Link}", link);
-                return (false, default);
-            }
+            await using var stream = await WaitForDownloadAsync(url, progress, Stopwatch.StartNew(), ct);
+            await (stream?.CopyToAsync(destinationStream, ct) ?? Task.CompletedTask);
        }

        /* --------------------------------------------------------------------- */

-        private async Task<ByteDocument?> WaitForDownloadAsync(
-            string link, Stopwatch sw, CancellationToken ct) {
+        private async Task<Stream?> WaitForDownloadAsync(
+            string link, IProgress<IDownloadReport> progress, Stopwatch sw, CancellationToken ct) {
            const int PollDelayMs = 250;   // how often we look
            const int StableDelayMs = 1000;  // size-unchanged window

@@ -80,6 +67,9 @@ namespace Beam.Stealth {
                // track growth
                long size = new FileInfo(finalPath).Length;
                if (size == 0 || size != lastSize) {
+                    progress?.Report(new DownloadReport() {
+                        BytesDownloaded = size - lastSize,
+                    });
                    lastSize = size;
                    lastChange = DateTime.UtcNow;
                    await Task.Delay(PollDelayMs, ct);
@@ -104,11 +94,7 @@ namespace Beam.Stealth {
                    }
                }

-                byte[] bytes = await File.ReadAllBytesAsync(finalPath, ct);
-                Logger?.LogInformation("Download completed {Path} ({Size} bytes)",
-                                       finalPath, bytes.Length);
-
-                return new ByteDocument(Path.GetFileName(finalPath), bytes);
+                return File.OpenRead(finalPath);
            }

            Logger?.LogWarning("Download timed out after {Elapsed}", sw.Elapsed);
@@ -5,39 +5,29 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Beam.Abstractions;
 using Beam.Downloaders;
 using Beam.Models;

 namespace Beam.Stealth {
-    public class StealthUnitPageDownloader<T> : UnitDownloader<T> {
+    public class StealthUnitPageDownloader<RawType, OutType> : UnitDownloader<RawType, OutType> where RawType : IDocument {
        public StealthConfig Config { get; }
        public StealthAsyncManipulator Manipulator { get; }
-
        private ILogger? Logger => Config.Logger;

-        public StealthUnitPageDownloader(HtmlWeb web, StealthConfig config, StealthAsyncManipulator manipulator, AsyncTransformer<HtmlDocument, T> transformer, AsyncDownloadFailurePredicate<HtmlDocument>?[]? failurePredicate = null) : base(web, transformer, failurePredicate) {
+        public StealthUnitPageDownloader(UnitDownloaderOptions<RawType, OutType> options, StealthConfig config, StealthAsyncManipulator manipulator) : base(options) {
            Config = config;
            Manipulator = manipulator;
        }

-        protected async override Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
-            try {
-                var driver = Config.Driver;
+        protected override async Task DownloadToStream(string url, int bufferSize, Stream destinationStream, IProgress<IDownloadReport> progress, CancellationToken ct) {
+            var driver = Config.Driver;

-                await driver.Navigate().GoToUrlAsync(link);
-                await Manipulator(driver);
+            await driver.Navigate().GoToUrlAsync(url);
+            await Manipulator(driver);

-                HtmlDocument doc = new();
-                doc.LoadHtml(driver.PageSource);
-
-                if (await IsFailure(doc))
-                    return (false, default);
-
-                return (true, await Transformer(doc));
-            } catch (Exception e) {
-                Logger?.LogError(e, "Error occurred downloading {}", link);
-                return (false, default);
-            }
+            byte[] bytes = Encoding.UTF8.GetBytes(driver.PageSource);
+            await destinationStream.WriteAsync(bytes, ct);
        }
    }
 }
@@ -7,10 +7,10 @@
        <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
    </PropertyGroup>
    <ItemGroup>
-        <PackageReference Include="aeqw89.DataKeys" Version="2.0.1" />
+        <PackageReference Include="aeqw89.DataKeys" Version="2.1.1" />
        <PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.9" />
-        <PackageReference Include="System.Linq.Async" Version="6.0.1" />
+        <PackageReference Include="System.Linq.Async" Version="6.0.3" />
    </ItemGroup>
    <ItemGroup>
      <ProjectReference Include="..\Beam.Dynamic\Beam.Dynamic.csproj" />
@@ -7,11 +7,12 @@
        <Title>Beam</Title>
        <Authors>aeqw89</Authors>
        <Company>qwsdcvghyu</Company>
-        <Version>2.1.6</Version>
+        <Version>2.2.0</Version>
        <Description>A library for downloading internet resources</Description>
        <PackageProjectUrl>https://github.com/qwsdcvghyu89/Beam</PackageProjectUrl>
        <RepositoryUrl>https://github.com/qwsdcvghyu89/Beam</RepositoryUrl>
        <PackageId>aeqw89.Beam</PackageId>
+        <PackageVersion>2.2.0</PackageVersion>
    </PropertyGroup>
    <ItemGroup>
        <ProjectReference Include="..\Beam.Dynamic\Beam.Dynamic.csproj">
@@ -32,7 +33,7 @@
        <ProjectReference Include="..\Beam\Beam.csproj">
            <PrivateAssets>all</PrivateAssets>
        </ProjectReference>
-        <PackageReference Include="aeqw89.DataKeys" Version="2.0.1">
+        <PackageReference Include="aeqw89.DataKeys" Version="2.1.1">
            <Transitive>true</Transitive>
        </PackageReference>
        <PackageReference Include="aeqw89.PersistentData" Version="1.4.5">
@@ -53,7 +54,7 @@
        <PackageReference Include="Selenium.WebDriver" Version="4.34.0">
            <Transitive>true</Transitive>
        </PackageReference>
-        <PackageReference Include="System.Linq.Async" Version="6.0.1">
+        <PackageReference Include="System.Linq.Async" Version="6.0.3">
            <Transitive>true</Transitive>
        </PackageReference>
        <PackageReference Include="EntityFramework" Version="6.5.1">
@@ -101,10 +102,10 @@
            <PackagePath>lib\$(TargetFramework)</PackagePath>
            <Pack>true</Pack>
        </Content>
-        <Content Include="..\Beam.Temporary.Cli\bin\$(Configuration)\$(TargetFramework)\HtmlAgilityPack.dll">
-            <PackagePath>lib\$(TargetFramework)\</PackagePath>
-            <Pack>true</Pack>
-        </Content>
+        <!--        <Content Include="..\Beam.Temporary.Cli\bin\$(Configuration)\$(TargetFramework)\HtmlAgilityPack.dll">-->
+        <!--            <PackagePath>lib\$(TargetFramework)\</PackagePath>-->
+        <!--            <Pack>true</Pack>-->
+        <!--        </Content>-->
        <Content Include="..\Beam.Api\bin\$(Configuration)\$(TargetFramework)\Beam.Api.dll">
            <PackagePath>lib\$(TargetFramework)</PackagePath>
            <Pack>true</Pack>
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <Title>Beam</Title>
+        <Authors>aeqw89</Authors>
+        <Company>qwsdcvghyu</Company>
+        <Version>2.1.6</Version>
+        <Description>A library for downloading internet resources</Description>
+        <PackageProjectUrl>https://github.com/qwsdcvghyu89/Beam</PackageProjectUrl>
+        <RepositoryUrl>https://github.com/qwsdcvghyu89/Beam</RepositoryUrl>
+        <PackageId>aeqw89.Beam</PackageId>
+        <PackageVersion>2.1.6</PackageVersion>
+    </PropertyGroup>
+    <ItemGroup>
+        <ProjectReference Include="..\Beam.Dynamic\Beam.Dynamic.csproj">
+            <PrivateAssets>all</PrivateAssets>
+        </ProjectReference>
+        <ProjectReference Include="..\Beam.Exports\Beam.Exports.csproj">
+            <PrivateAssets>all</PrivateAssets>
+        </ProjectReference>
+        <ProjectReference Include="..\Beam.Playwright\Beam.Playwright.csproj">
+            <PrivateAssets>all</PrivateAssets>
+        </ProjectReference>
+        <ProjectReference Include="..\Beam.Stealth\Beam.Stealth.csproj">
+            <PrivateAssets>all</PrivateAssets>
+        </ProjectReference>
+        <!--        <ProjectReference Include="..\Beam.Temporary.Cli\Beam.Temporary.Cli.csproj">-->
+        <!--            <PrivateAssets>all</PrivateAssets>-->
+        <!--        </ProjectReference>-->
+        <ProjectReference Include="..\Beam\Beam.csproj">
+            <PrivateAssets>all</PrivateAssets>
+        </ProjectReference>
+        <PackageReference Include="aeqw89.DataKeys" Version="2.1.1">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="aeqw89.PersistentData" Version="1.4.5">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="HtmlAgilityPack" Version="1.11.72">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="Microsoft.Recognizers.Text.Number" Version="1.8.13">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="Microsoft.Playwright" Version="1.52.0">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.9">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="Selenium.WebDriver" Version="4.34.0">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="System.Linq.Async" Version="6.0.1">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="EntityFramework" Version="6.5.1">
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.8">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <Transitive>true</Transitive>
+        </PackageReference>
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="9.0.8">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <Transitive>true</Transitive>
+        </PackageReference>
+    </ItemGroup>
+    <ItemGroup>
+        <Content Include="..\Beam\bin\$(Configuration)\$(TargetFramework)\Beam.dll">
+            <PackagePath>lib\$(TargetFramework)\</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Dynamic\bin\$(Configuration)\$(TargetFramework)\Beam.Dynamic.dll">
+            <PackagePath>lib\$(TargetFramework)\</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Exports\bin\$(Configuration)\$(TargetFramework)\Beam.Exports.dll">
+            <PackagePath>lib\$(TargetFramework)\</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Playwright\bin\$(Configuration)\$(TargetFramework)\Beam.Playwright.dll">
+            <PackagePath>lib\$(TargetFramework)\</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Stealth\bin\$(Configuration)\$(TargetFramework)\Beam.Stealth.dll">
+            <PackagePath>lib\$(TargetFramework)\</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <!--        <Content Include="..\Beam.Temporary.Cli\bin\$(Configuration)\$(TargetFramework)\Beam.Temporary.Cli.dll">-->
+        <!--            <PackagePath>lib\$(TargetFramework)\</PackagePath>-->
+        <!--            <Pack>true</Pack>-->
+        <!--        </Content>-->
+        <Content Include="..\Beam.Fluent\bin\$(Configuration)\$(TargetFramework)\Beam.Fluent.dll">
+            <PackagePath>lib\$(TargetFramework)</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Models\bin\$(Configuration)\$(TargetFramework)\Beam.Models.dll">
+            <PackagePath>lib\$(TargetFramework)</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <!--        <Content Include="..\Beam.Temporary.Cli\bin\$(Configuration)\$(TargetFramework)\HtmlAgilityPack.dll">-->
+        <!--            <PackagePath>lib\$(TargetFramework)\</PackagePath>-->
+        <!--            <Pack>true</Pack>-->
+        <!--        </Content>-->
+        <Content Include="..\Beam.Api\bin\$(Configuration)\$(TargetFramework)\Beam.Api.dll">
+            <PackagePath>lib\$(TargetFramework)</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Data\bin\$(Configuration)\$(TargetFramework)\Beam.Data.dll">
+            <PackagePath>lib\$(TargetFramework)</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Abstractions\bin\$(Configuration)\$(TargetFramework)\Beam.Abstractions.dll">
+            <PackagePath>lib\$(TargetFramework)</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Downloaders\bin\$(Configuration)\$(TargetFramework)\Beam.Downloaders.dll">
+            <PackagePath>lib\$(TargetFramework)</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+        <Content Include="..\Beam.Exceptions\bin\$(Configuration)\$(TargetFramework)\Beam.Exceptions.dll">
+            <PackagePath>lib\$(TargetFramework)</PackagePath>
+            <Pack>true</Pack>
+        </Content>
+    </ItemGroup>
+</Project>