Refactor downloaders to use generic options and unify logic
Replaces specialized binary and HTML downloaders with a generic, options-driven UnitDownloader and UnitFragmentDownloader pattern. Introduces UnitDownloaderOptions and builder classes for flexible configuration, updates interfaces and method signatures to support progress reporting, and removes redundant binary-specific classes. Updates Playwright and Stealth downloaders to use the new generic base, and adds improved error handling and reporting. Also updates dependency versions and makes minor API consistency improvements across the Fluent and Models layers.
This commit is contained in:
qwsdcvghyu89
@@ -0,0 +1,198 @@
|
||||
using Beam.Models;
|
||||
|
||||
namespace Beam.Downloaders;
|
||||
|
||||
public record class UnitDownloaderOptions<RawType, OutType> {
|
||||
public HttpClient Client { get; init; } = new();
|
||||
|
||||
public FailurePredicateOptions<RawType>? FailurePredicateOptions { get; init; }
|
||||
public FragmentOptions? FragmentOptions { get; init; }
|
||||
public required AsyncTransformer<RawType, OutType> AsyncTransformer { get; init; }
|
||||
public string? DownloadFolder { get; init; } = null;
|
||||
public int BufferSize { get; init; } = 80 * 1024; // 80kb
|
||||
}
|
||||
|
||||
public record class FailurePredicateOptions<RawType> {
|
||||
public required AsyncDownloadFailurePredicate<RawType>?[]? AsyncDownloadFailurePredicates { get; init; }
|
||||
public bool ProcessInParallel { get; init; } = false;
|
||||
public int? ParallelThreads { get; init; }
|
||||
}
|
||||
|
||||
public record class FragmentOptions {
|
||||
public required int FragmentSize { get; init; }
|
||||
public bool DownloadInParallel { get; init; } = false;
|
||||
public int? ParallelThreads { get; init; }
|
||||
}
|
||||
|
||||
|
||||
// ---------- UnitDownloaderOptions Builder ----------
|
||||
public sealed class UnitDownloaderOptionsBuilder<TRaw, TOut>
|
||||
{
|
||||
private HttpClient _client = new HttpClient();
|
||||
private FailurePredicateOptions<TRaw>? _failureOptions;
|
||||
private FragmentOptions? _fragmentOptions;
|
||||
private AsyncTransformer<TRaw, TOut>? _asyncTransformer;
|
||||
private string? _downloadFolder = null;
|
||||
private int _bufferSize = 80 * 1024;
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithClient(HttpClient client)
|
||||
{
|
||||
_client = client ?? throw new System.ArgumentNullException(nameof(client));
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFailurePredicateOptions(FailurePredicateOptions<TRaw>? options)
|
||||
{
|
||||
_failureOptions = options;
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFailurePredicates(System.Action<FailurePredicateOptionsBuilder<TRaw>> configure)
|
||||
{
|
||||
if (configure == null) throw new System.ArgumentNullException(nameof(configure));
|
||||
var b = new FailurePredicateOptionsBuilder<TRaw>();
|
||||
configure(b);
|
||||
_failureOptions = b.Build();
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFragmentOptions(FragmentOptions? options)
|
||||
{
|
||||
_fragmentOptions = options;
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFragments(System.Action<FragmentOptionsBuilder> configure)
|
||||
{
|
||||
if (configure == null) throw new System.ArgumentNullException(nameof(configure));
|
||||
var b = new FragmentOptionsBuilder();
|
||||
configure(b);
|
||||
_fragmentOptions = b.Build();
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithAsyncTransformer(AsyncTransformer<TRaw, TOut> transformer)
|
||||
{
|
||||
_asyncTransformer = transformer ?? throw new System.ArgumentNullException(nameof(transformer));
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithDownloadFolder(string? downloadFolder)
|
||||
{
|
||||
_downloadFolder = downloadFolder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithBufferSize(int bytes)
|
||||
{
|
||||
if (bytes <= 0) throw new System.ArgumentOutOfRangeException(nameof(bytes));
|
||||
_bufferSize = bytes;
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptions<TRaw, TOut> Build()
|
||||
{
|
||||
if (_asyncTransformer == null)
|
||||
throw new System.InvalidOperationException("AsyncTransformer must be provided.");
|
||||
|
||||
return new UnitDownloaderOptions<TRaw, TOut>
|
||||
{
|
||||
Client = _client,
|
||||
FailurePredicateOptions = _failureOptions,
|
||||
FragmentOptions = _fragmentOptions,
|
||||
AsyncTransformer = _asyncTransformer,
|
||||
DownloadFolder = _downloadFolder,
|
||||
BufferSize = _bufferSize
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- FailurePredicateOptions Builder ----------
|
||||
public sealed class FailurePredicateOptionsBuilder<TRaw>
|
||||
{
|
||||
private readonly System.Collections.Generic.List<AsyncDownloadFailurePredicate<TRaw>?> _predicates =
|
||||
new System.Collections.Generic.List<AsyncDownloadFailurePredicate<TRaw>?>();
|
||||
private bool _processInParallel = false;
|
||||
private int? _parallelThreads = null;
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithPredicate(AsyncDownloadFailurePredicate<TRaw>? predicate)
|
||||
{
|
||||
_predicates.Add(predicate);
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithPredicates(System.Collections.Generic.IEnumerable<AsyncDownloadFailurePredicate<TRaw>?> predicates)
|
||||
{
|
||||
if (predicates == null) throw new System.ArgumentNullException(nameof(predicates));
|
||||
_predicates.AddRange(predicates);
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithPredicates(params AsyncDownloadFailurePredicate<TRaw>?[] predicates)
|
||||
{
|
||||
_predicates.Clear();
|
||||
if (predicates != null) _predicates.AddRange(predicates);
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithProcessInParallel(bool value = true)
|
||||
{
|
||||
_processInParallel = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithParallelThreads(int? threads)
|
||||
{
|
||||
if (threads.HasValue && threads.Value <= 0)
|
||||
throw new System.ArgumentOutOfRangeException(nameof(threads));
|
||||
_parallelThreads = threads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptions<TRaw> Build()
|
||||
{
|
||||
var arr = _predicates.Count == 0 ? [] : _predicates.ToArray();
|
||||
return new FailurePredicateOptions<TRaw>
|
||||
{
|
||||
AsyncDownloadFailurePredicates = arr,
|
||||
ProcessInParallel = _processInParallel,
|
||||
ParallelThreads = _parallelThreads
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- FragmentOptions Builder ----------
|
||||
public sealed class FragmentOptionsBuilder {
|
||||
private int? _fragmentSize;
|
||||
private bool _downloadInParallel = false;
|
||||
private int? _parallelThreads = null;
|
||||
|
||||
public FragmentOptionsBuilder WithFragmentSize(int bytes) {
|
||||
if (bytes <= 0) throw new System.ArgumentOutOfRangeException(nameof(bytes));
|
||||
_fragmentSize = bytes;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FragmentOptionsBuilder WithDownloadInParallel(bool value = true) {
|
||||
_downloadInParallel = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FragmentOptionsBuilder WithParallelThreads(int? threads) {
|
||||
if (threads.HasValue && threads.Value <= 0)
|
||||
throw new System.ArgumentOutOfRangeException(nameof(threads));
|
||||
_parallelThreads = threads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FragmentOptions Build() {
|
||||
if (!_fragmentSize.HasValue)
|
||||
throw new System.InvalidOperationException("FragmentSize must be provided.");
|
||||
|
||||
return new FragmentOptions {
|
||||
FragmentSize = _fragmentSize.Value,
|
||||
DownloadInParallel = _downloadInParallel,
|
||||
ParallelThreads = _parallelThreads
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user