Refactor downloaders to use ByteDocument and add options builders
Replaces generic RawType with ByteDocument in downloaders and context classes, simplifying type usage. Adds builder classes for FailurePredicateOptions, FragmentOptions, SkipPredicateOptions, and UnitDownloaderOptions to improve configuration flexibility. Introduces DownloadTarget enum and SkipPredicate delegate for more granular download control. Refactors Fluent API interfaces and implementations to remove RawType generics and streamline usage. Adds Playwright and Stealth download strategies for extensibility.
This commit is contained in:
qwsdcvghyu89
@@ -1,198 +1,38 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Beam.Models;
|
||||
|
||||
namespace Beam.Downloaders;
|
||||
|
||||
public record class UnitDownloaderOptions<RawType, OutType> {
|
||||
public record class UnitDownloaderOptions<OutType> {
|
||||
public HttpClient Client { get; init; } = new();
|
||||
|
||||
public DownloadTarget Target { get; init; } = DownloadTarget.URL;
|
||||
|
||||
public FailurePredicateOptions<RawType>? FailurePredicateOptions { get; init; }
|
||||
public SkipPredicateOptions<OutType>? SkipPredicateOptions { get; init; }
|
||||
public FailurePredicateOptions<ByteDocument>? FailurePredicateOptions { get; init; }
|
||||
public FragmentOptions? FragmentOptions { get; init; }
|
||||
public required AsyncTransformer<RawType, OutType> AsyncTransformer { get; init; }
|
||||
public required AsyncTransformer<ByteDocument, OutType> AsyncTransformer { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The location where the download is stored.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// If not defined, <c>UnitDownloader.TryDownload()</c> downloads to memory.
|
||||
/// </remarks>
|
||||
public string? DownloadFolder { get; init; } = null;
|
||||
public int BufferSize { get; init; } = 80 * 1024; // 80kb
|
||||
|
||||
public string GetFileNameForDownload(string url, byte[] additionalData) {
|
||||
byte[] bytes = [..Encoding.UTF8.GetBytes(url), ..additionalData];
|
||||
var name = Convert.ToBase64String(System.IO.Hashing.XxHash64.Hash(bytes));
|
||||
return name.Replace('+', '-').Replace('/', '_').Replace('=', ' ').Trim();
|
||||
}
|
||||
}
|
||||
|
||||
public record class FailurePredicateOptions<RawType> {
|
||||
public required AsyncDownloadFailurePredicate<RawType>?[]? AsyncDownloadFailurePredicates { get; init; }
|
||||
public bool ProcessInParallel { get; init; } = false;
|
||||
public int? ParallelThreads { get; init; }
|
||||
}
|
||||
// ---------- UnitDownloaderOptions Builder ----------
|
||||
|
||||
public record class FragmentOptions {
|
||||
public required int FragmentSize { get; init; }
|
||||
public bool DownloadInParallel { get; init; } = false;
|
||||
public int? ParallelThreads { get; init; }
|
||||
}
|
||||
// ---------- FailurePredicateOptions Builder ----------
|
||||
|
||||
|
||||
// ---------- UnitDownloaderOptions Builder ----------
|
||||
public sealed class UnitDownloaderOptionsBuilder<TRaw, TOut>
|
||||
{
|
||||
private HttpClient _client = new HttpClient();
|
||||
private FailurePredicateOptions<TRaw>? _failureOptions;
|
||||
private FragmentOptions? _fragmentOptions;
|
||||
private AsyncTransformer<TRaw, TOut>? _asyncTransformer;
|
||||
private string? _downloadFolder = null;
|
||||
private int _bufferSize = 80 * 1024;
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithClient(HttpClient client)
|
||||
{
|
||||
_client = client ?? throw new System.ArgumentNullException(nameof(client));
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFailurePredicateOptions(FailurePredicateOptions<TRaw>? options)
|
||||
{
|
||||
_failureOptions = options;
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFailurePredicates(System.Action<FailurePredicateOptionsBuilder<TRaw>> configure)
|
||||
{
|
||||
if (configure == null) throw new System.ArgumentNullException(nameof(configure));
|
||||
var b = new FailurePredicateOptionsBuilder<TRaw>();
|
||||
configure(b);
|
||||
_failureOptions = b.Build();
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFragmentOptions(FragmentOptions? options)
|
||||
{
|
||||
_fragmentOptions = options;
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithFragments(System.Action<FragmentOptionsBuilder> configure)
|
||||
{
|
||||
if (configure == null) throw new System.ArgumentNullException(nameof(configure));
|
||||
var b = new FragmentOptionsBuilder();
|
||||
configure(b);
|
||||
_fragmentOptions = b.Build();
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithAsyncTransformer(AsyncTransformer<TRaw, TOut> transformer)
|
||||
{
|
||||
_asyncTransformer = transformer ?? throw new System.ArgumentNullException(nameof(transformer));
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithDownloadFolder(string? downloadFolder)
|
||||
{
|
||||
_downloadFolder = downloadFolder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptionsBuilder<TRaw, TOut> WithBufferSize(int bytes)
|
||||
{
|
||||
if (bytes <= 0) throw new System.ArgumentOutOfRangeException(nameof(bytes));
|
||||
_bufferSize = bytes;
|
||||
return this;
|
||||
}
|
||||
|
||||
public UnitDownloaderOptions<TRaw, TOut> Build()
|
||||
{
|
||||
if (_asyncTransformer == null)
|
||||
throw new System.InvalidOperationException("AsyncTransformer must be provided.");
|
||||
|
||||
return new UnitDownloaderOptions<TRaw, TOut>
|
||||
{
|
||||
Client = _client,
|
||||
FailurePredicateOptions = _failureOptions,
|
||||
FragmentOptions = _fragmentOptions,
|
||||
AsyncTransformer = _asyncTransformer,
|
||||
DownloadFolder = _downloadFolder,
|
||||
BufferSize = _bufferSize
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- FailurePredicateOptions Builder ----------
|
||||
public sealed class FailurePredicateOptionsBuilder<TRaw>
|
||||
{
|
||||
private readonly System.Collections.Generic.List<AsyncDownloadFailurePredicate<TRaw>?> _predicates =
|
||||
new System.Collections.Generic.List<AsyncDownloadFailurePredicate<TRaw>?>();
|
||||
private bool _processInParallel = false;
|
||||
private int? _parallelThreads = null;
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithPredicate(AsyncDownloadFailurePredicate<TRaw>? predicate)
|
||||
{
|
||||
_predicates.Add(predicate);
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithPredicates(System.Collections.Generic.IEnumerable<AsyncDownloadFailurePredicate<TRaw>?> predicates)
|
||||
{
|
||||
if (predicates == null) throw new System.ArgumentNullException(nameof(predicates));
|
||||
_predicates.AddRange(predicates);
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithPredicates(params AsyncDownloadFailurePredicate<TRaw>?[] predicates)
|
||||
{
|
||||
_predicates.Clear();
|
||||
if (predicates != null) _predicates.AddRange(predicates);
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithProcessInParallel(bool value = true)
|
||||
{
|
||||
_processInParallel = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptionsBuilder<TRaw> WithParallelThreads(int? threads)
|
||||
{
|
||||
if (threads.HasValue && threads.Value <= 0)
|
||||
throw new System.ArgumentOutOfRangeException(nameof(threads));
|
||||
_parallelThreads = threads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FailurePredicateOptions<TRaw> Build()
|
||||
{
|
||||
var arr = _predicates.Count == 0 ? [] : _predicates.ToArray();
|
||||
return new FailurePredicateOptions<TRaw>
|
||||
{
|
||||
AsyncDownloadFailurePredicates = arr,
|
||||
ProcessInParallel = _processInParallel,
|
||||
ParallelThreads = _parallelThreads
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- FragmentOptions Builder ----------
|
||||
public sealed class FragmentOptionsBuilder {
|
||||
private int? _fragmentSize;
|
||||
private bool _downloadInParallel = false;
|
||||
private int? _parallelThreads = null;
|
||||
|
||||
public FragmentOptionsBuilder WithFragmentSize(int bytes) {
|
||||
if (bytes <= 0) throw new System.ArgumentOutOfRangeException(nameof(bytes));
|
||||
_fragmentSize = bytes;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FragmentOptionsBuilder WithDownloadInParallel(bool value = true) {
|
||||
_downloadInParallel = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FragmentOptionsBuilder WithParallelThreads(int? threads) {
|
||||
if (threads.HasValue && threads.Value <= 0)
|
||||
throw new System.ArgumentOutOfRangeException(nameof(threads));
|
||||
_parallelThreads = threads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FragmentOptions Build() {
|
||||
if (!_fragmentSize.HasValue)
|
||||
throw new System.InvalidOperationException("FragmentSize must be provided.");
|
||||
|
||||
return new FragmentOptions {
|
||||
FragmentSize = _fragmentSize.Value,
|
||||
DownloadInParallel = _downloadInParallel,
|
||||
ParallelThreads = _parallelThreads
|
||||
};
|
||||
}
|
||||
}
|
||||
// ---------- FragmentOptions Builder ----------
|
||||
Reference in New Issue
Block a user