riley/Beam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor downloaders to use ByteDocument and add options builders

Browse Source 
Replaces generic RawType with ByteDocument in downloaders and context classes, simplifying type usage. Adds builder classes for FailurePredicateOptions, FragmentOptions, SkipPredicateOptions, and UnitDownloaderOptions to improve configuration flexibility. Introduces DownloadTarget enum and SkipPredicate delegate for more granular download control. Refactors Fluent API interfaces and implementations to remove RawType generics and streamline usage. Adds Playwright and Stealth download strategies for extensibility.
This commit is contained in:
qwsdcvghyu89
2025-11-15 22:51:46 +11:00
parent 647b2b0f37
commit f52aa6123b
34 changed files with 648 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Beam.Downloaders/DownloadContextBuilder.cs
+17 -17
View File
@@ -5,12 +5,12 @@ using Microsoft.Extensions.Logging;
namespace Beam.Downloaders {
public class DownloadContextBuilder<RawType> {
public class DownloadContextBuilder {
private HtmlWeb _web;
private HttpClient _client;
private IProgress<IDownloadReport>? _downloadReporter;
private IProgress<IRetryReport>? _retryReporter;
private AsyncDownloadFailurePredicate<RawType>?[] _asyncFailurePredicates = [];
private AsyncDownloadFailurePredicate<ByteDocument>?[] _asyncFailurePredicates = [];
private TimeSpan _timeOut;
private IEnumerable<string> _links;
private CancellationToken _cancellationToken;
@@ -26,60 +26,60 @@ namespace Beam.Downloaders {
_links = [];
}
public DownloadContextBuilder<RawType> WithWeb(HtmlWeb web) {
public DownloadContextBuilder WithWeb(HtmlWeb web) {
_web = web;
return this;
}
public DownloadContextBuilder<RawType> WithClient(HttpClient client) {
public DownloadContextBuilder WithClient(HttpClient client) {
_client = client;
return this;
}
public DownloadContextBuilder<RawType> WithDownloadReporter(IProgress<IDownloadReport> downloadReporter) {
public DownloadContextBuilder WithDownloadReporter(IProgress<IDownloadReport> downloadReporter) {
_downloadReporter = downloadReporter;
return this;
}
public DownloadContextBuilder<RawType> WithRetryReporter(IProgress<IRetryReport> retryReporter) {
public DownloadContextBuilder WithRetryReporter(IProgress<IRetryReport> retryReporter) {
_retryReporter = retryReporter;
return this;
}
public DownloadContextBuilder<RawType> WithAsyncFailurePredicates(params AsyncDownloadFailurePredicate<RawType>[] predicates) {
public DownloadContextBuilder WithAsyncFailurePredicates(params AsyncDownloadFailurePredicate<ByteDocument>[] predicates) {
_asyncFailurePredicates = predicates;
return this;
}
public DownloadContextBuilder<RawType> WithTimeOut(TimeSpan timeOut) {
public DownloadContextBuilder WithTimeOut(TimeSpan timeOut) {
_timeOut = timeOut;
return this;
}
public DownloadContextBuilder<RawType> WithLinks(IEnumerable<string> links) {
public DownloadContextBuilder WithLinks(IEnumerable<string> links) {
_links = links;
return this;
}
public DownloadContextBuilder<RawType> WithCancellationToken(CancellationToken cancellationToken) {
public DownloadContextBuilder WithCancellationToken(CancellationToken cancellationToken) {
_cancellationToken = cancellationToken;
return this;
}
public DownloadContextBuilder<RawType> WithCache(DocumentCache cache) {
public DownloadContextBuilder WithCache(DocumentCache cache) {
_cache = cache;
return this;
}
public DownloadContextBuilder<RawType> WithDownloadLogger(ILogger downloadLogger) {
public DownloadContextBuilder WithDownloadLogger(ILogger downloadLogger) {
_downloadLogger = downloadLogger;
return this;
}
public DownloadContext<RawType> Build() {
public DownloadContext Build() {
// Construct the DownloadContext<T> using the collected values.
var context = new DownloadContext<RawType>(
var context = new DownloadContext(
web: _web,
client: _client,
links: _links,
@@ -100,15 +100,15 @@ namespace Beam.Downloaders {
return context;
}
public static DownloadContextBuilder<RawType> FromContext(DownloadContext<RawType> existing) {
public static DownloadContextBuilder FromContext(DownloadContext existing) {
if (existing == null) throw new ArgumentNullException(nameof(existing));
return new DownloadContextBuilder<RawType>(existing.Client, existing.Web)
return new DownloadContextBuilder(existing.Client, existing.Web)
.WithLinks(existing.Links)
.WithCancellationToken(existing.CancellationToken)
.WithDownloadReporter(existing.DownloadReporter!)
.WithRetryReporter(existing.RetryReporter!)
.WithAsyncFailurePredicates(existing.AsyncFailurePredicates ?? Array.Empty<AsyncDownloadFailurePredicate<RawType>>())
.WithAsyncFailurePredicates(existing.AsyncFailurePredicates ?? Array.Empty<AsyncDownloadFailurePredicate<ByteDocument>>())
.WithTimeOut(existing.TimeOut)
.WithDownloadLogger(existing.DownloadLogger!)
.WithCache(existing.Cache);