qwsdcvghyu89
f52aa6123b
Replaces generic RawType with ByteDocument in downloaders and context classes, simplifying type usage. Adds builder classes for FailurePredicateOptions, FragmentOptions, SkipPredicateOptions, and UnitDownloaderOptions to improve configuration flexibility. Introduces DownloadTarget enum and SkipPredicate delegate for more granular download control. Refactors Fluent API interfaces and implementations to remove RawType generics and streamline usage. Adds Playwright and Stealth download strategies for extensibility.
47 lines
2.1 KiB
C#
47 lines
2.1 KiB
C#
using Beam.Abstractions;
|
|
using Beam.Exceptions;
|
|
using Beam.Models;
|
|
using HtmlAgilityPack;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace Beam.Downloaders {
|
|
public class UnitFragmentDownloader<OutType>(UnitDownloaderOptions<OutType> options,
|
|
IUnitDownloader<OutType>? internalDownloader = null) : IUnitDownloader<Fragment<Ordered<OutType>>> {
|
|
|
|
public UnitDownloaderOptions<OutType> Options { get; } = options;
|
|
public int LinksPerDownload { get; set; }
|
|
private IUnitDownloader<OutType> UnitDownloader { get; } = internalDownloader ?? new UnitDownloader<OutType>(options);
|
|
|
|
async Task<(bool, Fragment<Ordered<OutType>>?)> IUnitDownloader<Fragment<Ordered<OutType>>>.TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount, IProgress<IDownloadReport>? downProgress, IProgress<IRetryReport>? tryProgress) {
|
|
Fragment<Ordered<OutType>> fragment = new Fragment<Ordered<OutType>>(link.Length);
|
|
if (!Fragment<Ordered<OutType>>.TryAcquireUpdater(fragment, out var updater))
|
|
throw new AssertionException(Exceptions.Exceptions.fragment_locked);
|
|
bool isFailure = false;
|
|
await Parallel.ForEachAsync(link, async (x, pct) => {
|
|
pct.ThrowIfCancellationRequested();
|
|
ct.ThrowIfCancellationRequested();
|
|
if (isFailure)
|
|
return;
|
|
var (result, downloadedT) = await UnitDownloader.TryDownload([x], ct, maximumRetryCount, downProgress, tryProgress);
|
|
if (!result) {
|
|
Interlocked.Exchange(ref isFailure, true);
|
|
return;
|
|
}
|
|
if (downloadedT == null) {
|
|
Interlocked.Exchange(ref isFailure, true);
|
|
return;
|
|
}
|
|
updater(new Ordered<OutType>(downloadedT, x.Order));
|
|
});
|
|
|
|
if (!isFailure)
|
|
Fragment<Ordered<OutType>>.SetComplete(fragment, true);
|
|
|
|
Fragment<Ordered<OutType>>.TryReleaseUpdater(fragment, updater);
|
|
|
|
return (!isFailure, fragment);
|
|
|
|
}
|
|
}
|
|
}
|