refactor: modularize Beam into new projects and interfaces
- Introduced modularity by splitting Beam into new projects: Beam.Abstractions, Beam.Models, and Beam.Downloaders. - Refactored existing classes into appropriate namespaces and projects. - Replaced specific implementations with abstractions (e.g., SourceLinkBuilder to LinkBuilder, State to IState, etc.). - Updated interfaces: added ITemplate, IArticleData, IDownloadReport, and others for improved extensibility. - Removed deprecated classes like SourceLinkBuilder and StateChangerFactory. - Enhanced link handling in downloaders by refactoring to use `string` over `SourceLink`. - Consolidated shared logic under Beam.Abstractions.
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
using Beam.Abstractions;
|
||||
using Beam.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Beam.Downloaders {
|
||||
public class SequentialDownloader<RawType, OutType> : IAsyncEnumerator<OutType> {
|
||||
public OutType Current { get; protected set; }
|
||||
public DownloadContext<RawType> Context { get; }
|
||||
public ILogger? Logger { get; set; }
|
||||
public int LastOrder { get; set; } = 0;
|
||||
|
||||
protected IEnumerator<string> LinksEnumerator;
|
||||
|
||||
public Func<IUnitDownloader<OutType>> GetUnitDownloader { get; set; }
|
||||
|
||||
public SequentialDownloader(DownloadContext<RawType> context, Func<DownloadContext<RawType>, IUnitDownloader<OutType>> getUnitDownloader, ILogger? logger = null) {
|
||||
Context = context;
|
||||
Logger = logger;
|
||||
LinksEnumerator = Context.Links.GetEnumerator();
|
||||
|
||||
try {
|
||||
LinksEnumerator.Reset();
|
||||
} catch (NotSupportedException) {
|
||||
Logger?.LogWarning("Enumerator of type {} does not support resets. This may cause buggy behavior", LinksEnumerator.GetType());
|
||||
}
|
||||
Current = default(OutType);
|
||||
GetUnitDownloader = () => getUnitDownloader(Context);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() {
|
||||
GC.SuppressFinalize(this);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public async ValueTask<bool> MoveNextAsync() {
|
||||
if (!LinksEnumerator.MoveNext())
|
||||
return false;
|
||||
|
||||
//Logger?.LogInformation("MoveNextAsync()");
|
||||
var unit = GetUnitDownloader(); // safe to instantiate per request.
|
||||
var idealLinkCount = unit.LinksPerDownload;
|
||||
List<Ordered<string>> links = [];
|
||||
|
||||
//Logger?.LogInformation("MoveNextAsync() \n\t -> Links.Current = {} ", LinksEnumerator.Current.Link.AbsoluteUri);
|
||||
links.Add(new Ordered<string>(LinksEnumerator.Current, LastOrder++));
|
||||
|
||||
while (LinksEnumerator.MoveNext() && !string.IsNullOrWhiteSpace(LinksEnumerator.Current) && links.Count < idealLinkCount)
|
||||
links.Add(new Ordered<string>(LinksEnumerator.Current, LastOrder++));
|
||||
//Logger?.LogInformation("MoveNextAsync() \n\t -> links.Count = {} ", links.Count);
|
||||
if (links.Count == 0) {
|
||||
Logger?.LogInformation("Out of links!");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (links.Any((x) => string.IsNullOrWhiteSpace(x.Data)))
|
||||
return false;
|
||||
|
||||
var (result, downloadedT) = await unit.TryDownload(
|
||||
links.ToArray(),
|
||||
Context.CancellationToken,
|
||||
tryProgress: Context.RetryReporter);
|
||||
|
||||
if (!result) {
|
||||
Logger?.LogWarning("Failed to download Unit<{}>", typeof(OutType).Name);
|
||||
return false; // unit download failed
|
||||
}
|
||||
if (downloadedT is null) {
|
||||
Logger?.LogWarning("Failed to download Unit<{}>", typeof(OutType).Name);
|
||||
return false; // unit download failed
|
||||
}
|
||||
|
||||
Current = downloadedT;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user