riley/Beam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance async capabilities and refactor project structure

Browse Source 
Updated project files for `Beam.Dynamic`, `Beam.Exports`, `Beam.Puppeteer`, `Beam.Temporary.Cli`, and `Beam` to include new XML headers, reorganize property groups, and add project references.

Modified `PuppetedUnitDownloader` to support additional parameters for async transformers. Changed return types in `CommonTransformers` to `AsyncTransformer` for asynchronous processing.

Significant refactoring in `DownloadBuilder`, `DownloadContext`, and `DownloadContextBuilder` to introduce generic parameters and improve context management. Updated `SequentialDownloader`, `SequentialFragmentDownloader`, and `UnitDownloader` to accommodate new async transformer types.

Introduced `TypeExtensions` for unique type name generation and added `UnitFragmentDownloaderBinary` for handling binary downloads. Updated solution file to include the new `aeqw89.Beam` project, ensuring proper references across the solution.

These changes enhance the asynchronous capabilities of the Beam library, improve type safety, and streamline the downloading process.
This commit is contained in:
qwsdcvghyu89
2025-06-23 20:30:09 +03:00
parent 482a46b568
commit 056e426572
23 changed files with 531 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Beam.Temporary.Cli/DownloadBuilder.cs
+100 -31
View File
@@ -4,6 +4,7 @@ using Beam;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using HtmlAgilityPack;
namespace Beam.Temporary.Cli {
/// <summary>
@@ -11,7 +12,7 @@ namespace Beam.Temporary.Cli {
/// (source → link selection → transformer) and surfaces operational knobs as first‑class
/// methods instead of magic parameters.
/// </summary>
public static class DownloadBuilder<T> {
public static class DownloadBuilder<RawType, OutType> {
/* ──────────────────────────── Entry points ─────────────────────────── */
public static ILinkStage FromMeta(DataKey<TextResource> novelKey, BeamDataDictionary data) =>
@@ -20,6 +21,9 @@ namespace Beam.Temporary.Cli {
public static ILinkStage FromText(DataKey<TextResource> novelKey, BeamDataDictionary data) =>
Create(novelKey, data, SourceKind.Text);
public static IAlternativeLinkStage FromScratch()
=> new LinkStage(null!, null!, null!, new());
/* ────────────────────────────── Stages ─────────────────────────────── */
public interface ILinkStage {
@@ -28,16 +32,25 @@ namespace Beam.Temporary.Cli {
ILinkStage WithRange(Range range);
}
public interface ITransformStage {
IContextStage<U> WithTransformer<U>(Func<DataBindings, AsyncTransformer<T, U>> factory);
public interface IAlternativeLinkStage {
IAlternativeTransformStage WithLinks(IEnumerable<SourceLink> links);
}
public interface IContextStage<U> {
IContextStage<U> Configure(Action<DownloadContextBuilder<T>> configure);
IContextStage<U> WithParallelism(int degree);
IContextStage<U> WithTimeout(TimeSpan timeout);
IContextStage<U> WithRetryReporter(IProgress<RetryReport> reporter);
DownloadEnumerable<T> Build();
public interface ITransformStage {
IContextStage WithTransformer(Func<DataBindings, AsyncTransformer<RawType, OutType>> factory);
}
public interface IAlternativeTransformStage {
IContextStage WithTransformer(AsyncTransformer<RawType, OutType> transformer);
}
public interface IContextStage {
IContextStage Configure(Action<DownloadContextBuilder<RawType>> configure);
IContextStage WithParallelism(int degree);
IContextStage WithTimeout(TimeSpan timeout);
IContextStage WithRetryReporter(IProgress<RetryReport> reporter);
DownloadEnumerable<OutType> Build();
IContextStage UseFragments();
}
/* ────────────────────────── Implementation ────────────────────────── */
@@ -46,7 +59,7 @@ namespace Beam.Temporary.Cli {
private static ILinkStage Create(DataKey<TextResource> novelKey, BeamDataDictionary data, SourceKind kind) {
var (source, initial) = Resolve(novelKey, data, kind);
var ctxBuilder = new DownloadContextBuilder<T>().WithLinks(Array.Empty<SourceLink>()); // placeholder, filled later.
var ctxBuilder = new DownloadContextBuilder<RawType>().WithLinks(Array.Empty<SourceLink>()); // placeholder, filled later.
return new LinkStage(source, initial, data, ctxBuilder);
}
@@ -71,11 +84,12 @@ namespace Beam.Temporary.Cli {
/* ──────────────────────────── Stage types ─────────────────────────── */
private sealed record LinkStage(
WebResource Source,
State Initial,
BeamDataDictionary Data,
DownloadContextBuilder<T> CtxBuilder) : ILinkStage {
DownloadContextBuilder<RawType> CtxBuilder) : ILinkStage, IAlternativeLinkStage {
private State? endState;
private bool linksFrozen = false;
@@ -97,6 +111,11 @@ namespace Beam.Temporary.Cli {
return new TransformStage(Source, Data, CtxBuilder);
}
public IAlternativeTransformStage WithLinks(IEnumerable<SourceLink> links) {
CtxBuilder.WithLinks(links);
return new TransformStage(Source, Data, CtxBuilder);
}
public ILinkStage WithRange(Range range) {
if (linksFrozen)
throw new InvalidOperationException($"WithRange must be called before WithLinkGenerator");
@@ -114,24 +133,29 @@ namespace Beam.Temporary.Cli {
private sealed record TransformStage(
WebResource Source,
BeamDataDictionary Data,
DownloadContextBuilder<T> CtxBuilder) : ITransformStage {
public IContextStage WithTransformer<U>(Func<DataBindings, Func<object, T>> factory) {
DownloadContextBuilder<RawType> CtxBuilder) : ITransformStage, IAlternativeTransformStage {
public IContextStage WithTransformer(Func<DataBindings, AsyncTransformer<RawType, OutType>> factory) {
var transformer = factory(Data.Bindings[Source.Bindings]);
return new ContextStage<U>(CtxBuilder, transformer);
return new ContextStage(CtxBuilder, transformer);
}
public IContextStage WithTransformer(AsyncTransformer<RawType, OutType> transformer) {
return new ContextStage(CtxBuilder, transformer);
}
}
private sealed class ContextStage<U> : IContextStage {
private readonly DownloadContextBuilder<T> _ctxBuilder;
private readonly Func<object, T> _transformer;
private sealed class ContextStage : IContextStage {
private readonly DownloadContextBuilder<RawType> _ctxBuilder;
private readonly AsyncTransformer<RawType, OutType> _transformer;
private int _parallelism = 4;
private bool useFragments = false;
public ContextStage(DownloadContextBuilder<T> ctxBuilder, Func<object, T> transformer) {
public ContextStage(DownloadContextBuilder<RawType> ctxBuilder, AsyncTransformer<RawType, OutType> transformer) {
_ctxBuilder = ctxBuilder;
_transformer = transformer;
}
public IContextStage Configure(Action<DownloadContextBuilder<T>> configure) {
public IContextStage Configure(Action<DownloadContextBuilder<RawType>> configure) {
configure(_ctxBuilder);
return this;
}
@@ -151,21 +175,66 @@ namespace Beam.Temporary.Cli {
return this;
}
public DownloadEnumerable<T> Build() {
var context = _ctxBuilder.Build();
SequentialFragmentDownloader<T> sequentialDownloader = new(
context,
ctx => new UnitFragmentDownloader<T>(
public IContextStage UseFragments() {
useFragments = true;
return this;
}
private object ConstructUnitDownloader(DownloadContext<RawType> context) {
return (useFragments, _transformer, context.AsyncFailurePredicates) switch {
// ──────────────── fragmented HTML ────────────────
(true, AsyncTransformer<HtmlDocument, OutType> asyncHtmlTransformer,
AsyncDownloadFailurePredicate<HtmlDocument>[] documentFailurePredicates)
=> new UnitFragmentDownloader<OutType>(
context.Web,
_transformer,
context.AsyncFailurePredicates,
asyncHtmlTransformer,
documentFailurePredicates,
_parallelism,
context.DownloadLogger),
context.DownloadLogger);
var enumerable = new DownloadEnumerable<T>(
sequentialDownloader
.UnwrapFragmented());
sequentialDownloader.DisposeAsync().AsTask().Wait();
// ──────────────── fragmented binary ────────────────
(true, AsyncTransformer<byte[], OutType> asyncBinaryTransformer,
AsyncDownloadFailurePredicate<byte[]>[] responseFailurePredicates)
=> new UnitFragmentDownloaderBinary<OutType>(
context.Client,
asyncBinaryTransformer,
responseFailurePredicates,
_parallelism,
context.DownloadLogger),
// ──────────────── single HTML ────────────────
(false, AsyncTransformer<HtmlDocument, OutType> asyncHtmlTransformer,
AsyncDownloadFailurePredicate<HtmlDocument>[] documentFailurePredicates)
=> new UnitDownloader<OutType>(
context.Web,
asyncHtmlTransformer,
documentFailurePredicates),
// ──────────────── single binary ────────────────
(false, AsyncTransformer<byte[], OutType> asyncBinaryTransformer,
AsyncDownloadFailurePredicate<byte[]>[] responseFailurePredicates)
=> new UnitDownloaderBinary<OutType>(
context.Client,
asyncBinaryTransformer,
responseFailurePredicates),
_ => throw new Exception($"Unsupported transformer / failure-predicate combination. Missing pattern: {useFragments} , {_transformer.GetType().AsUniqueName()} , {context.AsyncFailurePredicates?.GetType().AsUniqueName()}"),
};
}
private IAsyncEnumerator<Ordered<OutType>> ConstructDownloader(DownloadContext<RawType> context) {
var copyOfContext = context.CreateBuilder().Build();
return useFragments switch {
true => new SequentialFragmentDownloader<RawType, OutType>(
copyOfContext,
ctx => (IUnitDownloader<Fragment<Ordered<OutType>>>)ConstructUnitDownloader(ctx),
context.DownloadLogger).UnwrapFragmented(),
false => new SequentialDownloader<RawType, OutType>(
copyOfContext,
ctx => (IUnitDownloader<OutType>)ConstructUnitDownloader(ctx),
context.DownloadLogger).WrapOrdered()
};
}
public DownloadEnumerable<OutType> Build() {
var context = _ctxBuilder.Build();
var enumerable = new DownloadEnumerable<OutType>(ConstructDownloader(context));
return enumerable;
}
}