riley/Beam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance project metadata and refactor core classes

Browse Source 
Updated project files for `Beam.Dynamic`, `Beam.Exports`, `Beam.Temporary.Cli`, and `Beam` to include additional metadata and specific package versions. Refactored `DataBindings` and `ResolvedBindings` to records, added a new `Text` property in `Binding.cs`, and introduced `ParseNumbers` in `OnlineCleaner`. New classes `PuppetContext` and `PuppetUnitDownloader` added for Playwright integration. Introduced `ImmutableState` struct and `UnitDownloaderBinary` class for improved download management. Updated tests in `UnitTest1.cs` for number localization. Added `Beam.Puppeteer` project to the solution.
This commit is contained in:
qwsdcvghyu89
2025-06-23 02:11:19 +03:00
parent a9a22ea23d
commit 482a46b568
27 changed files with 354 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Beam.Temporary.Cli/Beam.Temporary.Cli.csproj
+13 -15
View File
@@ -5,9 +5,22 @@
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Title>Beam.Temporary.Cli</Title>
<Authors>aeqw89</Authors>
<Company>qwsdcvghyu</Company>
<Description>A temporary CLI for Beam providing several useful mechanisms</Description>
<PackageProjectUrl>https://github.com/qwsdcvghyu89/Beam</PackageProjectUrl>
<RepositoryUrl>https://github.com/qwsdcvghyu89/Beam</RepositoryUrl>
<PackageId>aeqw89.Beam.Temporary.Cli</PackageId>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="aeqw89.Beam.Exports" Version="1.0.0" />
<PackageReference Include="aeqw89.DataKeys" Version="1.0.1" />
<PackageReference Include="aeqw89.PersistentData" Version="1.0.0" />
<PackageReference Include="aeqw89.Beam" Version="1.0.0" />
<PackageReference Include="aeqw89.Beam.Dynamic" Version="1.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
@@ -16,19 +29,4 @@
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Beam.Dynamic\Beam.Dynamic.csproj" />
<ProjectReference Include="..\Beam.Exports\Beam.Exports.csproj" />
<ProjectReference Include="..\Beam\Beam.csproj" />
</ItemGroup>
<ItemGroup>
<Reference Include="aeqw89.DataKeys">
<HintPath>..\..\aeqw89.DataKeys\aeqw89.DataKeys\bin\Debug\net9.0\aeqw89.DataKeys.dll</HintPath>
</Reference>
<Reference Include="aeqw89.PersistentData">
<HintPath>..\..\aeqw89.PersistentData\aeqw89.PersistentData\bin\Release\net9.0\aeqw89.PersistentData.dll</HintPath>
</Reference>
</ItemGroup>
</Project>
Beam.Temporary.Cli/DownloadBuilder.cs
+37 -14
View File
@@ -1,5 +1,6 @@
using aeqw89.DataKeys;
using Beam.Dynamic;
using Beam;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
@@ -24,17 +25,18 @@ namespace Beam.Temporary.Cli {
public interface ILinkStage {
ITransformStage WithLink();
ITransformStage WithLinkGenerator();
ILinkStage WithRange(Range range);
}
public interface ITransformStage {
IContextStage WithTransformer(Func<DataBindings, HtmlTransformer<T>> factory);
IContextStage<U> WithTransformer<U>(Func<DataBindings, AsyncTransformer<T, U>> factory);
}
public interface IContextStage {
IContextStage Configure(Action<DownloadContextBuilder<T>> configure);
IContextStage WithParallelism(int degree);
IContextStage WithTimeout(TimeSpan timeout);
IContextStage WithRetryReporter(IProgress<RetryReport> reporter);
public interface IContextStage<U> {
IContextStage<U> Configure(Action<DownloadContextBuilder<T>> configure);
IContextStage<U> WithParallelism(int degree);
IContextStage<U> WithTimeout(TimeSpan timeout);
IContextStage<U> WithRetryReporter(IProgress<RetryReport> reporter);
DownloadEnumerable<T> Build();
}
@@ -61,7 +63,7 @@ namespace Beam.Temporary.Cli {
initial = textRecord.Resource.MetaTemplateInitialData ?? throw new InvalidOperationException("Meta template data missing.");
} else {
source = textRecord.AssociatedSource ?? throw new InvalidOperationException($"Text source missing for '{novelKey}'.");
initial = textRecord.Resource.TemplateInitialData ?? throw new InvalidOperationException("Template initial data missing.");
initial = textRecord.Resource.TemplateInitialData;
}
return (source, initial);
@@ -74,6 +76,10 @@ namespace Beam.Temporary.Cli {
State Initial,
BeamDataDictionary Data,
DownloadContextBuilder<T> CtxBuilder) : ILinkStage {
private State? endState;
private bool linksFrozen = false;
public ITransformStage WithLink() {
var link = Data.Templates[Source.Key].Builder.Build(Initial);
CtxBuilder.WithLinks(new[] { link });
@@ -85,28 +91,45 @@ namespace Beam.Temporary.Cli {
var generator = SourceLinkEnumerable.FromGenerator(new OrderedSourceLinkGenerator(
template.Builder,
new NumberedStateChanger(template.Factory.Behavior),
Initial));
Initial, endState));
CtxBuilder.WithLinks(generator);
linksFrozen = true;
return new TransformStage(Source, Data, CtxBuilder);
}
public ILinkStage WithRange(Range range) {
if (linksFrozen)
throw new InvalidOperationException($"WithRange must be called before WithLinkGenerator");
if (range.End.Value < range.Start.Value)
throw new ArgumentOutOfRangeException(nameof(range), $" start must be < end");
var template = Data.Templates[Source.Key];
var stateChanger = new NumberedStateChanger(template.Factory.Behavior);
endState = Initial.Copy();
stateChanger.Apply(Initial, range.Start.Value - 1);
stateChanger.Apply(endState, range.End.Value - 1);
return this;
}
}
private sealed record TransformStage(
WebResource Source,
BeamDataDictionary Data,
DownloadContextBuilder<T> CtxBuilder) : ITransformStage {
public IContextStage WithTransformer(Func<DataBindings, HtmlTransformer<T>> factory) {
public IContextStage WithTransformer<U>(Func<DataBindings, Func<object, T>> factory) {
var transformer = factory(Data.Bindings[Source.Bindings]);
CtxBuilder.WithTransformer(transformer);
return new ContextStage(CtxBuilder);
return new ContextStage<U>(CtxBuilder, transformer);
}
}
private sealed class ContextStage : IContextStage {
private sealed class ContextStage<U> : IContextStage {
private readonly DownloadContextBuilder<T> _ctxBuilder;
private readonly Func<object, T> _transformer;
private int _parallelism = 4;
public ContextStage(DownloadContextBuilder<T> ctxBuilder) => _ctxBuilder = ctxBuilder;
public ContextStage(DownloadContextBuilder<T> ctxBuilder, Func<object, T> transformer) {
_ctxBuilder = ctxBuilder;
_transformer = transformer;
}
public IContextStage Configure(Action<DownloadContextBuilder<T>> configure) {
configure(_ctxBuilder);
@@ -134,7 +157,7 @@ namespace Beam.Temporary.Cli {
context,
ctx => new UnitFragmentDownloader<T>(
context.Web,
context.AsyncTranformer,
_transformer,
context.AsyncFailurePredicates,
_parallelism,
context.DownloadLogger),
Beam.Temporary.Cli/LinkCollection.cs
+13
View File
@@ -0,0 +1,13 @@
using aeqw89.DataKeys;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Beam.Temporary.Cli {
internal class LinkCollection(DataKey<string> key, List<SourceLink> links) {
public DataKey<string> Key { get; set; } = key;
public List<SourceLink> Links { get; set; } = links;
}
}
Beam.Temporary.Cli/NovelStatics.cs
+2 -2
View File
@@ -50,8 +50,8 @@ namespace Beam.Temporary.Cli {
FriendlyName = "My House Of Horrors",
AssociatedSource = wdsAgg,
AssociatedMetaSource = wdsAux,
TemplateInitialData = new State(["24349", "2896325"]),
MetaTemplateInitialData = new State(["24349"])
TemplateInitialData = new ImmutableState(["24349", "2896325"]),
MetaTemplateInitialData = new ImmutableState(["24349"])
};
sdd.Novels.TryAdd(novel.Key, novel);
Beam.Temporary.Cli/Program.cs
+10 -18
View File
@@ -49,6 +49,8 @@ namespace Beam.Temporary.Cli {
NovelStatics.Define_WoDuShu_HouseOfHorrors(BeamData);
ClassicTemplates.Register(BeamData);
await sharedContext.ForceSave();
CancellationTokenSource cts = new();
HtmlTransformer<IDocumentMetaData> ArticleDataTransformer(DataBindings? binding) => (x) => {
@@ -75,38 +77,23 @@ namespace Beam.Temporary.Cli {
};
var novel = new DataKey<TextResource>("novels:house_of_horrors");
var context_aux = Architecture.GetMeta(web, novel, BeamData, cts.Token);
context_aux.RetryReporter = new Progress<RetryReport>((x) => Console.WriteLine($"Failed. Trying again. {x.TryNumber}"));
var metaDownloader = new DownloadEnumerable<IDocumentMetaData>(
new SequentialFragmentDownloader<IDocumentMetaData>(
context_aux,
(c) => new UnitFragmentDownloader<IDocumentMetaData>(c.Web, c.AsyncTranformer, c.AsyncFailurePredicates, 4, logger),
logger)
.UnwrapFragmented());
var metadata = (await metaDownloader.FirstAsync());
var metadata2 = await DownloadBuilder<IDocumentMetaData>.FromMeta(novel, BeamData)
.WithLink()
.WithTransformer(ArticleDataTransformer)
.Configure((x) => x
.WithDownloadLogger(logger)
.WithRetryReporter(new Progress<RetryReport>())
.WithTimeOut(TimeSpan.FromSeconds(15)))
.Build()
.FirstAsync();
var context = Architecture.GetTextRecord(web, novel, BeamData, metadata.Data, cts.Token);
context.DownloadReporter = new Progress<DownloadReport>((x) => Console.WriteLine(x));
var downloader = new DownloadEnumerable<IDocument>(
new SequentialFragmentDownloader<IDocument>(
context,
(c) => new UnitFragmentDownloader<IDocument>(c.Web, c.AsyncTranformer, c.AsyncFailurePredicates, 4, logger),
logger)
.UnwrapFragmented());
var downloader2 = DownloadBuilder<IDocument>.FromText(novel, BeamData)
.WithRange(1..5)
.WithLinkGenerator()
.WithTransformer((x) => DocumentTransformer(x, metadata2.Data))
.Configure((x) => x
.WithDownloadLogger(logger)
.WithDownloadReporter(new Progress<DownloadReport>((x) => logger.LogInformation(x.ToString())))
.WithTimeOut(TimeSpan.FromSeconds(15))
)
@@ -122,11 +109,16 @@ namespace Beam.Temporary.Cli {
continue;
if (meta is not ArticleData articleMetaData)
continue;
if (!download.Data.MetaData.TryGetValue(Architecture.BookKey, out var bookmeta))
continue;
if (meta is not ArticleData bookMetaData)
continue;
//Console.WriteLine($"Title: {data.Name}");
//Console.WriteLine($"Description: {data.Description}");
//Console.WriteLine($"Categories: {data.Categories.Aggregate((x, y) => $"{x}; {y}")}");
//Console.WriteLine($"Authors: {data.Authors.Aggregate((x,y) => $"{x}; {y}")}");
Console.WriteLine($"Chapter title: {articleMetaData.Name}");
Console.WriteLine($"Book title: {bookMetaData.Name}");
//Console.WriteLine($"Content: {download}");
//translationTasks.Add(Task.Run(async () => {
Beam.Temporary.Cli/StateChangerFactory.cs
+1
View File
@@ -7,6 +7,7 @@ using System.Threading.Tasks;
namespace Beam.Temporary.Cli {
public class StateChangerFactory {
[JsonIgnore]
public IStateChangeBehaviour Behavior => FactoryTable[StateChangerKey]();
[JsonInclude]
Beam.Temporary.Cli/TextResource.cs
+2 -2
View File
@@ -13,8 +13,8 @@ namespace Beam.Temporary.Cli {
public string? FriendlyName { get; set; }
public DataKey<WebResource>? AssociatedSource { get; set; }
public DataKey<WebResource>? AssociatedMetaSource { get; set; }
public required State TemplateInitialData { get; set; }
public State? MetaTemplateInitialData { get; set; }
public required ImmutableState TemplateInitialData { get; set; }
public ImmutableState? MetaTemplateInitialData { get; set; }
public TextResourceRecord ToRecord(BeamDataDictionary sdd) {
return new(this,