refactor: modularize Beam into new projects and interfaces

- Introduced modularity by splitting Beam into new projects: Beam.Abstractions, Beam.Models, and Beam.Downloaders.
- Refactored existing classes into appropriate namespaces and projects.
- Replaced specific implementations with abstractions (e.g., SourceLinkBuilder to LinkBuilder, State to IState, etc.).
- Updated interfaces: added ITemplate, IArticleData, IDownloadReport, and others for improved extensibility.
- Removed deprecated classes like SourceLinkBuilder and StateChangerFactory.
- Enhanced link handling in downloaders by refactoring to use `string` over `SourceLink`.
- Consolidated shared logic under Beam.Abstractions.
This commit is contained in:
qwsdcvghyu89
2025-09-22 01:51:46 +10:00
parent a7d148a96f
commit 7ed05abdb8
128 changed files with 2058 additions and 1804 deletions
@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<RootNamespace>Beam.Abstract</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="aeqw89.DataKeys" Version="2.1.0" />
<PackageReference Include="aeqw89.PersistentData" Version="1.3.3" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
</ItemGroup>
</Project>
+10
View File
@@ -0,0 +1,10 @@
namespace Beam.Abstractions;
public interface IArticleData : IDocumentMetaData, IEquatable<IArticleData?> {
string? Name { get; set; }
string[] Authors { get; set; }
string? Language { get; set; }
string[] Categories { get; set; }
string? Version { get; set; }
string? Description { get; set; }
}
+27
View File
@@ -0,0 +1,27 @@
using Beam.Dynamic;
namespace Beam.Abstractions;
public interface IDataBindings {
IDataProvider<string>? Title { get; set; }
IDataProvider<string[]>? Authors { get; set; }
IDataProvider<string>? Description { get; set; }
IDataProvider<string>? Content { get; set; }
IDataProvider<string[]>? Language { get; set; }
IDataProvider<string[]>? Tags { get; set; }
IDataProvider<string>? Publisher { get; set; }
IDataProvider<DateTimeOffset>? PublicationDate { get; set; }
IDataProvider<string>? ISBN { get; set; }
IDataProvider<int>? PageCount { get; set; }
IDataProvider<string>? CoverImage { get; set; }
IDataProvider<string[]>? Series { get; set; }
IDataProvider<int>? Edition { get; set; }
IDataProvider<string[]>? Contributors { get; set; }
IDataProvider<string[]>? Subjects { get; set; }
IDataProvider<string>? Rights { get; set; }
IDataProvider<string[]>? TableOfContents { get; set; }
IDataProvider<string[]>? PagesDropDown { get; set; }
IDataProvider<string>? NextPageButton { get; set; }
IDataProvider<string>? PreviousPageButton { get; set; }
Dictionary<string, IDataProvider?> Providers { get; set; }
}
+14
View File
@@ -0,0 +1,14 @@
using HtmlAgilityPack;
using System.Text.Json.Serialization;
namespace Beam.Dynamic;
public interface IDataProvider {
public string GetString(HtmlDocument document)
=> (this as IDataProvider<object>)?.Get(document)?.ToString() ?? "";
}
public interface IDataProvider<out T> : IDataProvider {
public T Get(HtmlDocument document);
//public HtmlNode? GetNode(HtmlDocument document);
}
+28
View File
@@ -0,0 +1,28 @@
using aeqw89.DataKeys;
namespace Beam.Abstractions;
public interface IDocument {
/// <summary>
/// The file name of the document. Must be valid in both <c>UNIX</c>,
/// <c>WINDOWS</c>, <c>APPLE</c>, and <c>ANDROID</c> file systems.
/// </summary>
string Filename { get; }
/// <summary>
/// Additional descriptive data
/// </summary>
IDictionary<IDataKey<IDocumentMetaData>, IDocumentMetaData> MetaData { get; }
/// <summary>
/// Retrieves the binary representation for the <see cref="IDocument"/>
/// </summary>
/// <returns>Binary representation of the <see cref="IDocument"/></returns>
byte[] ToBytes();
/// <summary>
/// Retrieves the string representation for the <see cref="IDocument"/>
/// </summary>
/// <returns>String representation of the <see cref="IDocument"/></returns>
string ToString();
}
+7
View File
@@ -0,0 +1,7 @@
using System.Text.Json;
namespace Beam.Abstractions;
public interface IDocumentMetaData {
string AsJson(JsonSerializerOptions? options = null);
}
+3
View File
@@ -0,0 +1,3 @@
namespace Beam.Abstractions;
public interface IDownloadReport { }
+11
View File
@@ -0,0 +1,11 @@
using Beam.Models;
namespace Beam.Abstractions;
public interface ILinkBuilder {
/// <summary>
/// Produces a concrete <see cref="SourceLink"/> using values from an external <see cref="State"/> object.
/// </summary>
/// <param name="parameterValues">Object providing positional values.</param>
string Build(IReadOnlyState parameterValues);
}
+6
View File
@@ -0,0 +1,6 @@
namespace Beam.Abstractions;
public interface IOrdered<out T> {
T Data { get; }
int Order { get; }
}
+6
View File
@@ -0,0 +1,6 @@
namespace Beam.Models;
public interface IReadOnlyState {
public string[] GetState();
IReadOnlyState Copy();
}
+3
View File
@@ -0,0 +1,3 @@
namespace Beam.Abstractions;
public interface IResourceDictionary { }
+6
View File
@@ -0,0 +1,6 @@
namespace Beam.Abstractions;
public interface IRetryReport {
int TryNumber { get; }
string Link { get; }
}
+8
View File
@@ -0,0 +1,8 @@
using Beam.Models;
namespace Beam.Abstractions;
public interface IState : IReadOnlyState {
void SetState(string[] state);
new IState Copy();
}
@@ -0,0 +1,10 @@
using Beam.Models;
namespace Beam.Abstractions;
/// <summary>
/// Defines how a url template should should be updated, in what order, and by how much
/// </summary>
public interface IStateChangeBehaviour {
public void Apply(IState state, object stimulus);
}
@@ -0,0 +1,3 @@
namespace Beam.Abstractions;
public interface IStateChangerFactory { }
+8
View File
@@ -0,0 +1,8 @@
using aeqw89.DataKeys;
namespace Beam.Abstractions;
public interface ITemplate : IKeyed<ITemplate> {
IStateChangerFactory Factory { get; set; }
ILinkBuilder Builder { get; set; }
}
+7
View File
@@ -0,0 +1,7 @@
namespace Beam.Abstractions;
public interface IUnitDownloader<T> {
public int LinksPerDownload { get; }
public Task<(bool, T?)> TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount = 7, IProgress<IRetryReport>? tryProgress = null);
}
@@ -5,6 +5,7 @@ using System.Net;
using System.Reflection.PortableExecutable;
using System.Text;
using System.Threading.Tasks;
using Beam.Abstractions;
namespace Beam {
public class ApiCallBuilder(HttpClient client) {
@@ -25,10 +26,6 @@ namespace Beam {
return WithUri(uri.AbsoluteUri);
}
public ApiCallBuilder WithUri(SourceLink uri) {
return WithUri(uri.Link);
}
public ApiCallBuilder WithRequestData(object? data) {
Data = data;
return this;
+50
View File
@@ -0,0 +1,50 @@
using aeqw89.DataKeys;
using aeqw89.PersistentData;
using Beam.Dynamic;
using Beam.Models;
namespace Beam.Data {
using BeamFile = Models.File;
public class BeamDataContext : BaseDataDictionary {
#region Tables
public Table<Template> Templates {
get => GetOrCreateTable<Template>(nameof(Templates));
set => Set(nameof(Templates), value);
}
public Table<DataBindings> Bindings {
get => GetOrCreateTable<DataBindings>(nameof(Bindings));
set => Set(nameof(Bindings), value);
}
public Table<WebResource> Resources {
get => GetOrCreateTable<WebResource>(nameof(Resources));
set => Set(nameof(Resources), value);
}
public Table<ResourceDictionary> ResourceDictionaries {
get => GetOrCreateTable<ResourceDictionary>(nameof(ResourceDictionaries));
set => Set(nameof(ResourceDictionaries), value);
}
public Table<ImmutableState> InitialStates {
get => GetOrCreateTable<ImmutableState>(nameof(InitialStates));
set => Set(nameof(InitialStates), value);
}
public Table<BeamFile> Files {
get => GetOrCreateTable<BeamFile>(nameof(Files));
set => Set(nameof(Files), value);
}
#endregion
#region Junctions
public Junction<WebResource, ResourceDictionary> WebResourceToResourceDictionaryJunction {
get => GetOrCreateJunction<WebResource, ResourceDictionary>(nameof(WebResourceToResourceDictionaryJunction));
set => Set(nameof(WebResourceToResourceDictionaryJunction), value);
}
#endregion
#region Computed
public Dictionary<DataKey<WebResource>, ResourceDictionary[]> ResourceDictionariesByNovel =>
Resources.Keys.ToDictionary(x => x,
x => WebResourceToResourceDictionaryJunction[x].Select(y => ResourceDictionaries[y]).ToArray());
#endregion
}
}
@@ -1,10 +1,9 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Text;
using Beam.Abstractions;
using Beam.Models;
using static Beam.Exceptions.Exceptions;
namespace Beam {
namespace Beam.Data {
/// <summary>
/// Describes where a <see cref="Parameter"/> token should be inserted relative to the runtime value
/// that ultimately replaces it when building a <see cref="SourceLink"/>.
@@ -129,7 +128,7 @@ namespace Beam {
/// </remarks>
/// <param name="host">DNS host name (e.g. <c>api.example.com</c>).</param>
/// <param name="protocol">Transport protocol; defaults to <c>https</c>.</param>
public class SourceLinkBuilder(string host, string protocol = "https") {
public class LinkBuilder(string host, string protocol = "https") : ILinkBuilder {
/// <summary>
/// Gets or sets the scheme part of the URL (e.g. <c>https</c>, <c>http</c>).
/// </summary>
@@ -148,8 +147,8 @@ namespace Beam {
/// <summary>
/// Produces a deep copy whose <see cref="Segments"/> and contained collections are detached from the original.
/// </summary>
public SourceLinkBuilder Clone()
=> new SourceLinkBuilder(Host, Protocol) {
public LinkBuilder Clone()
=> new LinkBuilder(Host, Protocol) {
Segments = [.. Segments.Select(static x => x.Clone())]
};
@@ -241,7 +240,7 @@ namespace Beam {
/// Replaces the whole <see cref="Segments"/> collection with the supplied <paramref name="segments"/>, each represented as a <see cref="LinkSegment"/>.
/// </summary>
/// <returns>This instance for fluent calls.</returns>
public SourceLinkBuilder WithSegments(params IEnumerable<string> segments) {
public LinkBuilder WithSegments(params IEnumerable<string> segments) {
Segments = segments.Select(static x => new LinkSegment(x)).ToList();
return this;
}
@@ -250,7 +249,7 @@ namespace Beam {
/// Replaces the <see cref="Segments"/> collection with <paramref name="count"/> empty segments.
/// </summary>
/// <param name="count">Number of segments to create.</param>
public SourceLinkBuilder WithSegments(int count)
public LinkBuilder WithSegments(int count)
=> WithSegments(Enumerable.Repeat("", count));
#endregion
@@ -258,7 +257,7 @@ namespace Beam {
/// <summary>
/// Replaces parameters of the <paramref name="i"/>th segment using the supplied identifiers.
/// </summary>
public SourceLinkBuilder WithParameters(int i, params string[] parameters) {
public LinkBuilder WithParameters(int i, params string[] parameters) {
Segments[i].WithParameters(parameters);
return this;
}
@@ -266,7 +265,7 @@ namespace Beam {
/// <summary>
/// Replaces parameters of the <paramref name="i"/>th segment using explicit name/position tuples.
/// </summary>
public SourceLinkBuilder WithParameters(int i, params (string, Position)[] parameters) {
public LinkBuilder WithParameters(int i, params (string, Position)[] parameters) {
Segments[i].WithParameters(parameters);
return this;
}
@@ -325,12 +324,15 @@ namespace Beam {
return count;
}
public string Build(IReadOnlyState state)
=> Build(state.GetState().ToArray<object>());
#region Build
/// <summary>
/// Produces a concrete <see cref="SourceLink"/> using values from an external <see cref="State"/> object.
/// </summary>
/// <param name="parameterValues">Object providing positional values.</param>
public SourceLink Build(State parameterValues)
public string Build(State parameterValues)
=> Build(parameterValues.GetState());
/// <summary>
@@ -339,7 +341,7 @@ namespace Beam {
/// <param name="parameterValues">Flat array of values that will be written in the order that parameters appear when segments are enumerated lefttoright. Any optional parameters must still appear as null if missing.</param>
/// <returns>The completed <see cref="SourceLink"/>.</returns>
/// <exception cref="ArgumentOutOfRangeException">If the supplied value count does not match <see cref="GetParameterCount"/>().</exception>
public SourceLink Build(params object?[] parameterValues) {
public string Build(params object?[] parameterValues) {
ArgumentOutOfRangeException.ThrowIfNotEqual(parameterValues.Length, GetParameterCount());
StringBuilder link = new();
@@ -357,10 +359,10 @@ namespace Beam {
if (segment.Parameters[i].Position.HasFlag(Position.Optional))
continue;
else
throw new ArgumentException(S.M.RequiredArgumentMissing);
throw new ArgumentException(string.Format(link_builder_argument_missing, pvC, segment.Parameters[i].Name));
if (segment.Parameters[i].Position.HasFlag(Position.Query) && Segments[^1] != segment)
throw new ArgumentException(S.M.QueryParametersOnlyAtLastSegment);
throw new ArgumentException(string.Format(link_builder_query_only_at_last, i));
if (segment.Parameters[i].Position.HasFlag(Position.Query))
if (!startedQueryString) {
@@ -378,10 +380,10 @@ namespace Beam {
if (parameterValues[pvC] is not null)
link.Append(parameterValues[pvC++]);
else if (!segment.Parameters[i].Position.HasFlag(Position.Optional))
throw new ArgumentException(S.M.RequiredArgumentMissing);
throw new ArgumentException(string.Format(link_builder_argument_missing, pvC, segment.Parameters[i].Name));
if (segment.Parameters[i].Position.HasFlag(Position.Query | Position.After))
throw new ArgumentException(S.M.QueryFlagIncompatibleWithAfterFlag);
throw new ArgumentException(string.Format(link_builder_incompatible_flag, nameof(Position.Query), nameof(Position.After)));
if (segment.Parameters[i].Position.HasFlag(Position.After))
link.Append(segment.Parameters[i].Name);
@@ -393,7 +395,7 @@ namespace Beam {
link.Append(segment.Suffix);
}
return new SourceLink(link.ToString());
return link.ToString();
}
#endregion
}
@@ -1,16 +1,10 @@

using System.Text.Json.Serialization;
using aeqw89.DataKeys;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using Beam.Abstractions;
using Beam.Models;
namespace Beam.Models {
public class ResourceDictionary : IKeyed<ResourceDictionary> {
namespace Beam.Data {
public class ResourceDictionary : IKeyed<ResourceDictionary>, IResourceDictionary {
public required DataKey<ResourceDictionary> Key { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+19
View File
@@ -0,0 +1,19 @@
using aeqw89.DataKeys;
using Beam.Abstractions;
using Beam.Dynamic;
namespace Beam.Data {
public record class Template : ITemplate {
public required DataKey<ITemplate> Key { get; set; }
public required StateChangerFactory Factory { get; set; }
IStateChangerFactory ITemplate.Factory {
get => Factory;
set => Factory = (StateChangerFactory)value;
}
public required LinkBuilder Builder { get; set; }
ILinkBuilder ITemplate.Builder {
get => Builder;
set => Builder = (LinkBuilder)value;
}
}
}
@@ -1,13 +1,7 @@
using aeqw89.PersistentData;
using aeqw89.DataKeys;
using aeqw89.DataKeys;
using Beam.Dynamic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Beam.Models {
namespace Beam.Data {
/// <summary>
/// Represents a specific resource accessible online; e.g. a book's contents.
/// </summary>
@@ -1,41 +1,33 @@
using HtmlAgilityPack;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Abstractions;
using Beam.Models;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
namespace Beam {
namespace Beam.Downloaders {
//public delegate T HtmlTransformer<out T>(HtmlDocument doc);
public delegate Task<U> AsyncTransformer<in T, U>(T elem);
//public delegate Task<T> AsyncHtmlTransformer<T>(HtmlDocument doc);
//public delegate Task<T> AsyncBinaryTransformer<T>(byte[] bin);
public class DownloadContext<RawType> : IDisposable {
public class DownloadContext<RawType> {
private bool disposedValue;
public DownloadContextBuilder<RawType> CreateBuilder()
=> DownloadContextBuilder<RawType>.FromContext(this);
public HttpClient Client { get; }
public HtmlWeb Web { get; }
public IProgress<DownloadReport>? DownloadReporter { get; set; }
public IProgress<RetryReport>? RetryReporter { get; set; }
public IProgress<IDownloadReport>? DownloadReporter { get; set; }
public IProgress<IRetryReport>? RetryReporter { get; set; }
public AsyncDownloadFailurePredicate<RawType>?[]? AsyncFailurePredicates { get; }
public TimeSpan TimeOut { get; set; }
public IEnumerable<SourceLink> Links { get; }
public IEnumerable<string> Links { get; }
public CancellationToken CancellationToken { get; }
public DocumentCache Cache { get; private set; } = [];
public ILogger? DownloadLogger { get; set; }
public DownloadContext(HtmlWeb web,
HttpClient client,
IEnumerable<SourceLink> links,
IEnumerable<string> links,
CancellationToken cancellationToken = default,
IProgress<DownloadReport>? downloadReporter = null,
IProgress<RetryReport>? retryReporter = null,
IProgress<IDownloadReport>? downloadReporter = null,
IProgress<IRetryReport>? retryReporter = null,
AsyncDownloadFailurePredicate<RawType>?[]? asyncFailurePredicates = null,
TimeSpan? timeOut = null,
ILogger? downloadLogger = null) {
@@ -1,21 +1,18 @@
using HtmlAgilityPack;
using Beam.Abstractions;
using Beam.Models;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Beam {
namespace Beam.Downloaders {
public class DownloadContextBuilder<RawType> {
private HtmlWeb _web;
private HttpClient _client;
private IProgress<DownloadReport>? _downloadReporter;
private IProgress<RetryReport>? _retryReporter;
private IProgress<IDownloadReport>? _downloadReporter;
private IProgress<IRetryReport>? _retryReporter;
private AsyncDownloadFailurePredicate<RawType>?[] _asyncFailurePredicates = [];
private TimeSpan _timeOut;
private IEnumerable<SourceLink> _links;
private IEnumerable<string> _links;
private CancellationToken _cancellationToken;
private DocumentCache _cache;
private ILogger? _downloadLogger;
@@ -39,12 +36,12 @@ namespace Beam {
return this;
}
public DownloadContextBuilder<RawType> WithDownloadReporter(IProgress<DownloadReport> downloadReporter) {
public DownloadContextBuilder<RawType> WithDownloadReporter(IProgress<IDownloadReport> downloadReporter) {
_downloadReporter = downloadReporter;
return this;
}
public DownloadContextBuilder<RawType> WithRetryReporter(IProgress<RetryReport> retryReporter) {
public DownloadContextBuilder<RawType> WithRetryReporter(IProgress<IRetryReport> retryReporter) {
_retryReporter = retryReporter;
return this;
}
@@ -59,7 +56,7 @@ namespace Beam {
return this;
}
public DownloadContextBuilder<RawType> WithLinks(IEnumerable<SourceLink> links) {
public DownloadContextBuilder<RawType> WithLinks(IEnumerable<string> links) {
_links = links;
return this;
}
@@ -1,14 +1,15 @@
using HtmlAgilityPack;
using Beam.Abstractions;
using Beam.Models;
using Microsoft.Extensions.Logging;
namespace Beam {
namespace Beam.Downloaders {
public class SequentialDownloader<RawType, OutType> : IAsyncEnumerator<OutType> {
public OutType Current { get; protected set; }
public DownloadContext<RawType> Context { get; }
public ILogger? Logger { get; set; }
public int LastOrder { get; set; } = 0;
protected IEnumerator<SourceLink> LinksEnumerator;
protected IEnumerator<string> LinksEnumerator;
public Func<IUnitDownloader<OutType>> GetUnitDownloader { get; set; }
@@ -41,17 +42,17 @@ namespace Beam {
List<Ordered<string>> links = [];
//Logger?.LogInformation("MoveNextAsync() \n\t -> Links.Current = {} ", LinksEnumerator.Current.Link.AbsoluteUri);
links.Add(new Ordered<string>(LinksEnumerator.Current.Link.AbsoluteUri, LastOrder++));
links.Add(new Ordered<string>(LinksEnumerator.Current, LastOrder++));
while (LinksEnumerator.MoveNext() && LinksEnumerator.Current != SourceLink.InvalidLink && links.Count < idealLinkCount)
links.Add(new Ordered<string>(LinksEnumerator.Current.Link.AbsoluteUri, LastOrder++));
while (LinksEnumerator.MoveNext() && !string.IsNullOrWhiteSpace(LinksEnumerator.Current) && links.Count < idealLinkCount)
links.Add(new Ordered<string>(LinksEnumerator.Current, LastOrder++));
//Logger?.LogInformation("MoveNextAsync() \n\t -> links.Count = {} ", links.Count);
if (links.Count == 0) {
Logger?.LogInformation("Out of links!");
return false;
}
if (links.Any((x) => x.Data == SourceLink.InvalidLink.Link.AbsoluteUri))
if (links.Any((x) => string.IsNullOrWhiteSpace(x.Data)))
return false;
var (result, downloadedT) = await unit.TryDownload(
@@ -1,7 +1,8 @@
using Microsoft.Extensions.Logging;
using System.Collections.Concurrent;
using Beam.Abstractions;
using Beam.Models;
using Microsoft.Extensions.Logging;
namespace Beam {
namespace Beam.Downloaders {
public class SequentialFragmentDownloader<RawType, OutType> : SequentialDownloader<RawType, Fragment<Ordered<OutType>>> {
public SequentialFragmentDownloader(
DownloadContext<RawType> context,
@@ -1,8 +1,8 @@
using HtmlAgilityPack;
namespace Beam {
public delegate Task<bool> AsyncDownloadFailurePredicate<in T>(T download);
using Beam.Abstractions;
using Beam.Models;
using HtmlAgilityPack;
namespace Beam.Downloaders {
/// <summary>
/// A download managing class that manages a singular download with failure-detection and exponential-backoff retries. This class is safe to instantiate per request.
/// </summary>
@@ -45,7 +45,7 @@ namespace Beam {
}
}
public async Task<(bool, T?)> TryDownload(Ordered<string>[] link, CancellationToken ct, int maximumRetryCount = 7, IProgress<RetryReport>? tryProgress = null) {
public async Task<(bool, T?)> TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount = 7, IProgress<IRetryReport>? tryProgress = null) {
if (link.Length == 0)
return (false, default);
@@ -1,8 +1,7 @@
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Beam.Abstractions;
using Beam.Models;
namespace Beam {
namespace Beam.Downloaders {
/// <summary>
/// A download-managing class that retrieves binary data through <see cref="HttpClient"/>,
/// applies an <see cref="AsyncBinaryTransformer{T}"/>, and supports failure detection
@@ -49,10 +48,10 @@ namespace Beam {
}
public async Task<(bool, T?)> TryDownload(
Ordered<string>[] link,
IOrdered<string>[] link,
CancellationToken ct,
int maximumRetryCount = 7,
IProgress<RetryReport>? tryProgress = null) {
IProgress<IRetryReport>? tryProgress = null) {
if (link.Length == 0) return (false, default);
T? result = default;
@@ -1,13 +1,10 @@
using HtmlAgilityPack;
using Beam.Abstractions;
using Beam.Exceptions;
using Beam.Models;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Beam {
namespace Beam.Downloaders {
public class UnitFragmentDownloader<T> : IUnitDownloader<Fragment<Ordered<T>>> {
public UnitFragmentDownloader(HtmlWeb web,
AsyncTransformer<HtmlDocument, T> transformer,
@@ -31,10 +28,10 @@ namespace Beam {
private readonly IUnitDownloader<T> UnitDownloader;
async Task<(bool, Fragment<Ordered<T>>?)> IUnitDownloader<Fragment<Ordered<T>>>.TryDownload(Ordered<string>[] link, CancellationToken ct, int maximumRetryCount, IProgress<RetryReport>? tryProgress) {
async Task<(bool, Fragment<Ordered<T>>?)> IUnitDownloader<Fragment<Ordered<T>>>.TryDownload(IOrdered<string>[] link, CancellationToken ct, int maximumRetryCount, IProgress<IRetryReport>? tryProgress) {
Fragment<Ordered<T>> fragment = new Fragment<Ordered<T>>(link.Length);
if (!Fragment<Ordered<T>>.TryAcquireUpdater(fragment, out var updater))
throw new S.AssertionException(S.M.NewFragmentShouldBeFree);
throw new AssertionException(Exceptions.Exceptions.fragment_locked);
bool isFailure = false;
await Parallel.ForEachAsync(link, async (x, pct) => {
pct.ThrowIfCancellationRequested();
@@ -42,12 +39,12 @@ namespace Beam {
var (result, downloadedT) = await UnitDownloader.TryDownload([x], ct, maximumRetryCount, tryProgress);
if (!result) {
Interlocked.Exchange(ref isFailure, true);
Logger?.LogError("Failed to retrieve {} order={}", x.Data, x.Order);
Logger?.LogError("Failed to retrieve {0} order={1}", x.Data, x.Order);
return;
}
if (downloadedT == null) {
Interlocked.Exchange(ref isFailure, true);
Logger?.LogCritical("Failed to retrieve {} order={}", x.Data, x.Order);
Logger?.LogCritical("Failed to retrieve {0} order={1}", x.Data, x.Order);
return;
}
updater(new Ordered<T>(downloadedT, x.Order));
@@ -1,10 +1,9 @@
using System;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Beam.Abstractions;
using Beam.Exceptions;
using Beam.Models;
using Microsoft.Extensions.Logging;
namespace Beam {
namespace Beam.Downloaders {
/// <summary>
/// Groups multiple binary downloads into a single Fragment, applying
/// failure detection and exponential-back-off retries for each link.
@@ -35,13 +34,13 @@ namespace Beam {
private readonly IUnitDownloader<T> UnitDownloader;
async Task<(bool, Fragment<Ordered<T>>?)> IUnitDownloader<Fragment<Ordered<T>>>.TryDownload(
Ordered<string>[] link,
IOrdered<string>[] link,
CancellationToken ct,
int maximumRetryCount,
IProgress<RetryReport>? tryProgress) {
IProgress<IRetryReport>? tryProgress) {
var fragment = new Fragment<Ordered<T>>(link.Length);
if (!Fragment<Ordered<T>>.TryAcquireUpdater(fragment, out var updater))
throw new S.AssertionException(S.M.NewFragmentShouldBeFree);
throw new AssertionException(Exceptions.Exceptions.fragment_locked);
var isFailure = false;
+1 -15
View File
@@ -6,7 +6,7 @@ using System.Text;
using System.Threading.Tasks;
namespace Beam.Dynamic {
public class AnchorCollectionDataProvider : IDataProvider<string[]>, IDataProvider<SourceLink[]> {
public class AnchorCollectionDataProvider : IDataProvider<string[]> {
public IBinding? Content { get; set; }
public string? RelativeTo { get; set; }
@@ -35,19 +35,5 @@ namespace Beam.Dynamic {
return links.Where(x => !string.IsNullOrWhiteSpace(x)).ToArray();
}
SourceLink[] IDataProvider<SourceLink[]>.Get(HtmlDocument document) {
var links = Get(document);
if (links.Length == 0)
return [];
List<SourceLink> slinks = [];
foreach (var link in links)
if (Uri.TryCreate(GetAbsolute(RelativeTo, link), UriKind.RelativeOrAbsolute, out _))
slinks.Add(new SourceLink(GetAbsolute(RelativeTo, link)));
return slinks.ToArray();
}
}
}
+1 -12
View File
@@ -6,7 +6,7 @@ using System.Text;
using System.Threading.Tasks;
namespace Beam.Dynamic {
public class AnchorDataProvider : IDataProvider<SourceLink>, IDataProvider<string> {
public class AnchorDataProvider : IDataProvider<string> {
public IBinding? Content { get; set; }
public string Get(HtmlDocument document) {
@@ -16,16 +16,5 @@ namespace Beam.Dynamic {
return Content.Select(document)?.GetAttributeValue("href", "") ?? "";
}
SourceLink IDataProvider<SourceLink>.Get(HtmlDocument document) {
var content = Get(document);
if (content is null)
return SourceLink.InvalidLink;
if (!Uri.TryCreate(content, UriKind.RelativeOrAbsolute, out _))
return SourceLink.InvalidLink;
return new SourceLink(content);
}
}
}
@@ -1,13 +1,7 @@
using aeqw89.DataKeys;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Abstractions;
namespace Beam.Temporary.Cli {
namespace Beam.Dynamic {
/// <summary>
/// <para>
/// A collection of specific useful methods and constants that facilitate the use of the application; allows other parts of the application to depend on architecture-specific arbitrary choices without compromising the Single-Responsibility principle or increasing redundant code.
+4 -4
View File
@@ -7,13 +7,13 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="aeqw89.DataKeys" Version="2.0.1" />
<PackageReference Include="aeqw89.PersistentData" Version="1.1.0" />
<PackageReference Include="aeqw89.PersistentData" Version="1.3.3" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
<PackageReference Include="Microsoft.Recognizers.Text.Number" Version="1.8.13" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Beam\Beam.csproj">
<PrivateAssets>all</PrivateAssets>
</ProjectReference>
<ProjectReference Include="..\Beam.Abstractions\Beam.Abstractions.csproj" />
<ProjectReference Include="..\Beam.Exceptions\Beam.Exceptions.csproj" />
<ProjectReference Include="..\Beam.Models\Beam.Models.csproj" />
</ItemGroup>
</Project>
@@ -3,33 +3,35 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Abstractions;
namespace Beam {
namespace Beam.Dynamic {
public static class CommonStateChangers {
public static IStateChangeBehaviour LastAsNumber => new NumberedStateChanger((x, i) => {
object last = x[^1];
object last = x.GetState()[^1];
if (!int.TryParse(last.ToString(), out var number))
throw new InvalidOperationException(S.M.StateChangeError);
x[^1] = (number + i).ToString();
throw new InvalidOperationException(Exceptions.Exceptions.state_change_error); // TODO use more specific exception
x.GetState()[^1] = (number + i).ToString();
});
public static IStateChangeBehaviour Constant => new ConstantStateChanger();
public static IStateChangeBehaviour NthAsNumber(Index n, bool keepSuffix = true)
=> new NumberedStateChanger((x, i) => {
string? nth = x[n]?.ToString();
string? nth = x.GetState()[n]?.ToString();
string[] xState = x.GetState();
if (nth is null)
throw new InvalidOperationException(S.M.StateChangeError);
throw new InvalidOperationException(Exceptions.Exceptions.state_change_error); // TODO use more specific exception
if (!int.TryParse(nth, out var number))
if (keepSuffix) {
string[] split = nth.Split('.');
if (!int.TryParse(split[0], out number))
throw new InvalidOperationException(S.M.StateChangeError);
x[n] = (number + i) + split[1..].Aggregate((x, y) => $"{x}.{y}");
throw new InvalidOperationException(Exceptions.Exceptions.state_change_error); // TODO use more specific exception
xState[n] = (number + i) + split[1..].Aggregate((x, y) => $"{x}.{y}");
return;
} else
throw new InvalidOperationException(S.M.StateChangeError);
x[n] = (number + i).ToString();
throw new InvalidOperationException(Exceptions.Exceptions.state_change_error); // TODO use more specific exception
xState[n] = (number + i).ToString();
});
+45
View File
@@ -0,0 +1,45 @@
using aeqw89.DataKeys;
using Beam.Abstractions;
using Beam.Models;
using HtmlAgilityPack;
namespace Beam.Dynamic;
public static class CommonTransformers {
public static AsyncTransformer<HtmlDocument, ArticleData> ArticleDataTransformer(DataBindings? binding) =>
(x) => {
return Task.FromResult(new ArticleData() {
Authors = binding?.Authors?.Get(x)?.Select(StringCleaner.Clean)?.ToArray() ?? [],
Name = StringCleaner.Clean(binding?.Title?.Get(x) ?? ""),
Categories = binding?.Tags?.Get(x)?.Select(StringCleaner.Clean)?.ToArray() ?? [],
Description = StringCleaner.Clean(binding?.Description?.Get(x) ?? "")
});
};
public static AsyncTransformer<HtmlDocument, TableOfContentsData>
TableOfContentsTransformer(DataBindings? binding) => (x) => {
return Task.FromResult(new TableOfContentsData() {
Authors = binding?.Authors?.Get(x)?.Select(StringCleaner.Clean)?.ToArray() ?? [],
Name = StringCleaner.Clean(binding?.Title?.Get(x) ?? ""),
Categories = binding?.Tags?.Get(x)?.Select(StringCleaner.Clean)?.ToArray() ?? [],
Description = StringCleaner.Clean(binding?.Description?.Get(x) ?? ""),
ContentLinks = binding?.TableOfContents?.Get(x) ?? [],
PagesLinks = binding?.PagesDropDown?.Get(x) ?? []
});
};
public static AsyncTransformer<HtmlDocument, StringDocument> DocumentTransformer(DataBindings? binding,
IDocumentMetaData? metaData = null) => (x) => {
var resolved = binding?.Resolve(x);
var articleData = new ArticleData() {
Name = StringCleaner.Clean(resolved?.Title),
};
Dictionary<DataKey<IDocumentMetaData>, IDocumentMetaData> meta = [];
meta.Add(IArchitecture.Default.ChapterKey, articleData);
if (metaData is not null)
meta.Add(IArchitecture.Default.BookKey, metaData);
return Task.FromResult(new StringDocument(Path.GetRandomFileName(), StringCleaner.Clean(resolved?.Content)) {
MetaData = meta
});
};
}
+9
View File
@@ -0,0 +1,9 @@
using Beam.Abstractions;
namespace Beam.Dynamic {
public class ConstantStateChanger : IStateChangeBehaviour {
public void Apply(IState state, object stimulus) {
return;
}
}
}
+183 -182
View File
@@ -1,191 +1,192 @@
using HtmlAgilityPack;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Beam.Abstractions;
using HtmlAgilityPack;
namespace Beam.Dynamic {
public record class DataBindings {
#region ---------------------- Common Bindings ----------------------
[JsonIgnore]
public IDataProvider<string>? Title {
get => Get<string>(nameof(Title));
set => Providers[nameof(Title)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Authors {
get => Get<string[]>(nameof(Authors));
set => Providers[nameof(Authors)] = value;
}
[JsonIgnore]
public IDataProvider<string>? Description {
get => Get<string>(nameof(Description));
set => Providers[nameof(Description)] = value;
}
[JsonIgnore]
public IDataProvider<string>? Content {
get => Get<string>(nameof(Content));
set => Providers[nameof(Content)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Language {
get => Get<string[]>(nameof(Language));
set => Providers[nameof(Language)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Tags {
get => Get<string[]>(nameof(Tags));
set => Providers[nameof(Tags)] = value;
}
[JsonIgnore]
public IDataProvider<string>? Publisher {
get => Get<string>(nameof(Publisher));
set => Providers[nameof(Publisher)] = value;
}
[JsonIgnore]
public IDataProvider<DateTimeOffset>? PublicationDate {
get => Get<DateTimeOffset>(nameof(PublicationDate));
set => Providers[nameof(PublicationDate)] = value;
}
[JsonIgnore]
public IDataProvider<string>? ISBN {
get => Get<string>(nameof(ISBN));
set => Providers[nameof(ISBN)] = value;
}
[JsonIgnore]
public IDataProvider<int>? PageCount {
get => Get<int>(nameof(PageCount));
set => Providers[nameof(PageCount)] = value;
}
[JsonIgnore]
public IDataProvider<SourceLink>? CoverImage {
get => Get<SourceLink>(nameof(CoverImage));
set => Providers[nameof(CoverImage)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Series {
get => Get<string[]>(nameof(Series));
set => Providers[nameof(Series)] = value;
}
[JsonIgnore]
public IDataProvider<int>? Edition {
get => Get<int>(nameof(Edition));
set => Providers[nameof(Edition)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Contributors {
get => Get<string[]>(nameof(Contributors));
set => Providers[nameof(Contributors)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Subjects {
get => Get<string[]>(nameof(Subjects));
set => Providers[nameof(Subjects)] = value;
}
[JsonIgnore]
public IDataProvider<string>? Rights {
get => Get<string>(nameof(Rights));
set => Providers[nameof(Rights)] = value;
}
[JsonIgnore]
public IDataProvider<SourceLink[]>? TableOfContents {
get => Get<SourceLink[]>(nameof(TableOfContents));
set => Providers[nameof(TableOfContents)] = value;
}
[JsonIgnore]
public IDataProvider<SourceLink[]>? PagesDropDown {
get => Get<SourceLink[]>(nameof(PagesDropDown));
set => Providers[nameof(PagesDropDown)] = value;
}
[JsonIgnore]
public IDataProvider<SourceLink>? NextPageButton {
get => Get<SourceLink>(nameof(NextPageButton));
set => Providers[nameof(NextPageButton)] = value;
}
[JsonIgnore]
public IDataProvider<SourceLink>? PreviousPageButton {
get => Get<SourceLink>(nameof(PreviousPageButton));
set => Providers[nameof(PreviousPageButton)] = value;
}
#endregion
namespace Beam.Dynamic;
public Dictionary<string, IDataProvider?> Providers { get; set; } = [];
public record class DataBindings : IDataBindings {
#region ---------------------- Common Bindings ----------------------
[JsonIgnore]
public IDataProvider<string>? Title {
get => Get<string>(nameof(Title));
set => Providers[nameof(Title)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Authors {
get => Get<string[]>(nameof(Authors));
set => Providers[nameof(Authors)] = value;
}
[JsonIgnore]
public IDataProvider<string>? Description {
get => Get<string>(nameof(Description));
set => Providers[nameof(Description)] = value;
}
[JsonIgnore]
public IDataProvider<string>? Content {
get => Get<string>(nameof(Content));
set => Providers[nameof(Content)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Language {
get => Get<string[]>(nameof(Language));
set => Providers[nameof(Language)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Tags {
get => Get<string[]>(nameof(Tags));
set => Providers[nameof(Tags)] = value;
}
[JsonIgnore]
public IDataProvider<string>? Publisher {
get => Get<string>(nameof(Publisher));
set => Providers[nameof(Publisher)] = value;
}
[JsonIgnore]
public IDataProvider<DateTimeOffset>? PublicationDate {
get => Get<DateTimeOffset>(nameof(PublicationDate));
set => Providers[nameof(PublicationDate)] = value;
}
[JsonIgnore]
public IDataProvider<string>? ISBN {
get => Get<string>(nameof(ISBN));
set => Providers[nameof(ISBN)] = value;
}
[JsonIgnore]
public IDataProvider<int>? PageCount {
get => Get<int>(nameof(PageCount));
set => Providers[nameof(PageCount)] = value;
}
[JsonIgnore]
public IDataProvider<string>? CoverImage {
get => Get<string>(nameof(CoverImage));
set => Providers[nameof(CoverImage)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Series {
get => Get<string[]>(nameof(Series));
set => Providers[nameof(Series)] = value;
}
[JsonIgnore]
public IDataProvider<int>? Edition {
get => Get<int>(nameof(Edition));
set => Providers[nameof(Edition)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Contributors {
get => Get<string[]>(nameof(Contributors));
set => Providers[nameof(Contributors)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? Subjects {
get => Get<string[]>(nameof(Subjects));
set => Providers[nameof(Subjects)] = value;
}
[JsonIgnore]
public IDataProvider<string>? Rights {
get => Get<string>(nameof(Rights));
set => Providers[nameof(Rights)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? TableOfContents {
get => Get<string[]>(nameof(TableOfContents));
set => Providers[nameof(TableOfContents)] = value;
}
[JsonIgnore]
public IDataProvider<string[]>? PagesDropDown {
get => Get<string[]>(nameof(PagesDropDown));
set => Providers[nameof(PagesDropDown)] = value;
}
[JsonIgnore]
public IDataProvider<string>? NextPageButton {
get => Get<string>(nameof(NextPageButton));
set => Providers[nameof(NextPageButton)] = value;
}
[JsonIgnore]
public IDataProvider<string>? PreviousPageButton {
get => Get<string>(nameof(PreviousPageButton));
set => Providers[nameof(PreviousPageButton)] = value;
}
#endregion
private IDataProvider<T>? Get<T>(string key) {
if (Providers.TryGetValue(key, out var k) && k is IDataProvider<T> ks)
return ks;
return default;
}
public Dictionary<string, IDataProvider?> Providers { get; set; } = [];
public virtual ResolvedBindings Resolve(HtmlDocument doc) {
// explicit fields already handled below
var mappedKeys = new HashSet<string> {
nameof(Title), nameof(Authors), nameof(Description), nameof(Content),
nameof(Language), nameof(Tags), nameof(Publisher), nameof(PublicationDate),
nameof(ISBN), nameof(PageCount), nameof(CoverImage), nameof(Series),
nameof(Edition), nameof(Contributors), nameof(Subjects), nameof(Rights),
nameof(TableOfContents), nameof(PagesDropDown), nameof(NextPageButton),
nameof(PreviousPageButton)
};
var additional = new Dictionary<string, object?>();
foreach (var (key, provider) in Providers) {
if (!mappedKeys.Contains(key) && provider is not null) {
// dynamic call so any IDataProvider<T> works
additional[key] = ((dynamic)provider).Get(doc);
}
}
return new ResolvedBindings {
Title = Title?.Get(doc),
Authors = Authors?.Get(doc) ?? [],
Description = Description?.Get(doc),
Content = Content?.Get(doc),
Language = Language?.Get(doc),
Tags = Tags?.Get(doc) ?? [],
Publisher = Publisher?.Get(doc),
PublicationDate = PublicationDate?.Get(doc),
ISBN = ISBN?.Get(doc),
PageCount = PageCount?.Get(doc),
CoverImage = CoverImage?.Get(doc),
Series = Series?.Get(doc) ?? [],
Edition = Edition?.Get(doc),
Contributors = Contributors?.Get(doc) ?? [],
Subjects = Subjects?.Get(doc) ?? [],
Rights = Rights?.Get(doc),
TableOfContents = TableOfContents?.Get(doc) ?? [],
PagesDropDown = PagesDropDown?.Get(doc),
NextPageButton = NextPageButton?.Get(doc),
PreviousPageButton = PreviousPageButton?.Get(doc),
Additional = additional
};
}
private IDataProvider<T>? Get<T>(string key) {
if (Providers.TryGetValue(key, out var k) && k is IDataProvider<T> ks)
return ks;
return default;
}
public record class ResolvedBindings {
public string? Title { get; set; }
public string[]? Authors { get; set; }
public string? Description { get; set; }
public string? Content { get; set; }
public string[]? Language { get; set; }
public string[]? Tags { get; set; }
public string? Publisher { get; set; }
public DateTimeOffset? PublicationDate { get; set; }
public string? ISBN { get; set; }
public int? PageCount { get; set; }
public SourceLink? CoverImage { get; set; }
public string[]? Series { get; set; }
public int? Edition { get; set; }
public string[]? Contributors { get; set; }
public string[]? Subjects { get; set; }
public string? Rights { get; set; }
public SourceLink[]? TableOfContents { get; set; }
public SourceLink[]? PagesDropDown { get; set; }
public SourceLink? NextPageButton { get; set; }
public SourceLink? PreviousPageButton { get; set; }
public virtual ResolvedBindings Resolve(HtmlDocument doc) {
// explicit fields already handled below
var mappedKeys = new HashSet<string> {
nameof(Title), nameof(Authors), nameof(Description), nameof(Content),
nameof(Language), nameof(Tags), nameof(Publisher), nameof(PublicationDate),
nameof(ISBN), nameof(PageCount), nameof(CoverImage), nameof(Series),
nameof(Edition), nameof(Contributors), nameof(Subjects), nameof(Rights),
nameof(TableOfContents), nameof(PagesDropDown), nameof(NextPageButton),
nameof(PreviousPageButton)
};
/// <summary>
/// Values resolved from any providers whose keys arent represented
/// by the named properties above.
/// </summary>
public Dictionary<string, object?> Additional { get; set; } = [];
var additional = new Dictionary<string, object?>();
foreach (var (key, provider) in Providers) {
if (!mappedKeys.Contains(key) && provider is not null) {
// dynamic call so any IDataProvider<T> works
additional[key] = ((dynamic)provider).Get(doc);
}
}
return new ResolvedBindings {
Title = Title?.Get(doc),
Authors = Authors?.Get(doc) ?? [],
Description = Description?.Get(doc),
Content = Content?.Get(doc),
Language = Language?.Get(doc),
Tags = Tags?.Get(doc) ?? [],
Publisher = Publisher?.Get(doc),
PublicationDate = PublicationDate?.Get(doc),
ISBN = ISBN?.Get(doc),
PageCount = PageCount?.Get(doc),
CoverImage = CoverImage?.Get(doc),
Series = Series?.Get(doc) ?? [],
Edition = Edition?.Get(doc),
Contributors = Contributors?.Get(doc) ?? [],
Subjects = Subjects?.Get(doc) ?? [],
Rights = Rights?.Get(doc),
TableOfContents = TableOfContents?.Get(doc) ?? [],
PagesDropDown = PagesDropDown?.Get(doc),
NextPageButton = NextPageButton?.Get(doc),
PreviousPageButton = PreviousPageButton?.Get(doc),
Additional = additional
};
}
}
public record class ResolvedBindings {
public string? Title { get; set; }
public string[]? Authors { get; set; }
public string? Description { get; set; }
public string? Content { get; set; }
public string[]? Language { get; set; }
public string[]? Tags { get; set; }
public string? Publisher { get; set; }
public DateTimeOffset? PublicationDate { get; set; }
public string? ISBN { get; set; }
public int? PageCount { get; set; }
public string? CoverImage { get; set; }
public string[]? Series { get; set; }
public int? Edition { get; set; }
public string[]? Contributors { get; set; }
public string[]? Subjects { get; set; }
public string? Rights { get; set; }
public string[]? TableOfContents { get; set; }
public string[]? PagesDropDown { get; set; }
public string? NextPageButton { get; set; }
public string? PreviousPageButton { get; set; }
/// <summary>
/// Values resolved from any providers whose keys arent represented
/// by the named properties above.
/// </summary>
public Dictionary<string, object?> Additional { get; set; } = [];
}
@@ -0,0 +1,37 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
namespace Beam.Dynamic;
// [JsonDerivedType(typeof(ParagraphedContentDataProvider), "paragraphed")]
// [JsonDerivedType(typeof(ListContentDataProvider), "list")]
// [JsonDerivedType(typeof(ContentsArrayDataProvider), "array")]
// [JsonDerivedType(typeof(ContentsDataProvider), "single")]
// [JsonDerivedType(typeof(DropDownDataProvider), "dropdown")]
// [JsonDerivedType(typeof(AnchorCollectionDataProvider), "anchor-list")]
// [JsonDerivedType(typeof(AnchorDataProvider), "anchor")]
public class DataProviderJsonTypeInfoResolver : DefaultJsonTypeInfoResolver {
public override JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) {
JsonTypeInfo jsonTypeInfo = base.GetTypeInfo(type, options);
Type basePointType = typeof(IDataProvider);
if (jsonTypeInfo.Type == basePointType) {
jsonTypeInfo.PolymorphismOptions = new JsonPolymorphismOptions {
UnknownDerivedTypeHandling = JsonUnknownDerivedTypeHandling.FallBackToNearestAncestor,
DerivedTypes = {
new JsonDerivedType(typeof(ParagraphedContentDataProvider), "paragraphed"),
new JsonDerivedType(typeof(ListContentDataProvider), "list"),
new JsonDerivedType(typeof(ContentsArrayDataProvider), "array"),
new JsonDerivedType(typeof(ContentsDataProvider), "single"),
new JsonDerivedType(typeof(DropDownDataProvider), "dropdown"),
new JsonDerivedType(typeof(AnchorCollectionDataProvider), "anchor-list"),
new JsonDerivedType(typeof(AnchorDataProvider), "anchor")
}
};
}
return jsonTypeInfo;
}
}
+5 -8
View File
@@ -11,8 +11,7 @@ using System.Threading.Tasks;
namespace Beam.Dynamic {
public class DropDownDataProvider
: IDataProvider<string>,
IDataProvider<string[]>,
IDataProvider<SourceLink[]> {
IDataProvider<string[]> {
public IBinding? Content { get; set; }
public string? RelativeTo { get; set; }
@@ -27,26 +26,24 @@ namespace Beam.Dynamic {
return @base + '/' + relative;
}
public SourceLink[] Get(HtmlDocument document) {
public string[] Get(HtmlDocument document) {
if (Content is null)
return [];
var node = Content.Select(document);
if (node is null)
return [];
List<SourceLink> links = [];
List<string> links = [];
foreach (var child in node.ChildNodes.Where(x => x.Name == "option")) {
var childValue = child.GetAttributeValue("value", null);
if (!Uri.TryCreate(GetAbsolute(RelativeTo, childValue), UriKind.Absolute, out _))
continue;
links.Add(new SourceLink(GetAbsolute(RelativeTo, childValue)));
links.Add((GetAbsolute(RelativeTo, childValue)));
}
return links.ToArray();
}
string[] IDataProvider<string[]>.Get(HtmlDocument document) {
return this.Get(document).Select(x => x.Link.AbsoluteUri).ToArray();
}
string IDataProvider<string>.Get(HtmlDocument document) {
return JsonSerializer.Serialize(this.Get(document));
-21
View File
@@ -1,21 +0,0 @@
using HtmlAgilityPack;
using System.Text.Json.Serialization;
namespace Beam.Dynamic {
[JsonDerivedType(typeof(ParagraphedContentDataProvider), "paragraphed")]
[JsonDerivedType(typeof(ListContentDataProvider), "list")]
[JsonDerivedType(typeof(ContentsArrayDataProvider), "array")]
[JsonDerivedType(typeof(ContentsDataProvider), "single")]
[JsonDerivedType(typeof(DropDownDataProvider), "dropdown")]
[JsonDerivedType(typeof(AnchorCollectionDataProvider), "anchor-list")]
[JsonDerivedType(typeof(AnchorDataProvider), "anchor")]
public interface IDataProvider {
public string GetString(HtmlDocument document)
=> (this as IDataProvider<object>)?.Get(document)?.ToString() ?? "";
}
public interface IDataProvider<out T> : IDataProvider {
public T Get(HtmlDocument document);
//public HtmlNode? GetNode(HtmlDocument document);
}
}
@@ -1,9 +1,7 @@
using aeqw89.DataKeys;
using Beam.Dynamic;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using Beam.Abstractions;
namespace Beam.Temporary.Cli {
namespace Beam.Dynamic {
public partial interface IArchitecture {
private class MainArchitecture : IArchitecture {
public MainArchitecture() { }
@@ -1,15 +1,17 @@
namespace Beam {
using Beam.Abstractions;
namespace Beam.Dynamic {
public class NumberedStateChanger(NumberedStateChanger.MoveState moveState) : IStateChangeBehaviour {
public delegate void MoveState(State state, int amount);
public delegate void MoveState(IState state, int amount);
public MoveState MoveStateDlgte { get; set; } = moveState;
public virtual void Apply(State state, object stimulus) {
public virtual void Apply(IState state, object stimulus) {
if (stimulus is not int amount)
throw new ArgumentException(S.M.StimulusMustBeInt, nameof(stimulus));
throw new ArgumentException(string.Format(Exceptions.Exceptions.num_state_changer_stimulus_must_be_int, stimulus.GetType().Name), nameof(stimulus));
Apply(state, amount);
}
public virtual void Apply(State state, int amount) {
public virtual void Apply(IState state, int amount) {
MoveStateDlgte(state, amount);
}
+2 -1
View File
@@ -4,9 +4,10 @@ using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using Beam.Abstractions;
namespace Beam.Dynamic {
public class StateChangerFactory {
public class StateChangerFactory : IStateChangerFactory {
[JsonIgnore]
public IStateChangeBehaviour Behavior => FactoryTable[StateChangerKey]();
@@ -10,7 +10,7 @@ using System.Threading.Tasks;
using System.Web;
namespace Beam.Dynamic {
public static partial class OnlineCleaner {
public static partial class StringCleaner {
[GeneratedRegex("&#x?[\\d\\w]{1,4};")]
public static partial Regex MochaBlendUnicodeEscapeSequence();
+14
View File
@@ -0,0 +1,14 @@
namespace Beam.Exceptions;
/// <summary>
/// The kind of exception that should never happen
/// </summary>
[Serializable]
public class AssertionException : Exception {
public AssertionException() { }
public AssertionException(string message) : base(message) { }
public AssertionException(string message, Exception inner) : base(message, inner) { }
protected AssertionException(
System.Runtime.Serialization.SerializationInfo info,
System.Runtime.Serialization.StreamingContext context) : base(info, context) { }
}
+116
View File
@@ -0,0 +1,116 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
namespace Beam.Exceptions {
using System;
/// <summary>
/// A strongly-typed resource class, for looking up localized strings, etc.
/// </summary>
// This class was auto-generated by the StronglyTypedResourceBuilder
// class via a tool like ResGen or Visual Studio.
// To add or remove a member, edit your .ResX file then rerun ResGen
// with the /str option, or rebuild your VS project.
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")]
[global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
[global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
public class Exceptions {
private static global::System.Resources.ResourceManager resourceMan;
private static global::System.Globalization.CultureInfo resourceCulture;
[global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
internal Exceptions() {
}
/// <summary>
/// Returns the cached ResourceManager instance used by this class.
/// </summary>
[global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
public static global::System.Resources.ResourceManager ResourceManager {
get {
if (object.ReferenceEquals(resourceMan, null)) {
global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Beam.Exceptions.Exceptions", typeof(Exceptions).Assembly);
resourceMan = temp;
}
return resourceMan;
}
}
/// <summary>
/// Overrides the current thread's CurrentUICulture property for all
/// resource lookups using this strongly typed resource class.
/// </summary>
[global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
public static global::System.Globalization.CultureInfo Culture {
get {
return resourceCulture;
}
set {
resourceCulture = value;
}
}
/// <summary>
/// Looks up a localized string similar to A fragment is locked when it should be free; failed to obtain updater..
/// </summary>
public static string fragment_locked {
get {
return ResourceManager.GetString("fragment_locked", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to The argument at index &apos;{0}&apos; with name &apos;{1}&apos; is not marked optional and is missing..
/// </summary>
public static string link_builder_argument_missing {
get {
return ResourceManager.GetString("link_builder_argument_missing", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to The flag &apos;{0}&apos; is incompatible with the flag(s) &apos;{1}&apos;.
/// </summary>
public static string link_builder_incompatible_flag {
get {
return ResourceManager.GetString("link_builder_incompatible_flag", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to The query flag is only allowed on the last segment; found on segment index &apos;{0}&apos;.
/// </summary>
public static string link_builder_query_only_at_last {
get {
return ResourceManager.GetString("link_builder_query_only_at_last", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to The stimulus must be an integer; got &apos;{0}&apos;.
/// </summary>
public static string num_state_changer_stimulus_must_be_int {
get {
return ResourceManager.GetString("num_state_changer_stimulus_must_be_int", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Encountered an error while changing state.
/// </summary>
public static string state_change_error {
get {
return ResourceManager.GetString("state_change_error", resourceCulture);
}
}
}
}
+39
View File
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
<xsd:element name="root" msdata:IsDataSet="true">
</xsd:element>
</xsd:schema>
<resheader name="resmimetype">
<value>text/microsoft-resx</value>
</resheader>
<resheader name="version">
<value>1.3</value>
</resheader>
<resheader name="reader">
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<data name="link_builder_argument_missing" xml:space="preserve">
<value>The argument at index '{0}' with name '{1}' is not marked optional and is missing.</value>
</data>
<data name="link_builder_incompatible_flag" xml:space="preserve">
<value>The flag '{0}' is incompatible with the flag(s) '{1}'</value>
</data>
<data name="link_builder_query_only_at_last" xml:space="preserve">
<value>The query flag is only allowed on the last segment; found on segment index '{0}'</value>
</data>
<data name="num_state_changer_stimulus_must_be_int" xml:space="preserve">
<value>The stimulus must be an integer; got '{0}'</value>
</data>
<data name="state_change_error" xml:space="preserve">
<value>Encountered an error while changing state</value>
</data>
<data name="fragment_locked" xml:space="preserve">
<value>A fragment is locked when it should be free; failed to obtain updater.</value>
</data>
</root>
+11
View File
@@ -0,0 +1,11 @@
namespace Beam.Exceptions;
[Serializable]
public class MapException : ArgumentException {
public MapException() { }
public MapException(string message) : base(message) { }
public MapException(string message, Exception inner) : base(message, inner) { }
protected MapException(
System.Runtime.Serialization.SerializationInfo info,
System.Runtime.Serialization.StreamingContext context) : base(info, context) { }
}
+2 -4
View File
@@ -6,11 +6,9 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Beam.Models\Beam.Models.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Beam\Beam.csproj">
<PrivateAssets>all</PrivateAssets>
</ProjectReference>
</ItemGroup>
</Project>
+5 -2
View File
@@ -1,8 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using Beam.Abstractions;
using Beam.Models;
namespace Beam.Exports {
public class PlainTextExporter : IExporter, IAsyncExporter {
@@ -24,14 +27,14 @@ namespace Beam.Exports {
var text = Convert();
if (!Directory.Exists(Path.GetDirectoryName(path)))
throw new ArgumentException(S.M.FileDirectoryDoesNotExist, nameof(path));
File.WriteAllText(path, text, Encoding.Unicode);
System.IO.File.WriteAllText(path, text, Encoding.Unicode);
}
public virtual async Task WriteAsync(string path) {
var text = await ConvertAsync();
if (!Directory.Exists(path))
throw new ArgumentException(S.M.FileDirectoryDoesNotExist, nameof(path));
await File.WriteAllTextAsync(path, text);
await System.IO.File.WriteAllTextAsync(path, text);
}
}
}
+2
View File
@@ -1,4 +1,6 @@
using System.Text;
using Beam.Abstractions;
using Beam.Models;
namespace Beam.Exports {
public class HtmlExporter : PlainTextExporter {
+5 -4
View File
@@ -7,16 +7,19 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="aeqw89.DataKeys" Version="2.0.1" />
<PackageReference Include="aeqw89.PersistentData" Version="1.1.0" />
<PackageReference Include="aeqw89.PersistentData" Version="1.3.3" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.7" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.7" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.7" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Beam.Data\Beam.Data.csproj" />
<ProjectReference Include="..\Beam.Downloaders\Beam.Downloaders.csproj" />
<ProjectReference Include="..\Beam.Dynamic\Beam.Dynamic.csproj">
<PrivateAssets>all</PrivateAssets>
</ProjectReference>
<ProjectReference Include="..\Beam.Exceptions\Beam.Exceptions.csproj" />
<ProjectReference Include="..\Beam.Models\Beam.Models.csproj" />
<ProjectReference Include="..\Beam.Playwright\Beam.Playwright.csproj">
<PrivateAssets>all</PrivateAssets>
@@ -24,8 +27,6 @@
<ProjectReference Include="..\Beam.Stealth\Beam.Stealth.csproj">
<PrivateAssets>all</PrivateAssets>
</ProjectReference>
<ProjectReference Include="..\Beam\Beam.csproj">
<PrivateAssets>all</PrivateAssets>
</ProjectReference>
<ProjectReference Include="..\Beam\Beam.csproj" />
</ItemGroup>
</Project>
+7 -3
View File
@@ -1,6 +1,10 @@
using HtmlAgilityPack;
using Beam.Abstractions;
using Beam.Models;
using HtmlAgilityPack;
using Beam.Playwright;
using Beam.Stealth;
using Beam;
using Beam.Downloaders;
namespace Beam.Fluent {
public static partial class DownloadBuilder<RawType, OutType> {
@@ -33,7 +37,7 @@ public static partial class DownloadBuilder<RawType, OutType> {
return this;
}
public IContextStage WithRetryReporter(IProgress<RetryReport> reporter) {
public IContextStage WithRetryReporter(IProgress<IRetryReport> reporter) {
_ctxBuilder.WithRetryReporter(reporter);
return this;
}
@@ -163,7 +167,7 @@ public static partial class DownloadBuilder<RawType, OutType> {
}
private IAsyncEnumerator<Ordered<OutType>> ConstructDownloader(DownloadContext<RawType> context) {
var copyOfContext = context.CreateBuilder().Build();
var copyOfContext = DownloadContextBuilder<RawType>.FromContext(context).Build();
return _useFragments switch {
true => new SequentialFragmentDownloader<RawType, OutType>(
copyOfContext,
@@ -1,5 +1,6 @@
using System.Collections.Concurrent;
using System.Text.Json;
using Beam.Models;
namespace Beam.Fluent {
public static partial class DownloadBuilder<RawType, OutType> {
@@ -1,7 +1,9 @@
namespace Beam.Fluent {
using Beam.Models;
namespace Beam.Fluent {
public static partial class DownloadBuilder<RawType, OutType> {
public interface IAlternativeLinkStage {
IAlternativeTransformStage WithLinks(IEnumerable<SourceLink> links);
IAlternativeTransformStage WithLinks(IEnumerable<string> links);
}
}
}
@@ -1,4 +1,6 @@
namespace Beam.Fluent {
using Beam.Models;
namespace Beam.Fluent {
public static partial class DownloadBuilder<RawType, OutType> {
public interface IAlternativeTransformStage {
IContextStage WithTransformer(AsyncTransformer<RawType, OutType> transformer);
+5 -2
View File
@@ -1,4 +1,7 @@
using Beam.Playwright;
using Beam.Abstractions;
using Beam.Downloaders;
using Beam.Models;
using Beam.Playwright;
using Beam.Stealth;
namespace Beam.Fluent {
@@ -7,7 +10,7 @@ public static partial class DownloadBuilder<RawType, OutType> {
IContextStage Configure(Action<DownloadContextBuilder<RawType>> configure);
IContextStage WithParallelism(int degree);
IContextStage WithTimeout(TimeSpan timeout);
IContextStage WithRetryReporter(IProgress<RetryReport> reporter);
IContextStage WithRetryReporter(IProgress<IRetryReport> reporter);
DownloadEnumerable<OutType> Build();
IContextStage UseFragments();
IContextStage UsePlaywright(PlaywrightAsyncManipulator manipulator);
@@ -1,4 +1,5 @@
using Beam.Dynamic;
using Beam.Models;
namespace Beam.Fluent {
public static partial class DownloadBuilder<RawType, OutType> {
+6 -5
View File
@@ -1,10 +1,11 @@
using Beam.Models;
using Beam.Data;
using Beam.Downloaders;
using Beam.Dynamic;
using Beam.Models;
namespace Beam.Fluent {
public static partial class DownloadBuilder<RawType, OutType> {
/* ──────────────────────────── Stage types ─────────────────────────── */
private sealed record LinkStage(
WebResource Source,
State Initial,
@@ -22,7 +23,7 @@ namespace Beam.Fluent {
public ITransformStage WithLinkGenerator() {
var template = Data.Templates[Source.Key];
var generator = SourceLinkEnumerable.FromGenerator(new OrderedSourceLinkGenerator(
var generator = StringEnumerable.FromGenerator(new OrderedLinkGenerator(
template.Builder,
new NumberedStateChanger(template.Factory.Behavior),
Initial, endState));
@@ -31,7 +32,7 @@ namespace Beam.Fluent {
return new TransformStage(Source, Data, CtxBuilder);
}
public IAlternativeTransformStage WithLinks(IEnumerable<SourceLink> links) {
public IAlternativeTransformStage WithLinks(IEnumerable<string> links) {
CtxBuilder.WithLinks(links);
return new TransformStage(Source, Data, CtxBuilder);
}
@@ -1,4 +1,6 @@
using Beam.Dynamic;
using Beam.Data;
using Beam.Downloaders;
using Beam.Dynamic;
using Beam.Models;
namespace Beam.Fluent {
+3 -1
View File
@@ -3,6 +3,8 @@ using Beam;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using Beam.Data;
using Beam.Downloaders;
using Beam.Models;
namespace Beam.Fluent {
@@ -22,7 +24,7 @@ namespace Beam.Fluent {
private static ILinkStage Create(DataKey<ResourceDictionary> resourceDict, BeamDataContext data, string kind) {
var (source, initial) = Resolve(resourceDict, kind, data);
var ctxBuilder = new DownloadContextBuilder<RawType>().WithLinks(Array.Empty<SourceLink>()); // placeholder, filled later.
var ctxBuilder = new DownloadContextBuilder<RawType>().WithLinks([]); // placeholder, filled later.
return new LinkStage(source, initial, data, ctxBuilder);
}
@@ -0,0 +1,3 @@
namespace Beam.Models;
public delegate Task<bool> AsyncDownloadFailurePredicate<in T>(T download);
+3
View File
@@ -0,0 +1,3 @@
namespace Beam.Models;
public delegate Task<U> AsyncTransformer<in T, U>(T elem);
+7 -2
View File
@@ -7,11 +7,11 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Beam.Dynamic\Beam.Dynamic.csproj" />
<ProjectReference Include="..\Beam\Beam.csproj" />
<ProjectReference Include="..\Beam.Abstractions\Beam.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="aeqw89.PersistentData" Version="1.3.3" />
<PackageReference Include="EntityFramework" Version="6.5.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.8">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -23,4 +23,9 @@
</PackageReference>
</ItemGroup>
<ItemGroup>
<Folder Include="Abstract\" />
<Folder Include="Closed Concrete\" />
</ItemGroup>
</Project>
-50
View File
@@ -1,50 +0,0 @@
using aeqw89.PersistentData;
using aeqw89.DataKeys;
using Beam.Dynamic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using System.Data.Entity;
namespace Beam.Models {
public class BeamDataContext : BaseDataDictionary {
public Dictionary<DataKey<Template>, Template> Templates {
get => GetOrCreateDictionary<DataKey<Template>, Template>(nameof(Templates));
set => Set(nameof(Templates), value);
}
public Dictionary<DataKey<DataBindings>, DataBindings> Bindings {
get => GetOrCreateDictionary<DataKey<DataBindings>, DataBindings>(nameof(Bindings));
set => Set(nameof(Bindings), value);
}
public Dictionary<DataKey<WebResource>, HashSet<DataKey<ResourceDictionary>>> AggregatorNovels {
get => GetOrCreateDictionary<DataKey<WebResource>, HashSet<DataKey<ResourceDictionary>>>(nameof(AggregatorNovels));
set => Set(nameof(AggregatorNovels), value);
}
public Dictionary<DataKey<WebResource>, WebResource> Resources {
get => GetOrCreateDictionary<DataKey<WebResource>, WebResource>(nameof(Resources));
set => Set(nameof(Resources), value);
}
public Dictionary<DataKey<ResourceDictionary>, ResourceDictionary> ResourceDictionaries {
get => GetOrCreateDictionary<DataKey<ResourceDictionary>, ResourceDictionary>(nameof(ResourceDictionaries));
set => Set(nameof(ResourceDictionaries), value);
}
public Dictionary<DataKey<ImmutableState>, ImmutableState> InitialStates {
get => GetOrCreateDictionary<DataKey<ImmutableState>, ImmutableState>(nameof(InitialStates));
set => Set(nameof(InitialStates), value);
}
internal Dictionary<DataKey<File>, File> Files {
get => GetOrCreateDictionary<DataKey<File>, File>(nameof(Files));
set => Set(nameof(Files), value);
}
}
}
@@ -1,6 +1,6 @@
using System.Text;
namespace Beam {
namespace Beam.Models {
public class ByteDocument(string filename, byte[] content, Encoding? encoding = null) : Document(filename, encoding) {
public byte[] Content { get; set; } = content;
+7 -4
View File
@@ -1,13 +1,16 @@
using aeqw89.DataKeys;
using System.Text;
using System.Text;
using aeqw89.DataKeys;
using Beam.Abstractions;
namespace Beam {
namespace Beam.Models {
public abstract class Document(string filename, Encoding? encoding = null) : IDocument {
public string Filename { get; set; } = filename;
public Encoding Encoding { get; set; } = encoding ?? Encoding.UTF8;
public Dictionary<DataKey<IDocumentMetaData>, IDocumentMetaData> MetaData { get; set; } = [];
IDictionary<IDataKey<IDocumentMetaData>, IDocumentMetaData> IDocument.MetaData
=> MetaData.ToDictionary(IDataKey<IDocumentMetaData> (x) => x.Key, x => x.Value);
public abstract byte[] ToBytes();
public override abstract string ToString();
public abstract override string ToString();
}
}
@@ -1,10 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Abstractions;
namespace Beam {
namespace Beam.Models {
/// <summary>
/// Holds a collection of <see cref="IDocument"/> objects in memory to facilitate lazy loading
/// </summary>
+8
View File
@@ -0,0 +1,8 @@
using Beam.Abstractions;
namespace Beam.Models {
public struct DownloadReport : IDownloadReport {
// TODO implement download report
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
namespace Beam.Models {
internal class File(string path, params string[] tags) {
public class File(string path, params string[] tags) {
public string Path { get; set; } = path;
public string[] Tags { get; set; } = tags;
}
+1 -1
View File
@@ -1,7 +1,7 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
namespace Beam {
namespace Beam.Models {
public sealed class Fragment<T>(int size) {
public int Size => FragmentBag.Count;
public int MaxSize { get; } = size;
@@ -1,11 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using System.Text.Json.Serialization;
namespace Beam {
namespace Beam.Models {
public readonly struct ImmutableState {
readonly string[] state;
-13
View File
@@ -1,13 +0,0 @@
using aeqw89.DataKeys;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Beam.Models {
internal class LinkCollection(DataKey<string> key, List<SourceLink> links) {
public DataKey<string> Key { get; set; } = key;
public List<SourceLink> Links { get; set; } = links;
}
}
+6
View File
@@ -0,0 +1,6 @@
using Beam.Abstractions;
namespace Beam.Models {
public record Ordered<T>(T Data, int Order) : IOrdered<T>;
}
@@ -0,0 +1,41 @@
using System.Text.Json;
using Beam.Abstractions;
namespace Beam.Models {
public record class ArticleData : IArticleData {
public string? Name { get; set; }
public string[] Authors { get; set; } = [];
public string? Language { get; set; }
public string[] Categories { get; set; } = [];
public string? Version { get; set; }
public string? Description { get; set; }
public string AsJson(JsonSerializerOptions? options = null) {
return JsonSerializer.Serialize(this, options);
}
public virtual bool Equals(ArticleData? other) {
if (other is null) return false;
if (ReferenceEquals(this, other)) return true;
return Name == other.Name && Authors.Equals(other.Authors) && Language == other.Language && Categories.Equals(other.Categories) && Version == other.Version && Description == other.Description;
}
public virtual bool Equals(IArticleData? other) {
if (other is null) return false;
if (ReferenceEquals(this, other)) return true;
return Name == other.Name && Authors.Equals(other.Authors) && Language == other.Language && Categories.Equals(other.Categories) && Version == other.Version && Description == other.Description;
}
public override int GetHashCode() {
unchecked {
var hashCode = (Name != null ? Name.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ Authors.GetHashCode();
hashCode = (hashCode * 397) ^ (Language != null ? Language.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ Categories.GetHashCode();
hashCode = (hashCode * 397) ^ (Version != null ? Version.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (Description != null ? Description.GetHashCode() : 0);
return hashCode;
}
}
}
}
@@ -1,5 +1,7 @@
namespace Beam {
public readonly struct RetryReport {
using Beam.Abstractions;
namespace Beam.Models {
public readonly struct RetryReport : IRetryReport {
public RetryReport(int tryNumber, string link) {
TryNumber = tryNumber;
Link = link;
+8 -7
View File
@@ -1,11 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Abstractions;
namespace Beam {
public class State(string[] state) {
namespace Beam.Models {
public class State(string[] state) : IState {
string[] state = state;
public string[] GetState() => state;
@@ -14,6 +10,11 @@ namespace Beam {
public State Copy()
=> new((string[])state.Clone());
IReadOnlyState IReadOnlyState.Copy()
=> Copy();
IState IState.Copy()
=> Copy();
public string this[Index i] {
get => state[i];
set => state[i] = value;
@@ -1,6 +1,6 @@
using System.Text;
namespace Beam {
namespace Beam.Models {
internal class StreamDocument(string filename, Stream content, Encoding? encoding = null) : Document(filename) {
public Stream Content { get; set; } = content;
public Encoding Encoding { get; set; } = encoding ?? Encoding.UTF8;
@@ -1,10 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Text;
namespace Beam {
namespace Beam.Models {
public class StringDocument(string filename, string content, Encoding? encoding = null) : Document(filename, encoding) {
public string Content { get; set; } = content;
@@ -1,18 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Models;
namespace Beam.Temporary.Cli {
namespace Beam.Models {
public record class TableOfContentsData : ArticleData {
/// <summary>
/// The link collection of the actual content
/// </summary>
public SourceLink[]? ContentLinks { get; set; }
public string[]? ContentLinks { get; set; }
/// <summary>
/// The link collection of all the Table Of Content pages for this specific resource.
/// </summary>
public SourceLink[]? PagesLinks { get; set; }
public string[]? PagesLinks { get; set; }
}
}
-15
View File
@@ -1,15 +0,0 @@
using aeqw89.DataKeys;
using Beam.Dynamic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Beam.Models {
public record class Template : IKeyed<Template> {
public required DataKey<Template> Key { get; set; }
public required StateChangerFactory Factory { get; set; }
public required SourceLinkBuilder Builder { get; set; }
}
}
-11
View File
@@ -1,11 +0,0 @@
namespace Beam.Models {
internal class Tracked<T>(T obj) {
public T TrackedObject { get; set; } = obj;
public bool IsDirty { get; set; } = true;
public Tracked<T> SetDirty() {
IsDirty = true;
return this;
}
}
}
+1 -3
View File
@@ -9,8 +9,6 @@
<PackageReference Include="Microsoft.Playwright" Version="1.52.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Beam\Beam.csproj">
<PrivateAssets>all</PrivateAssets>
</ProjectReference>
<ProjectReference Include="..\Beam.Downloaders\Beam.Downloaders.csproj" />
</ItemGroup>
</Project>
+3 -1
View File
@@ -1,4 +1,6 @@
using Microsoft.Playwright;
using Beam.Downloaders;
using Beam.Models;
using Microsoft.Playwright;
namespace Beam.Playwright {
public class PlaywrightUnitDownloader<T> : UnitDownloaderBinary<T> {
@@ -1,4 +1,6 @@
using Beam.Downloaders;
using Beam.Models;
using HtmlAgilityPack;
using Microsoft.Playwright;
+1 -3
View File
@@ -10,8 +10,6 @@
<PackageReference Include="Selenium.WebDriver" Version="4.34.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Beam\Beam.csproj">
<PrivateAssets>all</PrivateAssets>
</ProjectReference>
<ProjectReference Include="..\Beam.Downloaders\Beam.Downloaders.csproj" />
</ItemGroup>
</Project>
@@ -4,6 +4,8 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Downloaders;
using Beam.Models;
namespace Beam.Stealth {
public class StealthFragmentDownloader<T> : UnitFragmentDownloaderBinary<T> {
@@ -5,6 +5,8 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Downloaders;
using Beam.Models;
namespace Beam.Stealth {
public class StealthFragmentPageDownloader<T> : UnitFragmentDownloader<T> {
+4
View File
@@ -6,8 +6,12 @@ using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Downloaders;
using Beam.Models;
namespace Beam.Stealth {
using File = System.IO.File;
public class StealthUnitDownloader<T> : UnitDownloaderBinary<T> {
public StealthConfig Config { get; }
public StealthAsyncManipulator Manipulator { get; }
@@ -5,6 +5,8 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Downloaders;
using Beam.Models;
namespace Beam.Stealth {
public class StealthUnitPageDownloader<T> : UnitDownloader<T> {
+1 -1
View File
@@ -12,7 +12,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="aeqw89.DataKeys" Version="2.0.1" />
<PackageReference Include="aeqw89.PersistentData" Version="1.1.0" />
<PackageReference Include="aeqw89.PersistentData" Version="1.3.3" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.7" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.7" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.7" />
-46
View File
@@ -1,46 +0,0 @@
using aeqw89.DataKeys;
using Beam.Dynamic;
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Beam.Temporary.Cli {
public static class CommonTransformers {
public static AsyncTransformer<HtmlDocument, ArticleData> ArticleDataTransformer(DataBindings? binding) => (x) => {
return Task.FromResult(new ArticleData() {
Authors = binding?.Authors?.Get(x)?.Select(OnlineCleaner.Clean)?.ToArray() ?? [],
Name = OnlineCleaner.Clean(binding?.Title?.Get(x) ?? ""),
Categories = binding?.Tags?.Get(x)?.Select(OnlineCleaner.Clean)?.ToArray() ?? [],
Description = OnlineCleaner.Clean(binding?.Description?.Get(x) ?? "")
});
};
public static AsyncTransformer<HtmlDocument, TableOfContentsData> TableOfContentsTransformer(DataBindings? binding) => (x) => {
return Task.FromResult(new TableOfContentsData() {
Authors = binding?.Authors?.Get(x)?.Select(OnlineCleaner.Clean)?.ToArray() ?? [],
Name = OnlineCleaner.Clean(binding?.Title?.Get(x) ?? ""),
Categories = binding?.Tags?.Get(x)?.Select(OnlineCleaner.Clean)?.ToArray() ?? [],
Description = OnlineCleaner.Clean(binding?.Description?.Get(x) ?? ""),
ContentLinks = binding?.TableOfContents?.Get(x) ?? [],
PagesLinks = binding?.PagesDropDown?.Get(x) ?? []
});
};
public static AsyncTransformer<HtmlDocument, StringDocument> DocumentTransformer(DataBindings? binding, IDocumentMetaData? metaData = null) => (x) => {
var resolved = binding?.Resolve(x);
var articleData = new ArticleData() {
Name = OnlineCleaner.Clean(resolved?.Title),
};
Dictionary<DataKey<IDocumentMetaData>, IDocumentMetaData> meta = [];
meta.Add(IArchitecture.Default.ChapterKey, articleData);
if (metaData is not null)
meta.Add(IArchitecture.Default.BookKey, metaData);
return Task.FromResult(new StringDocument(Path.GetRandomFileName(), OnlineCleaner.Clean(resolved?.Content)) {
MetaData = meta
});
};
}
}
+109 -103
View File
@@ -1,105 +1,111 @@
using aeqw89.DataKeys;
using Beam.Dynamic;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Collections.ObjectModel;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Beam.Models;
// using aeqw89.DataKeys;
// using Beam.Dynamic;
// using System;
// using System.Collections.Generic;
// using System.Collections.Immutable;
// using System.Collections.ObjectModel;
// using System.Linq;
// using System.Text;
// using System.Threading.Tasks;
// using Beam.Data;
// using Beam.Fluent;
// using Beam.Models;
/*
* MAJOR TODO FIX THIS MESS
*/
//
// namespace Beam.Temporary.Cli {
//
// public record class ResourceDictionaryBuilder(string SiteKey) {
// private List<Func<WebResourceBuilder>> _builders;
//
//
// private record class WebResourceBuilder(string ResourceKey) {
// private Func<Template> _template;
// private Func<IReadOnlyDictionary<DataKey<DataBindings>, DataBindings>> _bindings;
// private string _name;
// private string _description;
// private Uri _domain;
//
// }
//
// private record class ResourceDictionaryRegistrar(
// string SiteKey,
// string FriendlyName,
// IEnumerable<WebResource> Resources,
// IReadOnlyDictionary<string, Template> Templates,
// IReadOnlyDictionary<string, DataBindings> Bindings) : IResourceDictionaryRegistrar {
//
// private Dictionary<string, ImmutableState> _states;
//
// public IResourceDictionaryRegistrar AddInitialState(string key, ImmutableState state) {
// _states[key] = state;
// return this;
// }
//
// public void Register(BeamDataContext sdd) {
// foreach (var resource in Resources)
// sdd.Resources.TryAdd(resource.Key, resource);
// // foreach (var template in Templates)
// // sdd.Templates.TryAdd(new DataKey<WebResource>(template.Key), template.Value);
// foreach (var binding in Bindings)
// sdd.Bindings.TryAdd(new DataKey<DataBindings>(binding.Key), binding.Value);
// foreach (var state in _states)
// sdd.InitialStates.TryAdd(new DataKey<ImmutableState>(state.Key), state.Value);
//
// sdd.ResourceDictionaries.TryAdd(new DataKey<ResourceDictionary>(SiteKey), new ResourceDictionary() {
// Key = new DataKey<ResourceDictionary>(SiteKey),
// FriendlyName = FriendlyName,
// InitialStates =
// });
// }
// }
// }
//
// public interface IResourceDictionaryRegistrar {
// public IResourceDictionaryRegistrar AddInitialState(string key, ImmutableState state);
// public void Register(BeamDataContext sdd);
// }
//
// public interface IBindingsBuilder {
// public IBindingsBuilder AddBinding(DataBindings bindings);
// public IBindingsBuilder AddBinding(Action<DataBindings> configure);
// public IReadOnlyDictionary<DataKey<DataBindings>, DataBindings> Build();
// }
//
// public interface IResourceDictionaryBuilder {
// public IResourceDictionaryBuilder AddResource(Func<ITemplateBuilderStage, IWebResourceBuilderStage> configure);
// public IResourceDictionaryBuilder WithResources(Func<ITemplateBuilderStage, IWebResourceBuilderStage>[] configure);
// public IResourceDictionaryBuilder WithFriendlyName(string friendlyName);
// public IResourceDictionaryRegistrar Then();
// }
//
// public interface IWebResourceBuilderStage {
// public IWebResourceBuilderStage WithName(string name); // Stage 3
// public IWebResourceBuilderStage WithDescription(string description); // Stage 3
// public IWebResourceBuilderStage WithDomain(Uri domain); // Stage 3
// public WebResource Build();
// }
//
// public interface IBindingBuilderStage {
// public IWebResourceBuilderStage WithBindings(Action<IBindingsBuilder> configure);
// public IWebResourceBuilderStage WithBindings(IReadOnlyDictionary<DataKey<DataBindings>, DataBindings> bindings);
// }
//
// public interface ITemplateBuilderStage {
// public IBindingBuilderStage WithTemplate(Action<ITemplateBuilder> configure);
// public IBindingBuilderStage WithTemplate(Template template);
// }
//
// public interface ITemplateBuilder {
// public ITemplateBuilder WithFactory(StateChangerFactory factory);
// public ITemplateBuilder WithUrlBuilder(LinkBuilder builder);
// public ITemplateBuilder WithUrlBuilder(Action<LinkBuilder> configure);
// public Template Build();
// }
//
namespace Beam.Temporary.Cli {
public record class ResourceDictionaryBuilder(string SiteKey) {
private List<Func<WebResourceBuilder>> _builders;
private record class WebResourceBuilder(string ResourceKey) {
private Func<Template> _template;
private Func<IReadOnlyDictionary<DataKey<DataBindings>, DataBindings>> _bindings;
private string _name;
private string _description;
private Uri _domain;
}
private record class ResourceDictionaryRegistrar(
string SiteKey,
string FriendlyName,
IEnumerable<WebResource> Resources,
IReadOnlyDictionary<string, Template> Templates,
IReadOnlyDictionary<string, DataBindings> Bindings) : IResourceDictionaryRegistrar {
private Dictionary<string, ImmutableState> _states;
public IResourceDictionaryRegistrar AddInitialState(string key, ImmutableState state) {
_states[key] = state;
return this;
}
public void Register(BeamDataContext sdd) {
foreach (var resource in Resources)
sdd.Resources.TryAdd(resource.Key, resource);
foreach (var template in Templates)
sdd.Templates.TryAdd(new DataKey<WebResource>(template.Key), template.Value);
foreach (var binding in Bindings)
sdd.Bindings.TryAdd(new DataKey<DataBindings>(binding.Key), binding.Value);
foreach (var state in _states)
sdd.InitialStates.TryAdd(new DataKey<ImmutableState>(state.Key), state.Value);
sdd.ResourceDictionaries.TryAdd(new DataKey<ResourceDictionary>(SiteKey), new ResourceDictionary() {
Key = new DataKey<ResourceDictionary>(SiteKey),
FriendlyName = FriendlyName,
InitialStates =
});
}
}
}
public interface IResourceDictionaryRegistrar {
public IResourceDictionaryRegistrar AddInitialState(string key, ImmutableState state);
public void Register(BeamDataContext sdd);
}
public interface IBindingsBuilder {
public IBindingsBuilder AddBinding(DataBindings bindings);
public IBindingsBuilder AddBinding(Action<DataBindings> configure);
public IReadOnlyDictionary<DataKey<DataBindings>, DataBindings> Build();
}
public interface IResourceDictionaryBuilder {
public IResourceDictionaryBuilder AddResource(Func<ITemplateBuilderStage, IWebResourceBuilderStage> configure);
public IResourceDictionaryBuilder WithResources(Func<ITemplateBuilderStage, IWebResourceBuilderStage>[] configure);
public IResourceDictionaryBuilder WithFriendlyName(string friendlyName);
public IResourceDictionaryRegistrar Then();
}
public interface IWebResourceBuilderStage {
public IWebResourceBuilderStage WithName(string name); // Stage 3
public IWebResourceBuilderStage WithDescription(string description); // Stage 3
public IWebResourceBuilderStage WithDomain(Uri domain); // Stage 3
public WebResource Build();
}
public interface IBindingBuilderStage {
public IWebResourceBuilderStage WithBindings(Action<IBindingsBuilder> configure);
public IWebResourceBuilderStage WithBindings(IReadOnlyDictionary<DataKey<DataBindings>, DataBindings> bindings);
}
public interface ITemplateBuilderStage {
public IBindingBuilderStage WithTemplate(Action<ITemplateBuilder> configure);
public IBindingBuilderStage WithTemplate(Template template);
}
public interface ITemplateBuilder {
public ITemplateBuilder WithFactory(StateChangerFactory factory);
public ITemplateBuilder WithUrlBuilder(SourceLinkBuilder builder);
public ITemplateBuilder WithUrlBuilder(Action<SourceLinkBuilder> configure);
public Template Build();
}
}
// }
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More