refactor: unify binding & data provider interfaces
- Removed BindingType enum and all related logic from Binding. - Made Binding implement new IBinding and IKeyed interfaces. - Moved node selection logic to IBinding.Select; removed Resolve* methods from Binding. - Added new IBinding interface for XPath/CssPath selection. - Refactored IDataProvider to generic IDataProvider<T>; removed GetNode. - Updated ListContentDataProvider and ParagraphedContentDataProvider to use IBinding. - Added new ContentsDataProvider, ContentsArrayDataProvider, and DropDownDataProvider for flexible data extraction. - Updated DataBindings to use IDataProvider<T> properties instead of Binding. - Updated all usages to new interfaces and patterns.
This commit is contained in:
+2
-52
@@ -1,69 +1,19 @@
|
|||||||
|
|
||||||
using aeqw89.DataKeys;
|
using aeqw89.DataKeys;
|
||||||
using HtmlAgilityPack;
|
using HtmlAgilityPack;
|
||||||
|
using System.Text.Json;
|
||||||
using System.Text.Json.Serialization;
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
namespace Beam.Dynamic {
|
namespace Beam.Dynamic {
|
||||||
public class Binding(DataKey<Binding> key) : IKeyed<Binding> {
|
public class Binding(DataKey<Binding> key) : IBinding, IKeyed<Binding> {
|
||||||
public Binding(string key) : this(new DataKey<Binding>(key)) { }
|
public Binding(string key) : this(new DataKey<Binding>(key)) { }
|
||||||
public Binding() : this("") { }
|
public Binding() : this("") { }
|
||||||
|
|
||||||
[JsonRequired]
|
[JsonRequired]
|
||||||
public DataKey<Binding> Key { get; set; } = key;
|
public DataKey<Binding> Key { get; set; } = key;
|
||||||
[JsonRequired]
|
|
||||||
public BindingType Type { get; set; }
|
|
||||||
|
|
||||||
public string? ArrayDelimiters { get; set; }
|
|
||||||
public string? XPath { get; set; }
|
public string? XPath { get; set; }
|
||||||
public string? CssPath { get; set; }
|
public string? CssPath { get; set; }
|
||||||
public string? Text { get; set; }
|
public string? Text { get; set; }
|
||||||
private IDataProvider? Provider_;
|
|
||||||
public IDataProvider? Provider {
|
|
||||||
get => Provider_;
|
|
||||||
set {
|
|
||||||
if (value is null)
|
|
||||||
return;
|
|
||||||
if (value is not IDataProvider)
|
|
||||||
throw new InvalidOperationException();
|
|
||||||
var constructor = value.GetType().GetConstructor([]);
|
|
||||||
if (!constructor?.IsPublic ?? true)
|
|
||||||
throw new InvalidOperationException();
|
|
||||||
Provider_ = value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public HtmlNode? ResolveNode(HtmlDocument doc) {
|
|
||||||
if (XPath is not null)
|
|
||||||
return doc.DocumentNode.SelectSingleNode(XPath);
|
|
||||||
if (CssPath is not null)
|
|
||||||
return doc.DocumentNode.ThenByClasses(CssPath.Split('/'));
|
|
||||||
if (Provider is not null)
|
|
||||||
return Provider.GetNode(doc);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string ResolveString(HtmlDocument doc) {
|
|
||||||
if (XPath is not null)
|
|
||||||
return doc.DocumentNode.SelectSingleNode(XPath)?.InnerText ?? "";
|
|
||||||
if (CssPath is not null)
|
|
||||||
return doc.DocumentNode.ThenByClasses(CssPath.Split('/'))?.InnerText ?? "";
|
|
||||||
if (Provider is not null)
|
|
||||||
return Provider.Get(doc);
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
|
|
||||||
public string[] ResolveArray(HtmlDocument doc) {
|
|
||||||
if (Type is not BindingType.Array)
|
|
||||||
return [];
|
|
||||||
var str = ResolveString(doc);
|
|
||||||
return str.Split(ArrayDelimiters);
|
|
||||||
}
|
|
||||||
|
|
||||||
public dynamic? Resolve(HtmlDocument doc) => Type switch {
|
|
||||||
BindingType.Single => ResolveString(doc),
|
|
||||||
BindingType.Array => ResolveArray(doc),
|
|
||||||
BindingType.UseProvider => Provider?.Get(doc),
|
|
||||||
_ => null
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
namespace Beam.Dynamic {
|
|
||||||
public enum BindingType {
|
|
||||||
Single,
|
|
||||||
Array,
|
|
||||||
UseProvider
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
using HtmlAgilityPack;
|
||||||
|
|
||||||
|
namespace Beam.Dynamic {
|
||||||
|
public class ContentsArrayDataProvider : ContentsDataProvider, IDataProvider<string[]> {
|
||||||
|
public string[] ArrayDelimiters { get; set; } = [";"];
|
||||||
|
|
||||||
|
string[] IDataProvider<string[]>.Get(HtmlDocument document) {
|
||||||
|
if (Content is null)
|
||||||
|
return [];
|
||||||
|
|
||||||
|
return Content.Select(document)?.InnerText?.Split(ArrayDelimiters, StringSplitOptions.RemoveEmptyEntries) ?? [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
using HtmlAgilityPack;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace Beam.Dynamic {
|
||||||
|
public class ContentsDataProvider : IDataProvider<string> {
|
||||||
|
public IBinding? Content { get; set; }
|
||||||
|
|
||||||
|
public string Get(HtmlDocument document) {
|
||||||
|
if (Content is null)
|
||||||
|
return "";
|
||||||
|
|
||||||
|
return Content.Select(document)?.InnerText ?? "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,21 +2,21 @@
|
|||||||
|
|
||||||
namespace Beam.Dynamic {
|
namespace Beam.Dynamic {
|
||||||
public record class DataBindings {
|
public record class DataBindings {
|
||||||
public Binding? Title { get; set; }
|
public IDataProvider<string>? Title { get; set; }
|
||||||
public Binding? Authors { get; set; }
|
public IDataProvider<string[]>? Authors { get; set; }
|
||||||
public Binding? Description { get; set; }
|
public IDataProvider<string>? Description { get; set; }
|
||||||
public Binding? Content { get; set; }
|
public IDataProvider<string>? Content { get; set; }
|
||||||
public Binding? Language { get; set; }
|
public IDataProvider<string[]>? Language { get; set; }
|
||||||
public Binding? Tags { get; set; }
|
public IDataProvider<string[]>? Tags { get; set; }
|
||||||
|
|
||||||
public virtual ResolvedBindings Resolve(HtmlDocument doc) {
|
public virtual ResolvedBindings Resolve(HtmlDocument doc) {
|
||||||
return new ResolvedBindings() {
|
return new ResolvedBindings() {
|
||||||
Title = Title?.Resolve(doc),
|
Title = Title?.Get(doc),
|
||||||
Authors = Authors?.Resolve(doc) ?? Array.Empty<string>(),
|
Authors = Authors?.Get(doc) ?? [],
|
||||||
Language = Language?.Resolve(doc) ?? Array.Empty<string>(),
|
Language = Language?.Get(doc),
|
||||||
Content = Content?.Resolve(doc),
|
Content = Content?.Get(doc),
|
||||||
Description = Description?.Resolve(doc),
|
Description = Description?.Get(doc),
|
||||||
Tags = Tags?.Resolve(doc) ?? Array.Empty<string>()
|
Tags = Tags?.Get(doc) ?? []
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
using HtmlAgilityPack;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace Beam.Dynamic {
|
||||||
|
public class DropDownDataProvider
|
||||||
|
: IDataProvider<string>,
|
||||||
|
IDataProvider<string[]>,
|
||||||
|
IDataProvider<SourceLink[]> {
|
||||||
|
public IBinding? Content { get; set; }
|
||||||
|
|
||||||
|
public SourceLink[] Get(HtmlDocument document) {
|
||||||
|
if (Content is null)
|
||||||
|
return [];
|
||||||
|
var node = Content.Select(document);
|
||||||
|
if (node is null)
|
||||||
|
return [];
|
||||||
|
List<SourceLink> links = [];
|
||||||
|
foreach (var child in node.ChildNodes.Where(x => x.Name == "option")) {
|
||||||
|
var childValue = child.GetAttributeValue("value", null);
|
||||||
|
if (!Uri.TryCreate(childValue, UriKind.Absolute, out _))
|
||||||
|
continue;
|
||||||
|
links.Add(new SourceLink(childValue));
|
||||||
|
}
|
||||||
|
|
||||||
|
return links.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
string[] IDataProvider<string[]>.Get(HtmlDocument document) {
|
||||||
|
return this.Get(document).Select(x => x.Link.AbsoluteUri).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
string IDataProvider<string>.Get(HtmlDocument document) {
|
||||||
|
return JsonSerializer.Serialize(this.Get(document));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
using HtmlAgilityPack;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Reflection.Metadata;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace Beam.Dynamic {
|
||||||
|
[JsonDerivedType(typeof(Binding), "binding")]
|
||||||
|
public interface IBinding {
|
||||||
|
string? XPath { get; set; }
|
||||||
|
string? CssPath { get; set; }
|
||||||
|
|
||||||
|
HtmlNode? Select(HtmlDocument doc) {
|
||||||
|
if (XPath is not null)
|
||||||
|
return doc.DocumentNode.SelectSingleNode(XPath);
|
||||||
|
if (CssPath is not null)
|
||||||
|
return doc.DocumentNode.ThenByClasses(CssPath.Split('/'));
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,10 +1,14 @@
|
|||||||
using HtmlAgilityPack;
|
using HtmlAgilityPack;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
namespace Beam.Dynamic {
|
namespace Beam.Dynamic {
|
||||||
[System.Text.Json.Serialization.JsonDerivedType(typeof(ParagraphedContentDataProvider), "paragraphed-data-provider")]
|
[JsonDerivedType(typeof(ParagraphedContentDataProvider), "paragraphed")]
|
||||||
[System.Text.Json.Serialization.JsonDerivedType(typeof(ListContentDataProvider), "list-data-provider")]
|
[JsonDerivedType(typeof(ListContentDataProvider), "list")]
|
||||||
public interface IDataProvider {
|
[JsonDerivedType(typeof(ContentsArrayDataProvider), "array")]
|
||||||
public string Get(HtmlDocument document);
|
[JsonDerivedType(typeof(ContentsDataProvider), "single")]
|
||||||
public HtmlNode? GetNode(HtmlDocument document);
|
[JsonDerivedType(typeof(DropDownDataProvider), "dropdown")]
|
||||||
|
public interface IDataProvider<T> {
|
||||||
|
public T Get(HtmlDocument document);
|
||||||
|
//public HtmlNode? GetNode(HtmlDocument document);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2,14 +2,14 @@
|
|||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
namespace Beam.Dynamic {
|
namespace Beam.Dynamic {
|
||||||
public class ListContentDataProvider : IDataProvider {
|
public class ListContentDataProvider : IDataProvider<string> {
|
||||||
public Binding? Content { get; set; }
|
public IBinding? Content { get; set; }
|
||||||
|
|
||||||
public string Get(HtmlDocument document) {
|
public string Get(HtmlDocument document) {
|
||||||
if (Content is null)
|
if (Content is null)
|
||||||
return "";
|
return "";
|
||||||
|
|
||||||
var node = Content.ResolveNode(document);
|
var node = Content.Select(document);
|
||||||
if (node is null)
|
if (node is null)
|
||||||
return "";
|
return "";
|
||||||
|
|
||||||
@@ -23,9 +23,5 @@ namespace Beam.Dynamic {
|
|||||||
content.Append(node.ChildNodes.Last().InnerText.Trim());
|
content.Append(node.ChildNodes.Last().InnerText.Trim());
|
||||||
return content.ToString();
|
return content.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
public HtmlNode? GetNode(HtmlDocument document) {
|
|
||||||
return Content?.ResolveNode(document);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,14 +6,14 @@ using System.Text;
|
|||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
namespace Beam.Dynamic {
|
namespace Beam.Dynamic {
|
||||||
public class ParagraphedContentDataProvider : IDataProvider {
|
public class ParagraphedContentDataProvider : IDataProvider<string> {
|
||||||
public Binding? Content { get; set; }
|
public IBinding? Content { get; set; }
|
||||||
|
|
||||||
public string Get(HtmlDocument document) {
|
public string Get(HtmlDocument document) {
|
||||||
if (Content is null)
|
if (Content is null)
|
||||||
return "";
|
return "";
|
||||||
|
|
||||||
var node = Content.ResolveNode(document);
|
var node = Content.Select(document);
|
||||||
if (node is null)
|
if (node is null)
|
||||||
return "";
|
return "";
|
||||||
|
|
||||||
@@ -26,10 +26,5 @@ namespace Beam.Dynamic {
|
|||||||
|
|
||||||
return content.ToString();
|
return content.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
public HtmlNode? GetNode(HtmlDocument document) {
|
|
||||||
return Content?.ResolveNode(document);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,10 +11,10 @@ namespace Beam.Temporary.Cli {
|
|||||||
public static class CommonTransformers {
|
public static class CommonTransformers {
|
||||||
public static AsyncTransformer<HtmlDocument, IDocumentMetaData> ArticleDataTransformer(DataBindings? binding) => (x) => {
|
public static AsyncTransformer<HtmlDocument, IDocumentMetaData> ArticleDataTransformer(DataBindings? binding) => (x) => {
|
||||||
return Task.FromResult<IDocumentMetaData>(new ArticleData() {
|
return Task.FromResult<IDocumentMetaData>(new ArticleData() {
|
||||||
Authors = [OnlineCleaner.Clean(binding?.Authors?.Resolve(x) ?? "")],
|
Authors = binding?.Authors?.Get(x)?.Select(OnlineCleaner.Clean)?.ToArray() ?? [],
|
||||||
Name = OnlineCleaner.Clean(binding?.Title?.ResolveString(x) ?? ""),
|
Name = OnlineCleaner.Clean(binding?.Title?.Get(x) ?? ""),
|
||||||
Categories = OnlineCleaner.Clean(binding?.Tags?.ResolveString(x) ?? "").Split(';') ?? [],
|
Categories = binding?.Tags?.Get(x)?.Select(OnlineCleaner.Clean)?.ToArray() ?? [],
|
||||||
Description = OnlineCleaner.Clean(binding?.Description?.ResolveString(x) ?? "")
|
Description = OnlineCleaner.Clean(binding?.Description?.Get(x) ?? "")
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -103,36 +103,34 @@ namespace Beam.Temporary.Cli {
|
|||||||
var binding_aux = new DataKey<DataBindings>("aeqw89:bindings:wodushu_aux");
|
var binding_aux = new DataKey<DataBindings>("aeqw89:bindings:wodushu_aux");
|
||||||
|
|
||||||
sdd.Bindings.Add(binding_agg, new() {
|
sdd.Bindings.Add(binding_agg, new() {
|
||||||
Title = new Binding() {
|
Title = new ContentsDataProvider() {
|
||||||
XPath = "/html/body/div[4]/div/div/div[2]/h1",
|
Content = new Binding() {
|
||||||
Type = BindingType.Single
|
XPath = "/html/body/div[4]/div/div/div[2]/h1"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
Content = new Binding() {
|
|
||||||
Type = BindingType.UseProvider,
|
Content = new ParagraphedContentDataProvider() {
|
||||||
Provider = new ParagraphedContentDataProvider() {
|
Content = new Binding() {
|
||||||
Content = new Binding() {
|
XPath = "//*[@id=\"content\"]"
|
||||||
XPath = "//*[@id=\"content\"]"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
sdd.Bindings.Add(binding_aux, new() {
|
sdd.Bindings.Add(binding_aux, new() {
|
||||||
Title = new Binding() {
|
Title = new ContentsDataProvider() {
|
||||||
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[1]/h1",
|
Content = new Binding() {
|
||||||
Type = BindingType.Single
|
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[1]/h1"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
Authors = new Binding() {
|
Authors = new ContentsArrayDataProvider() {
|
||||||
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[1]/div/p[1]/a",
|
Content = new Binding() {
|
||||||
Type = BindingType.Single
|
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[1]/div/p[1]/a"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
Description = new Binding() {
|
Description = new ParagraphedContentDataProvider() {
|
||||||
Provider = new ParagraphedContentDataProvider() {
|
Content = new Binding() {
|
||||||
Content = new Binding() {
|
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[2]"
|
||||||
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[2]"
|
}
|
||||||
}
|
|
||||||
},
|
|
||||||
Type = BindingType.UseProvider
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net;
|
||||||
|
using System.Net.Http.Json;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace Beam {
|
||||||
|
public class ApiCall(HttpClient client, string uri, HttpMethod method, KeyValuePair<string, string[]>[] headers, object? requestData, object? body, params HashSet<HttpStatusCode> successCodes) {
|
||||||
|
public HttpClient Client { get; } = client;
|
||||||
|
public object? RequestData { get; } = requestData;
|
||||||
|
public object? Body { get; }
|
||||||
|
public string Uri { get; } = uri;
|
||||||
|
public HttpMethod Method { get; } = method;
|
||||||
|
public KeyValuePair<string, string[]>[] Headers { get; } = headers;
|
||||||
|
public HashSet<HttpStatusCode> SuccessCodes { get; } = successCodes;
|
||||||
|
|
||||||
|
public async Task<ApiResponse> GetResponse(ILogger<ApiResponse>? logger, (int @try, int max)? tries = null, CancellationToken ct = default) {
|
||||||
|
logger?.LogInformation("Fetching '{}' with method '{}'", Uri, Method);
|
||||||
|
var request = new HttpRequestMessage(Method, Uri);
|
||||||
|
request.Content = body is null ? request.Content : JsonContent.Create(body);
|
||||||
|
foreach (var header in Headers)
|
||||||
|
request.Headers.Add(header.Key, header.Value);
|
||||||
|
var response = await Client.SendAsync(request, ct);
|
||||||
|
|
||||||
|
if (tries is not null && tries?.@try < tries?.max && !SuccessCodes.Contains(response.StatusCode)) {
|
||||||
|
await Task.Delay((int)Math.Min(Math.Pow(2, tries.Value.@try), 60) * 1000, ct);
|
||||||
|
return await GetResponse(logger, (tries.Value.@try + 1, tries.Value.max), ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
return await ApiResponse.CreateAsync(response, logger, RequestData, ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async Task<ApiResponse> Get(HttpClient client, string url, ILoggerFactory factory)
|
||||||
|
=> await new ApiCall(client, url, HttpMethod.Get, [], null, null).GetResponse(factory.CreateLogger<ApiResponse>());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net;
|
||||||
|
using System.Reflection.PortableExecutable;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace Beam {
|
||||||
|
public class ApiCallBuilder(HttpClient client) {
|
||||||
|
HttpClient Client = client;
|
||||||
|
string Uri;
|
||||||
|
object? Data;
|
||||||
|
object? Body;
|
||||||
|
HttpMethod Method = HttpMethod.Get;
|
||||||
|
List<KeyValuePair<string, List<string>>> Headers = [];
|
||||||
|
HashSet<HttpStatusCode> SuccessCodes = [HttpStatusCode.OK];
|
||||||
|
|
||||||
|
public ApiCallBuilder WithUri(string uri) {
|
||||||
|
Uri = uri;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCallBuilder WithRequestData(object? data) {
|
||||||
|
Data = data;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCallBuilder WithBody(object? data) {
|
||||||
|
Body = data;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCallBuilder WithMethod(HttpMethod method) {
|
||||||
|
Method = method;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCallBuilder WithHeaders(IEnumerable<(string Key, IEnumerable<string> Value)> headers) {
|
||||||
|
Headers = headers.Select((x) => new KeyValuePair<string, List<string>>(x.Key, x.Value.ToList())).ToList();
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCallBuilder WithSuccessCodes(params HashSet<HttpStatusCode> codes) {
|
||||||
|
SuccessCodes = codes;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCallBuilder WithSuccessCodes(params IEnumerable<int> codes) {
|
||||||
|
SuccessCodes = codes.Cast<HttpStatusCode>().ToHashSet();
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCallBuilder AddHeader(string key, string value) {
|
||||||
|
if (Headers.Any((x) => x.Key == key))
|
||||||
|
Headers.FirstOrDefault((x) => x.Key == key).Value.Add(value);
|
||||||
|
else
|
||||||
|
Headers.Add(new KeyValuePair<string, List<string>>(key, [value]));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCall Build() {
|
||||||
|
if (Uri is null)
|
||||||
|
throw new InvalidOperationException();
|
||||||
|
if (Method is null)
|
||||||
|
throw new InvalidOperationException();
|
||||||
|
if (Headers is null)
|
||||||
|
Headers = [];
|
||||||
|
|
||||||
|
return new ApiCall(Client, Uri, Method, Headers.Select((x) => new KeyValuePair<string, string[]>(x.Key, x.Value.ToArray())).ToArray(), Data, Body, SuccessCodes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
// ApiCalls.cs
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Net;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace Beam {
|
||||||
|
/// <summary>
|
||||||
|
/// Executes a batch of <see cref="ApiCall"/>s using either sequential or parallel strategy.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ApiCalls {
|
||||||
|
private readonly IReadOnlyList<ApiCall> _calls;
|
||||||
|
private readonly int _maxDegree;
|
||||||
|
|
||||||
|
internal ApiCalls(IReadOnlyList<ApiCall> calls, int? maxDegree) {
|
||||||
|
_calls = calls ?? throw new ArgumentNullException(nameof(calls));
|
||||||
|
_maxDegree = Math.Max(1, maxDegree ?? 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Runs every call and returns the ordered list of <see cref="ApiResponse"/>s.
|
||||||
|
/// </summary>
|
||||||
|
public async Task<IReadOnlyList<ApiResponse>> ExecuteAsync(
|
||||||
|
ILogger<ApiResponse>? logger = null,
|
||||||
|
(int @try, int max)? tries = null,
|
||||||
|
CancellationToken ct = default) {
|
||||||
|
if (_maxDegree == 1) {
|
||||||
|
// sequential
|
||||||
|
var sequential = new List<ApiResponse>(_calls.Count);
|
||||||
|
foreach (var call in _calls)
|
||||||
|
sequential.Add(await call.GetResponse(logger, tries, ct));
|
||||||
|
return sequential;
|
||||||
|
}
|
||||||
|
|
||||||
|
// parallel
|
||||||
|
var bag = new ConcurrentBag<(int idx, ApiResponse res)>();
|
||||||
|
await Parallel.ForEachAsync(
|
||||||
|
_calls.Select((c, i) => (call: c, idx: i)),
|
||||||
|
new ParallelOptions { MaxDegreeOfParallelism = _maxDegree, CancellationToken = ct },
|
||||||
|
async (item, token) => {
|
||||||
|
var response = await item.call.GetResponse(logger, tries, token);
|
||||||
|
bag.Add((item.idx, response));
|
||||||
|
});
|
||||||
|
|
||||||
|
// keep original ordering
|
||||||
|
return bag.OrderBy(x => x.idx).Select(x => x.res).ToList();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
// ApiCallsBuilder.cs
|
||||||
|
using System.Net;
|
||||||
|
|
||||||
|
namespace Beam {
|
||||||
|
/// <summary>
|
||||||
|
/// Fluent builder for <see cref="ApiCalls"/>.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ApiCallsBuilder {
|
||||||
|
private readonly List<ApiCall> _calls = [];
|
||||||
|
private int? _parallelism = 1; // default = sequential
|
||||||
|
|
||||||
|
public ApiCallsBuilder Add(ApiCall call) {
|
||||||
|
_calls.Add(call ?? throw new ArgumentNullException(nameof(call)));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCallsBuilder AddRange(IEnumerable<ApiCall> calls) {
|
||||||
|
_calls.AddRange(calls ?? throw new ArgumentNullException(nameof(calls)));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Adds the same <paramref name="prototype"/> call <paramref name="times"/> times.</summary>
|
||||||
|
public ApiCallsBuilder Repeat(ApiCall prototype, int times) {
|
||||||
|
if (times < 1) throw new ArgumentOutOfRangeException(nameof(times));
|
||||||
|
for (var i = 0; i < times; i++)
|
||||||
|
Add(prototype);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Run with the specified degree of parallelism (≥ 1).</summary>
|
||||||
|
public ApiCallsBuilder UseParallel(int maxDegree) => SetDegree(Math.Max(1, maxDegree));
|
||||||
|
|
||||||
|
/// <summary>Run sequentially (same as <c>UseParallel(1)</c>).</summary>
|
||||||
|
public ApiCallsBuilder UseSequential() => SetDegree(1);
|
||||||
|
|
||||||
|
private ApiCallsBuilder SetDegree(int degree) {
|
||||||
|
_parallelism = degree;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApiCalls Build() {
|
||||||
|
if (_calls.Count == 0)
|
||||||
|
throw new InvalidOperationException("At least one ApiCall is required.");
|
||||||
|
return new ApiCalls(_calls, _parallelism);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using System.Net;
|
||||||
|
using System.Net.Http.Json;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
namespace Beam {
|
||||||
|
/// <summary>
|
||||||
|
/// Wrapper that lets the response body be read any number of times (even concurrently).
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ApiResponse {
|
||||||
|
private readonly byte[] _buffer;
|
||||||
|
|
||||||
|
private ApiResponse(HttpResponseMessage response, byte[] buffer, ILogger<ApiResponse>? logger, object? requestData = null) {
|
||||||
|
Response = response;
|
||||||
|
_buffer = buffer;
|
||||||
|
Logger = logger;
|
||||||
|
RequestData = requestData;
|
||||||
|
}
|
||||||
|
|
||||||
|
public HttpResponseMessage Response { get; }
|
||||||
|
public object? RequestData { get; }
|
||||||
|
public ILogger<ApiResponse>? Logger { get; }
|
||||||
|
|
||||||
|
/* ---------- creation ---------- */
|
||||||
|
|
||||||
|
public static async Task<ApiResponse> CreateAsync(
|
||||||
|
HttpResponseMessage response,
|
||||||
|
ILogger<ApiResponse>? logger = null,
|
||||||
|
object? requestData = null,
|
||||||
|
CancellationToken ct = default) {
|
||||||
|
if (response is null) throw new ArgumentNullException(nameof(response));
|
||||||
|
|
||||||
|
var buffer = response.Content is null
|
||||||
|
? []
|
||||||
|
: await response.Content.ReadAsByteArrayAsync(ct).ConfigureAwait(false);
|
||||||
|
|
||||||
|
return new ApiResponse(response, buffer, logger, requestData);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---------- status helpers ---------- */
|
||||||
|
|
||||||
|
public bool Is404 => Response.StatusCode == HttpStatusCode.NotFound;
|
||||||
|
public bool Is403 => Response.StatusCode == HttpStatusCode.Forbidden;
|
||||||
|
public bool Is500 => Response.StatusCode == HttpStatusCode.InternalServerError;
|
||||||
|
public bool Is400 => Response.StatusCode == HttpStatusCode.BadRequest;
|
||||||
|
public bool Is200 => Response.IsSuccessStatusCode;
|
||||||
|
|
||||||
|
public ApiResponse OnError(Action<HttpStatusCode> errorHandler) {
|
||||||
|
if (!Is200) errorHandler(Response.StatusCode);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---------- content helpers ---------- */
|
||||||
|
|
||||||
|
public Task<T?> AsSerializedObject<T>(CancellationToken ct = default) {
|
||||||
|
if (!Is200) throw new InvalidOperationException();
|
||||||
|
if (Response.Content?.Headers.ContentType?.MediaType != "application/json")
|
||||||
|
Logger?.LogWarning("Content-Type is not JSON, yet JSON deserialization was requested.");
|
||||||
|
|
||||||
|
return Task.FromResult(JsonSerializer.Deserialize<T>(_buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<T?> AsDynamicObject<T>(T _, CancellationToken ct = default)
|
||||||
|
=> AsSerializedObject<T>(ct);
|
||||||
|
|
||||||
|
public Task<string> AsString(CancellationToken ct = default) {
|
||||||
|
if (!Is200) Logger?.LogWarning("Non-success response; attempting to read content.");
|
||||||
|
return Task.FromResult(Encoding.UTF8.GetString(_buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<byte[]> AsBinary(CancellationToken ct = default) {
|
||||||
|
if (!Is200) Logger?.LogWarning("Non-success response; attempting to read content.");
|
||||||
|
return Task.FromResult(_buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<Stream> AsStream(CancellationToken ct = default) {
|
||||||
|
if (!Is200) Logger?.LogWarning("Non-success response; attempting to read content.");
|
||||||
|
return Task.FromResult<Stream>(new MemoryStream(_buffer, writable: false));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,19 +5,13 @@ using System.Text;
|
|||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
namespace Beam {
|
namespace Beam {
|
||||||
//public class DownloadEnumerable<T>(IAsyncEnumerator<Ordered<T>> download) : IAsyncEnumerable<Ordered<T>> {
|
public class DownloadEnumerable<T>(IAsyncEnumerator<Ordered<T>> download) : IAsyncEnumerable<Ordered<T>> {
|
||||||
// public IAsyncEnumerator<Ordered<T>> Download { get; } = download;
|
public IAsyncEnumerator<Ordered<T>> Download { get; } = download;
|
||||||
|
|
||||||
// public IAsyncEnumerator<Ordered<T>> GetAsyncEnumerator(CancellationToken cancellationToken = default)
|
public IAsyncEnumerator<Ordered<T>> GetAsyncEnumerator(CancellationToken cancellationToken = default)
|
||||||
// => Download;
|
=> Download;
|
||||||
|
|
||||||
// public static DownloadEnumerable<T> Empty()
|
public static DownloadEnumerable<T> Empty()
|
||||||
// => new(Array.Empty<Ordered<T>>().ToAsyncEnumerable().GetAsyncEnumerator());
|
=> new(Array.Empty<Ordered<T>>().ToAsyncEnumerable().GetAsyncEnumerator());
|
||||||
//}
|
|
||||||
|
|
||||||
public class DownloadEnumerable<T>(IAsyncEnumerable<Ordered<T>> download) : IAsyncEnumerable<Ordered<T>> {
|
|
||||||
public IAsyncEnumerator<Ordered<T>> GetAsyncEnumerator(CancellationToken cancellationToken = default) {
|
|
||||||
return download.GetAsyncEnumerator(cancellationToken);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user