From 647b2b0f370c62abb6de6263f62b63a87d4e8f85 Mon Sep 17 00:00:00 2001 From: qwsdcvghyu89 <61093706+qwsdcvghyu89@users.noreply.github.com> Date: Sat, 15 Nov 2025 20:51:18 +1100 Subject: [PATCH] feat: introduce new composable data providers and increment version - Added `AnchorDataProvider`, `AnchorCollectionDataProvider`, `ContentsDataProvider`, `ContentsArrayDataProvider`, `DropDownDataProvider`, `ListContentDataProvider`, and `ParagraphedContentDataProvider` for enhanced data extraction flexibility. - Updated project version to 2.5.0. --- Beam.Dynamic/Beam.Dynamic.csproj.DotSettings | 2 + Beam.Dynamic/BindingsCollection.cs | 12 ++++ .../AnchorCollectionDataProvider.cs | 46 +++++++++++++++ .../DataProviders/AnchorDataProvider.cs | 32 ++++++++++ .../ContentsArrayDataProvider.cs | 18 ++++++ .../DataProviders/ContentsDataProvider.cs | 30 ++++++++++ .../DataProviders/DropDownDataProvider.cs | 59 +++++++++++++++++++ .../DataProviders/ListContentDataProvider.cs | 37 ++++++++++++ .../ParagraphedContentDataProvider.cs | 39 ++++++++++++ aeqw89.Beam/aeqw89.Beam.csproj | 4 +- aeqw89.Beam/aeqw89.Beam.csproj.bak | 4 +- 11 files changed, 279 insertions(+), 4 deletions(-) create mode 100644 Beam.Dynamic/Beam.Dynamic.csproj.DotSettings create mode 100644 Beam.Dynamic/BindingsCollection.cs create mode 100644 Beam.Dynamic/DataProviders/AnchorCollectionDataProvider.cs create mode 100644 Beam.Dynamic/DataProviders/AnchorDataProvider.cs create mode 100644 Beam.Dynamic/DataProviders/ContentsArrayDataProvider.cs create mode 100644 Beam.Dynamic/DataProviders/ContentsDataProvider.cs create mode 100644 Beam.Dynamic/DataProviders/DropDownDataProvider.cs create mode 100644 Beam.Dynamic/DataProviders/ListContentDataProvider.cs create mode 100644 Beam.Dynamic/DataProviders/ParagraphedContentDataProvider.cs diff --git a/Beam.Dynamic/Beam.Dynamic.csproj.DotSettings b/Beam.Dynamic/Beam.Dynamic.csproj.DotSettings new file mode 100644 index 0000000..82b21c1 --- /dev/null +++ b/Beam.Dynamic/Beam.Dynamic.csproj.DotSettings @@ -0,0 +1,2 @@ + + True \ No newline at end of file diff --git a/Beam.Dynamic/BindingsCollection.cs b/Beam.Dynamic/BindingsCollection.cs new file mode 100644 index 0000000..f78c95e --- /dev/null +++ b/Beam.Dynamic/BindingsCollection.cs @@ -0,0 +1,12 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using aeqw89.DataKeys; +using aeqw89.PersistentData; +using Beam.Abstractions; + +namespace Beam.Dynamic; + +public class BindingsCollection(Table? providers = null) { + public Table Providers { get; } = providers ?? []; + +} \ No newline at end of file diff --git a/Beam.Dynamic/DataProviders/AnchorCollectionDataProvider.cs b/Beam.Dynamic/DataProviders/AnchorCollectionDataProvider.cs new file mode 100644 index 0000000..9fd2fcb --- /dev/null +++ b/Beam.Dynamic/DataProviders/AnchorCollectionDataProvider.cs @@ -0,0 +1,46 @@ +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Beam.Abstractions; + +namespace Beam.Dynamic { + public class AnchorCollectionDataProvider : IComposableDataProvider { + public IBinding? Content { get; set; } + public Uri? RelativeTo { get; set; } + + public string[] Get(HtmlDocument document) { + var node = Select(document); + return node is null ? [] : Get(node); + } + + public string[] Get(HtmlNode node) { + List links = []; + foreach (var child in node.Descendants()) { + var href = child.GetAttributeValue("href", ""); + if (Uri.TryCreate(RelativeTo, href, out var uri)) + links.Add(uri.AbsoluteUri); + } + + return links.Where(x => !string.IsNullOrWhiteSpace(x)).ToArray(); + } + + public HtmlNode? Select(HtmlDocument doc) { + return Content?.Select(doc); + } + + public HtmlNode? Select(HtmlNode node) { + return node; + } + + + public HtmlNode[]? SelectMany(HtmlDocument doc) { + throw new NotImplementedException(); + } + public HtmlNode[]? SelectMany(HtmlNode[] node) { + throw new NotImplementedException(); + } + } +} diff --git a/Beam.Dynamic/DataProviders/AnchorDataProvider.cs b/Beam.Dynamic/DataProviders/AnchorDataProvider.cs new file mode 100644 index 0000000..ba54ff6 --- /dev/null +++ b/Beam.Dynamic/DataProviders/AnchorDataProvider.cs @@ -0,0 +1,32 @@ +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Beam.Abstractions; + +namespace Beam.Dynamic { + public class AnchorDataProvider : IComposableDataProvider { + public IBinding? Content { get; set; } + public Uri? RelativeTo { get; set; } + + public string Get(HtmlDocument document) { + var node = Select(document); + return node is null ? "" : Get(node); + } + + public virtual string Get(HtmlNode node) { + if (Uri.TryCreate(RelativeTo, node.GetAttributeValue("href", ""), out var uri)) + return uri.AbsoluteUri; + return ""; + } + public HtmlNode? Select(HtmlDocument doc) { + return Content?.Select(doc); + } + + public HtmlNode? Select(HtmlNode node) { + return node; + } + } +} diff --git a/Beam.Dynamic/DataProviders/ContentsArrayDataProvider.cs b/Beam.Dynamic/DataProviders/ContentsArrayDataProvider.cs new file mode 100644 index 0000000..a4f5937 --- /dev/null +++ b/Beam.Dynamic/DataProviders/ContentsArrayDataProvider.cs @@ -0,0 +1,18 @@ +using Beam.Abstractions; +using HtmlAgilityPack; + +namespace Beam.Dynamic { + public class ContentsArrayDataProvider : ContentsDataProvider, IComposableDataProvider { + public string[] ArrayDelimiters { get; set; } = [";"]; + + string[] IDataProvider.Get(HtmlDocument document) { + var node = Select(document); + return node is null ? [] : Get(node); + } + + public new string[] Get(HtmlNode node) { + return node.InnerText?.Split(ArrayDelimiters, StringSplitOptions.RemoveEmptyEntries) ?? []; + } + } + +} diff --git a/Beam.Dynamic/DataProviders/ContentsDataProvider.cs b/Beam.Dynamic/DataProviders/ContentsDataProvider.cs new file mode 100644 index 0000000..3bb5797 --- /dev/null +++ b/Beam.Dynamic/DataProviders/ContentsDataProvider.cs @@ -0,0 +1,30 @@ +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Beam.Abstractions; + +namespace Beam.Dynamic { + public class ContentsDataProvider : IComposableDataProvider { + public IBinding? Content { get; set; } + + public string Get(HtmlDocument document) { + var node = Select(document); + return node is null ? "" : Get(node); + } + + public string Get(HtmlNode node) { + return node.InnerText; + } + public HtmlNode? Select(HtmlDocument doc) { + return Content?.Select(doc); + } + + public HtmlNode? Select(HtmlNode node) { + return node; + } + } +} diff --git a/Beam.Dynamic/DataProviders/DropDownDataProvider.cs b/Beam.Dynamic/DataProviders/DropDownDataProvider.cs new file mode 100644 index 0000000..ac64183 --- /dev/null +++ b/Beam.Dynamic/DataProviders/DropDownDataProvider.cs @@ -0,0 +1,59 @@ +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Beam.Abstractions; + +namespace Beam.Dynamic; + +public class DropDownDataProvider : IComposableDataProvider, IComposableDataProvider { + public IBinding? Content { get; set; } + public Uri? RelativeTo { get; set; } + + public string[] Get(HtmlDocument document) { + if (Content is null) + return []; + var node = Select(document); + if (node is null) + return []; + return Get(node); + } + + string IDataProvider.Get(HtmlDocument document) { + var node = Select(document); + return node is null ? "" : (this as IComposableDataProvider).Get(node); + } + + public string[] Get(HtmlNode node) { + List links = []; + foreach (var child in node.ChildNodes.Where(x => x.Name == "option")) { + var childValue = child.GetAttributeValue("value", null); + if (!Uri.TryCreate(RelativeTo, childValue, out var uri)) + continue; + links.Add(uri.AbsoluteUri); + } + + return links.ToArray(); + } + + string IComposableDataProvider.Get(HtmlNode node) { + return JsonSerializer.Serialize(Get(node)); + } + + public HtmlNode? Select(HtmlDocument doc) { + return Content?.Select(doc); + } + + HtmlNode? IComposableDataProvider.Select(HtmlNode node) { + return node; + } + + HtmlNode? IComposableDataProvider.Select(HtmlNode node) { + return node; + } +} \ No newline at end of file diff --git a/Beam.Dynamic/DataProviders/ListContentDataProvider.cs b/Beam.Dynamic/DataProviders/ListContentDataProvider.cs new file mode 100644 index 0000000..b5b999f --- /dev/null +++ b/Beam.Dynamic/DataProviders/ListContentDataProvider.cs @@ -0,0 +1,37 @@ +using HtmlAgilityPack; +using System.Text; +using Beam.Abstractions; + +namespace Beam.Dynamic { + public class ListContentDataProvider : IComposableDataProvider { + public IBinding? Content { get; set; } + + public string Get(HtmlDocument document) { + if (Content is null) + return ""; + + var node = Select(document); + return node is null ? "" : Get(node); + } + + public string Get(HtmlNode node) { + + StringBuilder content = new(); + foreach(var childNode in node.ChildNodes.SkipLast(1)) { + if (childNode.Name != "li") + continue; + content.Append(childNode.InnerText.Trim() + ";"); + } + + content.Append(node.ChildNodes.Last().InnerText.Trim()); + return content.ToString(); + } + public HtmlNode? Select(HtmlDocument doc) { + return Content?.Select(doc); + } + + public HtmlNode? Select(HtmlNode node) { + return node; + } + } +} diff --git a/Beam.Dynamic/DataProviders/ParagraphedContentDataProvider.cs b/Beam.Dynamic/DataProviders/ParagraphedContentDataProvider.cs new file mode 100644 index 0000000..afeae0b --- /dev/null +++ b/Beam.Dynamic/DataProviders/ParagraphedContentDataProvider.cs @@ -0,0 +1,39 @@ +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Beam.Abstractions; + +namespace Beam.Dynamic { + public class ParagraphedContentDataProvider : IComposableDataProvider { + public IBinding? Content { get; set; } + + public string Get(HtmlDocument document) { + if (Content is null) + return ""; + + var node = Content.Select(document); + return node is null ? "" : Get(node); + } + + public string Get(HtmlNode node) { + StringBuilder content = new(); + foreach(var childNode in node.ChildNodes) { + if (childNode.Name != "p") + continue; + content.AppendLine(childNode.InnerText); + } + + return content.ToString(); + } + public HtmlNode? Select(HtmlDocument doc) { + return Content?.Select(doc); + } + + public HtmlNode? Select(HtmlNode node) { + return node; + } + } +} diff --git a/aeqw89.Beam/aeqw89.Beam.csproj b/aeqw89.Beam/aeqw89.Beam.csproj index a738cb3..32579b2 100644 --- a/aeqw89.Beam/aeqw89.Beam.csproj +++ b/aeqw89.Beam/aeqw89.Beam.csproj @@ -7,12 +7,12 @@ Beam aeqw89 qwsdcvghyu - 2.4.6 + 2.5.0 A library for downloading internet resources https://github.com/qwsdcvghyu89/Beam https://github.com/qwsdcvghyu89/Beam aeqw89.Beam - 2.4.6 + 2.5.0 diff --git a/aeqw89.Beam/aeqw89.Beam.csproj.bak b/aeqw89.Beam/aeqw89.Beam.csproj.bak index 933028d..a738cb3 100644 --- a/aeqw89.Beam/aeqw89.Beam.csproj.bak +++ b/aeqw89.Beam/aeqw89.Beam.csproj.bak @@ -7,12 +7,12 @@ Beam aeqw89 qwsdcvghyu - 2.4.5 + 2.4.6 A library for downloading internet resources https://github.com/qwsdcvghyu89/Beam https://github.com/qwsdcvghyu89/Beam aeqw89.Beam - 2.4.5 + 2.4.6