Introduce Beam.Fluent and Beam.Models projects

Added new Beam.Fluent and Beam.Models projects with staged download builder and data context models. Refactored and moved model classes from Beam.Temporary.Cli to Beam.Models. Added new data providers and extended DataBindings in Beam.Dynamic. Renamed Beam.Puppeteer to Beam.Playwright and updated related classes. Updated project references and package versions. Removed obsolete and unused files from Beam.Temporary.Cli.
This commit is contained in:
qwsdcvghyu89
2025-09-18 18:32:25 +10:00
parent 849bdcd089
commit a7d148a96f
72 changed files with 2100 additions and 721 deletions
+344 -68
View File
@@ -8,6 +8,7 @@ using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using Beam.Models;
namespace Beam.Temporary.Cli {
@@ -43,98 +44,373 @@ namespace Beam.Temporary.Cli {
// sdd.AggregatorNovels.TryAdd(lnwAggregator, [novel.Key]);
//}
public static void Define_WoDuShu_HouseOfHorrors(BeamDataDictionary sdd) {
var (wdsAgg, wdsAux) = CreateKeyPair<WebResource>("aggregators", "auxillaries", "wodushu", "aeqw89:document");
var novel = new TextResource() {
Key = new DataKey<TextResource>("novels:house_of_horrors"),
FriendlyName = "My House Of Horrors",
AssociatedSource = wdsAgg,
AssociatedMetaSource = wdsAux,
TemplateInitialData = new ImmutableState(["24349", "2896325"]),
MetaTemplateInitialData = new ImmutableState(["24349"])
};
//// -----------------------------------------------------------------------------
//// Helper: same as in the WoDuShu file
//private static (DataKey<T>, DataKey<T>) CreateKeyPair<T>(
// string pref1, string pref2, string common, string @namespace) {
// return (
// new DataKey<T>($"{pref1}:{common}").WithNamespace(@namespace),
// new DataKey<T>($"{pref2}:{common}").WithNamespace(@namespace)
// );
//}
sdd.Novels.TryAdd(novel.Key, novel);
// -----------------------------------------------------------------------------
// 1) Site-wide definition YeBiQuge (m.yebiquge.com)
public static void Define_YeBiQuge(BeamDataContext sdd) {
// ---------- keys ----------
var yb = new DataKey<WebResource>("aeqw89:yebiquge");
sdd.AggregatorNovels.TryAdd(wdsAgg, [novel.Key]);
}
var bindings = new DataKey<DataBindings>($"aeqw89:yebiquge:{nameof(IDocument).ToLower()}:bindings");
var bindings_info = new DataKey<DataBindings>($"aeqw89:yebiquge:{nameof(ArticleData).ToLower()}:bindings");
var bindings_toc = new DataKey<DataBindings>($"aeqw89:yebiquge:{nameof(TableOfContentsData).ToLower()}:bindings");
private static (DataKey<T>, DataKey<T>) CreateKeyPair<T>(string pref1, string pref2, string common, string @namespace) {
return (
new DataKey<T>(pref1 + ":" + common).WithNamespace(@namespace),
new DataKey<T>(pref2 + ":" + common).WithNamespace(@namespace)
);
}
public static void Define_WoDuShu(BeamDataDictionary sdd) {
var (wdsAgg, wdsAux) = CreateKeyPair<WebResource>("aggregators", "auxillaries", "wodushu", "aeqw89:document");
var bindings = new DataKey<DataBindings>("aeqw89:bindings:wodushu");
var aggregator = new WebResource(wdsAgg) {
Name = "WoDuShu.com",
Description = "A Chinese novel aggregator site",
Domain = "https://wodushu.com",
// ---------- web resources ----------
var aggregator = new WebResource(yb.InsertEnd(nameof(IDocument).ToLower())) {
Name = "YeBiQuge Chapters",
Description = "Chapter pages (mobile)",
Domain = "https://m.yebiquge.com",
Bindings = bindings
};
var auxiliary = new WebResource(wdsAux) {
Name = "WoDuShu.com",
Description = "A Chinese novel aggregator site",
Domain = "https://wodushu.com",
Bindings = bindings.WithSuffix("_aux")
var bookInfo = new WebResource(yb.InsertEnd(nameof(ArticleData).ToLower())) {
Name = "YeBiQuge Book Info",
Description = "Book information / latest updates page",
Domain = "https://m.yebiquge.com",
Bindings = bindings_info
};
sdd.Templates.TryAdd(wdsAgg, new() {
var tocPage = new WebResource(yb.InsertEnd(nameof(TableOfContentsData).ToLower())) {
Name = "YeBiQuge TOC",
Description = "Full chapter list (index*.html)",
Domain = "https://m.yebiquge.com",
Bindings = bindings_toc
};
sdd.Resources.TryAdd(aggregator.Key, aggregator);
sdd.Resources.TryAdd(bookInfo.Key, bookInfo);
sdd.Resources.TryAdd(tocPage.Key, tocPage);
// ---------- URL templates ----------
// 1-a) Chapter page /{catId}/{bookId}/{chapterId}.html
sdd.Templates.TryAdd(aggregator.Key, new() {
Factory = new(StateChangerFactory.LastAsNumber),
Builder = new SourceLinkBuilder("www.wodushu.com")
.WithSegments("read", "", "")
.WithParameters(1, "")
.WithParameters(2, (".html", Position.After))
Builder = new SourceLinkBuilder("m.yebiquge.com")
.WithSegments("", "", "") // /<cat>/<book>/<chap>
.WithParameters(0, "")
.WithParameters(1, "")
.WithParameters(2, (".html", Position.After)) // chapId.html
});
sdd.Templates.TryAdd(wdsAux, new() {
// 1-b) Book-info page /{catId}/{bookId}/
sdd.Templates.TryAdd(bookInfo.Key, new() {
Factory = new(StateChangerFactory.Constant),
Builder = new SourceLinkBuilder("www.wodushu.com")
.WithSegments("book", "")
.WithParameters(1, "")
Builder = new SourceLinkBuilder("m.yebiquge.com")
.WithSegments("", "") // /<cat>/<book>/
.WithParameters(0, "")
.WithParameters(1, "")
});
sdd.Aggregators.TryAdd(wdsAgg, aggregator);
sdd.Auxillaries.TryAdd(wdsAux, auxiliary);
// 1-c) TOC page /{catId}/{bookId}/index.html (first page)
sdd.Templates.TryAdd(tocPage.Key, new() {
Factory = new(StateChangerFactory.Constant),
Builder = new SourceLinkBuilder("m.yebiquge.com")
.WithSegments("", "", "index.html") // /<cat>/<book>/index.html
.WithParameters(0, "")
.WithParameters(1, "")
});
var binding_agg = new DataKey<DataBindings>("aeqw89:bindings:wodushu");
var binding_aux = new DataKey<DataBindings>("aeqw89:bindings:wodushu_aux");
sdd.Bindings.Add(binding_agg, new() {
Title = new ContentsDataProvider() {
Content = new Binding() {
XPath = "/html/body/div[4]/div/div/div[2]/h1"
}
// ---------- bindings ----------
// ── chapter page ────────────────────────────────────────────────
sdd.Bindings.Add(bindings, new() {
Title = new ContentsDataProvider {
Content = new Binding { XPath = "//*[@id='nr_title']" }
},
Content = new ParagraphedContentDataProvider() {
Content = new Binding() {
XPath = "//*[@id=\"content\"]"
}
Content = new ParagraphedContentDataProvider {
Content = new Binding { XPath = "//*[@id='chaptercontent']" }
},
});
sdd.Bindings.Add(binding_aux, new() {
Title = new ContentsDataProvider() {
Content = new Binding() {
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[1]/h1"
// ── book-info page ──────────────────────────────────────────────
sdd.Bindings.Add(bindings_info, new() {
Title = new ContentsDataProvider {
Content = new Binding { XPath = "//div[@class='book_info']//dt[@class='name']" }
},
Authors = new ContentsArrayDataProvider {
Content = new Binding {
XPath = "//div[@class='book_info']//span[contains(text(),'作者')]"
}
},
Authors = new ContentsArrayDataProvider() {
Content = new Binding() {
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[1]/div/p[1]/a"
}
Description = new ParagraphedContentDataProvider {
Content = new Binding { XPath = "//div[@class='book_about']/dl/dd" }
}
});
// ── TOC page ────────────────────────────────────────────────────
sdd.Bindings.Add(bindings_toc, new() {
PagesDropDown = new DropDownDataProvider {
Content = new Binding { XPath = "//div[@class='fenye']//select" },
RelativeTo = tocPage.Domain
},
Description = new ParagraphedContentDataProvider() {
Content = new Binding() {
XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[2]"
}
TableOfContents = new AnchorCollectionDataProvider {
Content = new Binding { XPath = "//div[@class='book_last']/dl" },
RelativeTo = tocPage.Domain
}
});
}
// -----------------------------------------------------------------------------
// 2) Concrete novel 《诡秘之主》 / Lord of the Mysteries
public static void Define_YeBiQuge_LordOfMysteries(BeamDataContext sdd) {
var yb = new DataKey<WebResource>("aeqw89:yebiquge:novels:lord_of_the_mysteries");
var ybAgg = yb.InsertEnd(nameof(IDocument).ToLower());
var ybInfo = yb.InsertEnd(nameof(ArticleData).ToLower());
var ybToc = yb.InsertEnd(nameof(TableOfContentsData).ToLower());
var novel = new ResourceDictionary {
Key = yb.To<ResourceDictionary>(),
FriendlyName = "Lord of the Mysteries",
Resources = {
{ nameof(IDocument) , ybAgg }, // chapters
{ nameof(ArticleData) , ybInfo }, // book info
{ nameof(TableOfContentsData), ybToc } // full TOC
},
// catId = 2 , bookId = 2958 , sample chapterId = 8699808
InitialStates = new Dictionary<DataKey<WebResource>, ImmutableState> {
{ ybAgg, new ImmutableState(["2","2958","8699808"]) },
{ ybInfo, new ImmutableState(["2","2958"]) },
{ ybToc, new ImmutableState(["2","2958"]) },
}
};
sdd.ResourceDictionaries.TryAdd(novel.Key, novel);
sdd.AggregatorNovels.TryAdd(ybAgg, [novel.Key]);
}
// -----------------------------------------------------------------------------
// 1) Site-wide definition KuaiShu5 (www.kuaishu5.com)
public static void Define_KuaiShu5(BeamDataContext sdd) {
// ---------- keys ----------
var ks = new DataKey<WebResource>("aeqw89:kuaishu5");
var bindings_chapter = new DataKey<DataBindings>($"aeqw89:kuaishu5:{nameof(IDocument).ToLower()}:bindings");
var bindings_info = new DataKey<DataBindings>($"aeqw89:kuaishu5:{nameof(ArticleData).ToLower()}:bindings");
var bindings_toc = new DataKey<DataBindings>($"aeqw89:kuaishu5:{nameof(TableOfContentsData).ToLower()}:bindings");
// ---------- web resources ----------
var chapters = new WebResource(ks.InsertEnd(nameof(IDocument).ToLower())) {
Name = "KuaiShu5 Chapters",
Description = "Chapter pages",
Domain = "https://www.kuaishu5.com",
Bindings = bindings_chapter
};
var bookInfo = new WebResource(ks.InsertEnd(nameof(ArticleData).ToLower())) {
Name = "KuaiShu5 Book Info",
Description = "Book information / landing page",
Domain = "https://www.kuaishu5.com",
Bindings = bindings_info
};
var tocPage = new WebResource(ks.InsertEnd(nameof(TableOfContentsData).ToLower())) {
Name = "KuaiShu5 TOC",
Description = "Full chapter list (index page)",
Domain = "https://www.kuaishu5.com",
Bindings = bindings_toc
};
sdd.Resources.TryAdd(chapters.Key, chapters);
sdd.Resources.TryAdd(bookInfo.Key, bookInfo);
sdd.Resources.TryAdd(tocPage.Key, tocPage);
// ---------- URL templates ----------
// 1-a) Chapter page /b{bookId}/{chapterId}.html
sdd.Templates.TryAdd(chapters.Key, new() {
Factory = new(StateChangerFactory.LastAsNumber),
Builder = new SourceLinkBuilder("www.kuaishu5.com")
.WithSegments("", "") // /<seg0>/<seg1>
.WithParameters(0, ("b", Position.Before)) // seg0: b{bookId}
.WithParameters(1, (".html", Position.After)) // seg1: {chapterId}.html
});
// 1-b) Book-info page /b{bookId}/
sdd.Templates.TryAdd(bookInfo.Key, new() {
Factory = new(StateChangerFactory.Constant),
Builder = new SourceLinkBuilder("www.kuaishu5.com")
.WithSegments("") // /<seg0>
.WithParameters(0, ("b", Position.Before)) // seg0: b{bookId}
});
// 1-c) TOC page /b{bookId}/ (same as book-info)
sdd.Templates.TryAdd(tocPage.Key, new() {
Factory = new(StateChangerFactory.Constant),
Builder = new SourceLinkBuilder("www.kuaishu5.com")
.WithSegments("") // /<seg0>
.WithParameters(0, ("b", Position.Before)) // seg0: b{bookId}
});
// ---------- bindings ----------
// ── chapter page ────────────────────────────────────────────────
sdd.Bindings.Add(bindings_chapter, new() {
Title = new ContentsDataProvider {
Content = new Binding { XPath = "//h1[@class='bookname']" }
},
Content = new ParagraphedContentDataProvider {
Content = new Binding { XPath = "//*[@id='booktxt']" }
}
});
// ── book-info page ──────────────────────────────────────────────
sdd.Bindings.Add(bindings_info, new() {
Title = new ContentsDataProvider {
Content = new Binding { XPath = "//*[@id='info']/h1" }
},
Authors = new ContentsArrayDataProvider {
Content = new Binding {
XPath = "//*[@id='info']//p[contains(text(),'作者')]/a"
}
},
Description = new ParagraphedContentDataProvider {
Content = new Binding { XPath = "//*[@id='intro']" }
}
});
// ── TOC page ────────────────────────────────────────────────────
sdd.Bindings.Add(bindings_toc, new() {
PagesDropDown = new DropDownDataProvider {
Content = new Binding { XPath = "//*[@id='indexselect']" },
RelativeTo = tocPage.Domain
},
TableOfContents = new AnchorCollectionDataProvider {
Content = new Binding { XPath = "//*[@id='content_1']" },
RelativeTo = tocPage.Domain
}
});
}
// -----------------------------------------------------------------------------
// 2) Concrete novel 《诡秘之主》 / Lord of the Mysteries
public static void Define_KuaiShu5_LordOfMysteries(BeamDataContext sdd) {
var ks = new DataKey<WebResource>("aeqw89:kuaishu5");
var ksChapters = ks.InsertEnd(nameof(IDocument).ToLower());
var ksInfo = ks.InsertEnd(nameof(ArticleData).ToLower());
var ksToc = ks.InsertEnd(nameof(TableOfContentsData).ToLower());
var novel = new ResourceDictionary {
Key = new DataKey<ResourceDictionary>("kuaishu5:novels:lord_of_the_mysteries"),
FriendlyName = "Lord of the Mysteries",
Resources =
{
{ nameof(IDocument) , ksChapters },
{ nameof(ArticleData) , ksInfo },
{ nameof(TableOfContentsData), ksToc }
}
};
// bookId = 122722 , sample chapterId = 288372
sdd.InitialStates = new Dictionary<DataKey<ImmutableState>, ImmutableState>
{
{ ksChapters.To<ImmutableState>(), new ImmutableState(["122722", "288372"]) },
{ ksInfo .To<ImmutableState>(), new ImmutableState(["122722"]) },
{ ksToc .To<ImmutableState>(), new ImmutableState(["122722"]) }
};
sdd.ResourceDictionaries.TryAdd(novel.Key, novel);
sdd.AggregatorNovels.TryAdd(ksChapters, [novel.Key]);
}
//public static void Define_WoDuShu_HouseOfHorrors(BeamDataDictionary sdd) {
// var (wdsAgg, wdsAux) = CreateKeyPair<WebResource>("aggregators", "auxillaries", "wodushu", "aeqw89:document");
// var novel = new ResourceDictionary() {
// Key = new DataKey<ResourceDictionary>("novels:house_of_horrors"),
// FriendlyName = "My House Of Horrors",
// AssociatedSource = wdsAgg,
// AssociatedMetaSource = wdsAux,
// TemplateInitialData = new ImmutableState(["24349", "2896325"]),
// MetaTemplateInitialData = new ImmutableState(["24349"])
// };
// sdd.ResourceDictionaries.TryAdd(novel.Key, novel);
// sdd.AggregatorNovels.TryAdd(wdsAgg, [novel.Key]);
//}
private static (DataKey<T>, DataKey<T>) CreateKeyPair<T>(string pref1, string pref2, string common, string @namespace) {
return (
new DataKey<T>(pref1 + ":" + common).InsertStart(@namespace),
new DataKey<T>(pref2 + ":" + common).InsertStart(@namespace)
);
}
//public static void Define_WoDuShu(BeamDataDictionary sdd) {
// var (wdsAgg, wdsAux) = CreateKeyPair<WebResource>("aggregators", "auxillaries", "wodushu", "aeqw89:document");
// var bindings = new DataKey<DataBindings>("aeqw89:bindings:wodushu");
// var aggregator = new WebResource(wdsAgg) {
// Name = "WoDuShu.com",
// Description = "A Chinese novel aggregator site",
// Domain = "https://wodushu.com",
// Bindings = bindings
// };
// var auxiliary = new WebResource(wdsAux) {
// Name = "WoDuShu.com",
// Description = "A Chinese novel aggregator site",
// Domain = "https://wodushu.com",
// Bindings = bindings.WithSuffix("_aux")
// };
// sdd.Templates.TryAdd(wdsAgg, new() {
// Factory = new(StateChangerFactory.LastAsNumber),
// Builder = new SourceLinkBuilder("www.wodushu.com")
// .WithSegments("read", "", "")
// .WithParameters(1, "")
// .WithParameters(2, (".html", Position.After))
// });
// sdd.Templates.TryAdd(wdsAux, new() {
// Factory = new(StateChangerFactory.Constant),
// Builder = new SourceLinkBuilder("www.wodushu.com")
// .WithSegments("book", "")
// .WithParameters(1, "")
// });
// sdd.Resources.TryAdd(wdsAgg, aggregator);
// sdd.Auxillaries.TryAdd(wdsAux, auxiliary);
// var binding_agg = new DataKey<DataBindings>("aeqw89:bindings:wodushu");
// var binding_aux = new DataKey<DataBindings>("aeqw89:bindings:wodushu_aux");
// sdd.Bindings.Add(binding_agg, new() {
// Title = new ContentsDataProvider() {
// Content = new Binding() {
// XPath = "/html/body/div[4]/div/div/div[2]/h1"
// }
// },
// Content = new ParagraphedContentDataProvider() {
// Content = new Binding() {
// XPath = "//*[@id=\"content\"]"
// }
// },
// });
// sdd.Bindings.Add(binding_aux, new() {
// Title = new ContentsDataProvider() {
// Content = new Binding() {
// XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[1]/h1"
// }
// },
// Authors = new ContentsArrayDataProvider() {
// Content = new Binding() {
// XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[1]/div/p[1]/a"
// }
// },
// Description = new ParagraphedContentDataProvider() {
// Content = new Binding() {
// XPath = "/html/body/div[3]/div[1]/div/div/div[2]/div[2]"
// }
// },
// });
//}
//public static void Define_NovelFull(SharedDataDictionary sdd) {
// var docNamespace = "aeqw89:document";
// var nfAgg = new DataKey<WebResource>("aggregators:novel_full").WithNamespace(docNamespace);