feat: add Puppeteer integration for web downloads

This introduces a new Puppeteer-based mechanism for downloading web content. It provides a flexible way to manipulate pages during downloads, enhancing the ability to handle dynamic content and improve the overall download process.
2025-06-25 13:42:24 +03:00
parent 2317db9d3f
commit 3baa31a7cc
5 changed files with 117 additions and 25 deletions
@@ -0,0 +1,37 @@
+
+using HtmlAgilityPack;
+using Microsoft.Playwright;
+
+namespace Beam.Puppeteer {
+    public class PuppetUnitPageDownloader<T> : UnitDownloader<T> {
+        public AsyncManipulator PuppetManipulator { get; }
+
+        public PuppetUnitPageDownloader(HtmlWeb web, AsyncManipulator puppetManipulator, AsyncTransformer<HtmlDocument, T> asyncHtmlTransformer, AsyncDownloadFailurePredicate<HtmlDocument>[] asyncDownloadFailurePredicates)
+            : base(web, asyncHtmlTransformer, asyncDownloadFailurePredicates) {
+            PuppetManipulator = puppetManipulator;
+        }
+
+        protected override async Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
+            var page = await PuppetContext.Browser.Value.NewPageAsync();
+            try {
+                await page.GotoAsync(link);
+                await PuppetManipulator(page);
+                var content = await page.ContentAsync();
+                await page.CloseAsync();
+
+                HtmlDocument doc = new();
+                doc.LoadHtml(content);
+                var transformed = await Transformer(doc);
+                if (FailurePredicates is null || !(await IsFailure(doc)))
+                    return (true, transformed);
+                return (false, default);
+            } catch (Exception) {
+                return (false, default);
+            } finally {
+                if (!page.IsClosed)
+                    await page.CloseAsync();
+            }
+        }
+    }
+
+}