Beam/Beam.Puppeteer/PuppetUnitPageDownloader.cs

using HtmlAgilityPack;
using Microsoft.Playwright;

namespace Beam.Puppeteer {
    public class PuppetUnitPageDownloader<T> : UnitDownloader<T> {
        public AsyncManipulator PuppetManipulator { get; }

        public PuppetUnitPageDownloader(HtmlWeb web, AsyncManipulator puppetManipulator, AsyncTransformer<HtmlDocument, T> asyncHtmlTransformer, AsyncDownloadFailurePredicate<HtmlDocument>[] asyncDownloadFailurePredicates)
            : base(web, asyncHtmlTransformer, asyncDownloadFailurePredicates) {
            PuppetManipulator = puppetManipulator;
        }

        protected override async Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
            var page = await PuppetContext.Browser.Value.NewPageAsync();
            try {
                await page.GotoAsync(link);
                await PuppetManipulator(page);
                var content = await page.ContentAsync();
                await page.CloseAsync();

                HtmlDocument doc = new();
                doc.LoadHtml(content);
                var transformed = await Transformer(doc);
                if (FailurePredicates is null || !(await IsFailure(doc)))
                    return (true, transformed);
                return (false, default);
            } catch (Exception) {
                return (false, default);
            } finally {
                if (!page.IsClosed)
                    await page.CloseAsync();
            }
        }
    }

}