Beam/Beam.Stealth/StealthUnitPageDownloader.cs

using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Beam.Stealth {
    public class StealthUnitPageDownloader<T> : UnitDownloader<T> {
        public StealthConfig Config { get; }
        public StealthAsyncManipulator Manipulator { get; }

        private ILogger? Logger => Config.Logger;

        public StealthUnitPageDownloader(HtmlWeb web, StealthConfig config, StealthAsyncManipulator manipulator, AsyncTransformer<HtmlDocument, T> transformer, AsyncDownloadFailurePredicate<HtmlDocument>?[]? failurePredicate = null) : base(web, transformer, failurePredicate) {
            Config = config;
            Manipulator = manipulator;
        }

        protected async override Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
            try {
                var driver = Config.Driver;

                await driver.Navigate().GoToUrlAsync(link);
                await Manipulator(driver);

                HtmlDocument doc = new();
                doc.LoadHtml(driver.PageSource);

                if (await IsFailure(doc))
                    return (false, default);

                return (true, await Transformer(doc));
            } catch (Exception e) {
                Logger?.LogError(e, "Error occurred downloading {}", link);
                return (false, default);
            }
        }
    }
}