using HtmlAgilityPack; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using Beam.Downloaders; using Beam.Models; namespace Beam.Stealth { public class StealthUnitPageDownloader : UnitDownloader { public StealthConfig Config { get; } public StealthAsyncManipulator Manipulator { get; } private ILogger? Logger => Config.Logger; public StealthUnitPageDownloader(HtmlWeb web, StealthConfig config, StealthAsyncManipulator manipulator, AsyncTransformer transformer, AsyncDownloadFailurePredicate?[]? failurePredicate = null) : base(web, transformer, failurePredicate) { Config = config; Manipulator = manipulator; } protected async override Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) { try { var driver = Config.Driver; await driver.Navigate().GoToUrlAsync(link); await Manipulator(driver); HtmlDocument doc = new(); doc.LoadHtml(driver.PageSource); if (await IsFailure(doc)) return (false, default); return (true, await Transformer(doc)); } catch (Exception e) { Logger?.LogError(e, "Error occurred downloading {}", link); return (false, default); } } } }