using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Beam.Stealth {
public class StealthUnitPageDownloader : UnitDownloader {
public StealthConfig Config { get; }
public StealthAsyncManipulator Manipulator { get; }
private ILogger? Logger => Config.Logger;
public StealthUnitPageDownloader(HtmlWeb web, StealthConfig config, StealthAsyncManipulator manipulator, AsyncTransformer transformer, AsyncDownloadFailurePredicate?[]? failurePredicate = null) : base(web, transformer, failurePredicate) {
Config = config;
Manipulator = manipulator;
}
protected async override Task<(bool, T?)> TryDownloadWithNoRetries(string link, CancellationToken ct) {
try {
var driver = Config.Driver;
await driver.Navigate().GoToUrlAsync(link);
await Manipulator(driver);
HtmlDocument doc = new();
doc.LoadHtml(driver.PageSource);
if (await IsFailure(doc))
return (false, default);
return (true, await Transformer(doc));
} catch (Exception e) {
Logger?.LogError(e, "Error occurred downloading {}", link);
return (false, default);
}
}
}
}