Beam/Beam.Downloaders/SequentialDownloader.cs

using Beam.Abstractions;
using Beam.Models;
using Microsoft.Extensions.Logging;

namespace Beam.Downloaders {
    public class SequentialDownloader<RawType, OutType> : IAsyncEnumerator<OutType> {
        public OutType Current { get; protected set; }
        public DownloadContext<RawType> Context { get; }
        public ILogger? Logger { get; set; }
        public int LastOrder { get; set; } = 0;

        protected IEnumerator<string> LinksEnumerator;

        public Func<IUnitDownloader<OutType>> GetUnitDownloader { get; set; }

        public SequentialDownloader(DownloadContext<RawType> context, Func<DownloadContext<RawType>, IUnitDownloader<OutType>> getUnitDownloader, ILogger? logger = null) {
            Context = context;
            Logger = logger;
            LinksEnumerator = Context.Links.GetEnumerator();

            try {
                LinksEnumerator.Reset();
            } catch (NotSupportedException) {
                Logger?.LogWarning("Enumerator of type {} does not support resets. This may cause buggy behavior", LinksEnumerator.GetType());
            }
            Current = default(OutType);
            GetUnitDownloader = () => getUnitDownloader(Context);
        }

        public ValueTask DisposeAsync() {
            GC.SuppressFinalize(this);
            return ValueTask.CompletedTask;
        }

        public async ValueTask<bool> MoveNextAsync() {
            if (!LinksEnumerator.MoveNext())
                return false;

            //Logger?.LogInformation("MoveNextAsync()");
            var unit = GetUnitDownloader(); // safe to instantiate per request.
            var idealLinkCount = unit.LinksPerDownload;
            List<Ordered<string>> links = [];

            //Logger?.LogInformation("MoveNextAsync() \n\t -> Links.Current = {} ", LinksEnumerator.Current.Link.AbsoluteUri);
            links.Add(new Ordered<string>(LinksEnumerator.Current, LastOrder++));

            while (LinksEnumerator.MoveNext() && !string.IsNullOrWhiteSpace(LinksEnumerator.Current) && links.Count < idealLinkCount)
                links.Add(new Ordered<string>(LinksEnumerator.Current, LastOrder++));
            //Logger?.LogInformation("MoveNextAsync() \n\t -> links.Count = {} ", links.Count);
            if (links.Count == 0) {
                Logger?.LogInformation("Out of links!");
                return false;
            }

            if (links.Any((x) => string.IsNullOrWhiteSpace(x.Data)))
                return false;

            var (result, downloadedT) = await unit.TryDownload(
                links.ToArray(),
                Context.CancellationToken, 
                tryProgress: Context.RetryReporter);

            if (!result) {
                Logger?.LogWarning("Failed to download Unit<{}>", typeof(OutType).Name);
                return false; // unit download failed
            }
            if (downloadedT is null) {
                Logger?.LogWarning("Failed to download Unit<{}>", typeof(OutType).Name);
                return false; // unit download failed
            }

            Current = downloadedT;
            return true;
        }
    }
}