refactor: modularize Beam into new projects and interfaces

- Introduced modularity by splitting Beam into new projects: Beam.Abstractions, Beam.Models, and Beam.Downloaders.
- Refactored existing classes into appropriate namespaces and projects.
- Replaced specific implementations with abstractions (e.g., SourceLinkBuilder to LinkBuilder, State to IState, etc.).
- Updated interfaces: added ITemplate, IArticleData, IDownloadReport, and others for improved extensibility.
- Removed deprecated classes like SourceLinkBuilder and StateChangerFactory.
- Enhanced link handling in downloaders by refactoring to use `string` over `SourceLink`.
- Consolidated shared logic under Beam.Abstractions.

Beam.Models/AsyncDownloadFailurePredicate.cs

@@ -0,0 +1,3 @@
+namespace Beam.Models;
+
+public delegate Task<bool> AsyncDownloadFailurePredicate<in T>(T download);

Beam.Models/AsyncTransformer.cs

@@ -0,0 +1,3 @@
+namespace Beam.Models;
+
+public delegate Task<U> AsyncTransformer<in T, U>(T elem);

Beam.Models/Beam.Models.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\Beam.Dynamic\Beam.Dynamic.csproj" />
-    <ProjectReference Include="..\Beam\Beam.csproj" />
+    <ProjectReference Include="..\Beam.Abstractions\Beam.Abstractions.csproj" />
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="aeqw89.PersistentData" Version="1.3.3" />
     <PackageReference Include="EntityFramework" Version="6.5.1" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.8">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -23,4 +23,9 @@
     </PackageReference>
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Abstract\" />
+    <Folder Include="Closed Concrete\" />
+  </ItemGroup>
+
 </Project>

Beam.Models/BeamDataContext.cs

@@ -1,50 +0,0 @@
-using aeqw89.PersistentData;
-using aeqw89.DataKeys;
-using Beam.Dynamic;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Text.Json;
-using System.Threading.Tasks;
-using System.Data.Entity;
-
-
-namespace Beam.Models {
-    public class BeamDataContext : BaseDataDictionary {
-        public Dictionary<DataKey<Template>, Template> Templates {
-            get => GetOrCreateDictionary<DataKey<Template>, Template>(nameof(Templates));
-            set => Set(nameof(Templates), value);
-        }
-
-        public Dictionary<DataKey<DataBindings>, DataBindings> Bindings {
-            get => GetOrCreateDictionary<DataKey<DataBindings>, DataBindings>(nameof(Bindings));
-            set => Set(nameof(Bindings), value);
-        }
-
-        public Dictionary<DataKey<WebResource>,  HashSet<DataKey<ResourceDictionary>>> AggregatorNovels {
-            get => GetOrCreateDictionary<DataKey<WebResource>, HashSet<DataKey<ResourceDictionary>>>(nameof(AggregatorNovels));
-            set => Set(nameof(AggregatorNovels), value);
-        }
-
-        public Dictionary<DataKey<WebResource>, WebResource> Resources {
-            get => GetOrCreateDictionary<DataKey<WebResource>, WebResource>(nameof(Resources));
-            set => Set(nameof(Resources), value);
-        }
-
-        public Dictionary<DataKey<ResourceDictionary>, ResourceDictionary> ResourceDictionaries {
-            get => GetOrCreateDictionary<DataKey<ResourceDictionary>, ResourceDictionary>(nameof(ResourceDictionaries));
-            set => Set(nameof(ResourceDictionaries), value);
-        }
-
-        public Dictionary<DataKey<ImmutableState>, ImmutableState> InitialStates {
-            get => GetOrCreateDictionary<DataKey<ImmutableState>, ImmutableState>(nameof(InitialStates));
-            set => Set(nameof(InitialStates), value);
-        }
-
-        internal Dictionary<DataKey<File>, File> Files {
-            get => GetOrCreateDictionary<DataKey<File>, File>(nameof(Files));
-            set => Set(nameof(Files), value);
-        }
-    }
-}

Beam.Models/ByteDocument.cs

@@ -0,0 +1,15 @@
+using System.Text;
+
+namespace Beam.Models {
+    public class ByteDocument(string filename, byte[] content, Encoding? encoding = null) : Document(filename, encoding) {
+        public byte[] Content { get; set; } = content;
+
+        public override byte[] ToBytes() {
+            return Content;
+        }
+
+        public override string ToString() {
+            return Encoding.GetString(Content);
+        }
+    }
+}

Beam.Models/Document.cs

@@ -0,0 +1,16 @@
+using System.Text;
+using aeqw89.DataKeys;
+using Beam.Abstractions;
+
+namespace Beam.Models {
+    public abstract class Document(string filename, Encoding? encoding = null) : IDocument {
+        public string Filename { get; set; } = filename;
+        public Encoding Encoding { get; set; } = encoding ?? Encoding.UTF8;
+        public Dictionary<DataKey<IDocumentMetaData>, IDocumentMetaData> MetaData { get; set; } = [];
+        IDictionary<IDataKey<IDocumentMetaData>, IDocumentMetaData> IDocument.MetaData 
+            => MetaData.ToDictionary(IDataKey<IDocumentMetaData> (x) => x.Key, x => x.Value);
+
+        public abstract byte[] ToBytes();
+        public abstract override string ToString();
+    }
+}

Beam.Models/DocumentCache.cs

@@ -0,0 +1,53 @@
+using Beam.Abstractions;
+
+namespace Beam.Models {
+    /// <summary>
+    /// Holds a collection of <see cref="IDocument"/> objects in memory to facilitate lazy loading
+    /// </summary>
+    public class DocumentCache : Dictionary<object, IDocument>, IDisposable {
+        private bool disposedValue;
+
+        /// <summary>
+        /// Calculates memory usage and checks if it does not exceed a certain limit
+        /// </summary>
+        /// <param name="allocatedSpaceInBytes">The memory limit</param>
+        /// <returns></returns>
+        public bool IsCapacityLessThan(int allocatedSpaceInBytes) {
+            return this.Count < CalculateMemorySpaceUsage(); 
+        }
+
+        /// <summary>
+        /// Gets an estimate of the space used by the IDocument objects (disregarding metadata) in bytes.
+        /// </summary>
+        /// <returns>Estimated memory usage in bytes</returns>
+        public long CalculateMemorySpaceUsage() {
+            return this.Select((x) => (x.Value.ToBytes().LongLength)).Aggregate((x, y) => x + y);
+        }
+
+        protected virtual void Dispose(bool disposing) {
+            if (!disposedValue) {
+                if (disposing) {
+                    // TODO: dispose managed state (managed objects)
+                    this.Clear();
+                }
+
+                // TODO: free unmanaged resources (unmanaged objects) and override finalizer
+                // TODO: set large fields to null
+                disposedValue = true;
+            }
+        }
+
+        // // TODO: override finalizer only if 'Dispose(bool disposing)' has code to free unmanaged resources
+        // ~DocumentCache()
+        // {
+        //     // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+        //     Dispose(disposing: false);
+        // }
+
+        public void Dispose() {
+            // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+            Dispose(disposing: true);
+            GC.SuppressFinalize(this);
+        }
+    }
+}

Beam.Models/DownloadReport.cs

@@ -0,0 +1,8 @@
+using Beam.Abstractions;
+
+namespace Beam.Models {
+    public struct DownloadReport : IDownloadReport {
+        // TODO implement download report
+    }
+
+}

Beam.Models/File.cs

@@ -1,5 +1,5 @@
 namespace Beam.Models {
-    internal class File(string path, params string[] tags) {
+    public class File(string path, params string[] tags) {
         public string Path { get; set; } = path;
         public string[] Tags { get; set; } = tags;
     }

Beam.Models/Fragment.cs

@@ -0,0 +1,37 @@
+using System.Collections.Concurrent;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Beam.Models {
+    public sealed class Fragment<T>(int size) {
+        public int Size => FragmentBag.Count;
+        public int MaxSize { get; } = size;
+        private ConcurrentBag<T> FragmentBag { get; set; } = new();
+        public bool TryTake([NotNullWhen(true)] out T? shard) {
+            return FragmentBag.TryTake(out shard) && shard != null;
+        }
+
+        private bool? Complete = false;
+        public bool IsComplete => Complete ?? Size == MaxSize;
+
+        private bool UpdaterLocked = false;
+
+        public static bool TryAcquireUpdater(Fragment<T> fragment, [NotNullWhen(true)] out Action<T>? updater) {
+            updater = null;
+            if (Interlocked.CompareExchange(ref fragment.UpdaterLocked, true, false) == true) 
+                // equivalent to : fragment.UpdaterLocked == true, side-effect: sets fragment.UpdaterLocked to true
+                return false;
+            updater = fragment.FragmentBag.Add;
+            return true;
+        }
+        public static bool TryReleaseUpdater(Fragment<T> fragment, Action<T> updater) {
+            if (updater == fragment.FragmentBag.Add) {
+                Interlocked.Exchange(ref fragment.UpdaterLocked, false);
+                return true;
+            }
+            return false;
+        }
+        public static void SetComplete(Fragment<T> fragment, bool status) {
+            fragment.Complete = status;
+        }
+    }
+}

Beam.Models/ImmutableState.cs

@@ -0,0 +1,26 @@
+using System.Text.Json.Serialization;
+
+namespace Beam.Models {
+    public readonly struct ImmutableState {
+        readonly string[] state;
+
+        [JsonConstructor]
+        public ImmutableState(string[] state) {
+            this.state = state ?? [];
+        }
+
+        public string[] State => state ?? [];
+
+        public readonly Span<string> AsSpan() => state ?? [];
+
+        public readonly State Copy()
+            => new((string[])(state ?? []).Clone());
+
+        public readonly string this[Index i] {
+            get => state[i];
+        }
+
+        public static implicit operator State(ImmutableState state)
+            => state.Copy();
+    }
+}

Beam.Models/LinkCollection.cs

@@ -1,13 +0,0 @@
-using aeqw89.DataKeys;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace Beam.Models {
-    internal class LinkCollection(DataKey<string> key, List<SourceLink> links) {
-        public DataKey<string> Key { get; set; } = key;
-        public List<SourceLink> Links { get; set; } = links;
-    }
-}

Beam.Models/Ordered.cs

@@ -0,0 +1,6 @@
+using Beam.Abstractions;
+
+namespace Beam.Models {
+
+    public record Ordered<T>(T Data, int Order) : IOrdered<T>;
+}

Beam.Models/Public Concrete/ArticleData.cs

@@ -0,0 +1,41 @@
+using System.Text.Json;
+using Beam.Abstractions;
+
+namespace Beam.Models {
+    public record class ArticleData : IArticleData {
+        public string? Name { get; set; }
+        public string[] Authors { get; set; } = [];
+        public string? Language { get; set; }
+        public string[] Categories { get; set; } = [];
+        public string? Version { get; set; }
+        public string? Description { get; set; }
+
+        public string AsJson(JsonSerializerOptions? options = null) {
+            return JsonSerializer.Serialize(this, options);
+        }
+
+        public virtual bool Equals(ArticleData? other) {
+            if (other is null) return false;
+            if (ReferenceEquals(this, other)) return true;
+            return Name == other.Name && Authors.Equals(other.Authors) && Language == other.Language && Categories.Equals(other.Categories) && Version == other.Version && Description == other.Description;
+        }
+
+        public virtual bool Equals(IArticleData? other) {
+            if (other is null) return false;
+            if (ReferenceEquals(this, other)) return true;
+            return Name == other.Name && Authors.Equals(other.Authors) && Language == other.Language && Categories.Equals(other.Categories) && Version == other.Version && Description == other.Description;
+        }
+
+        public override int GetHashCode() {
+            unchecked {
+                var hashCode = (Name != null ? Name.GetHashCode() : 0);
+                hashCode = (hashCode * 397) ^ Authors.GetHashCode();
+                hashCode = (hashCode * 397) ^ (Language != null ? Language.GetHashCode() : 0);
+                hashCode = (hashCode * 397) ^ Categories.GetHashCode();
+                hashCode = (hashCode * 397) ^ (Version != null ? Version.GetHashCode() : 0);
+                hashCode = (hashCode * 397) ^ (Description != null ? Description.GetHashCode() : 0);
+                return hashCode;
+            }
+        }
+    }
+}

Beam.Models/ResourceDictionary.cs

@@ -1,25 +0,0 @@
-
-
-using aeqw89.DataKeys;
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text;
-using System.Text.Json.Serialization;
-using System.Threading.Tasks;
-
-namespace Beam.Models {
-    public class ResourceDictionary : IKeyed<ResourceDictionary> {
-        public required DataKey<ResourceDictionary> Key { get; set; }
-
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public string? FriendlyName { get; set; }
-
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
-        public Dictionary<string, DataKey<WebResource>> Resources { get; set; } = [];
-
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
-        public Dictionary<DataKey<WebResource>, ImmutableState> InitialStates { get; set; } = [];
-    }
-}

Beam.Models/RetryReport.cs

@@ -0,0 +1,14 @@
+using Beam.Abstractions;
+
+namespace Beam.Models {
+    public readonly struct RetryReport  : IRetryReport {
+        public RetryReport(int tryNumber, string link) {
+            TryNumber = tryNumber;
+            Link = link;
+        }
+
+        public int TryNumber { get; }
+        public string Link { get; }
+    }
+
+}

Beam.Models/State.cs

@@ -0,0 +1,23 @@
+using Beam.Abstractions;
+
+namespace Beam.Models {
+    public class State(string[] state) : IState {
+        string[] state = state;
+
+        public string[] GetState() => state;
+        public void SetState(string[] state) => this.state = state;
+
+        public State Copy()
+            => new((string[])state.Clone());
+        
+        IReadOnlyState IReadOnlyState.Copy() 
+            => Copy();
+        IState IState.Copy() 
+            => Copy();
+
+        public string this[Index i] {
+            get => state[i];
+            set => state[i] = value;
+        } 
+    }
+}

Beam.Models/StreamDocument.cs

@@ -0,0 +1,18 @@
+using System.Text;
+
+namespace Beam.Models {
+    internal class StreamDocument(string filename, Stream content, Encoding? encoding = null) : Document(filename) {
+        public Stream Content { get; set; } = content;
+        public Encoding Encoding { get; set; } = encoding ?? Encoding.UTF8;
+
+        byte[] Content_ { get; set; } = [];
+
+        public override byte[] ToBytes() {
+            return Content_;
+        }
+
+        public override string ToString() {
+            return Encoding.GetString(Content_);
+        }
+    }
+}

Beam.Models/StringDocument.cs

@@ -0,0 +1,15 @@
+using System.Text;
+
+namespace Beam.Models {
+    public class StringDocument(string filename, string content, Encoding? encoding = null) : Document(filename, encoding) {
+        public string Content { get; set; } = content;
+
+        public override byte[] ToBytes() {
+            return Encoding.GetBytes(Content);
+        }
+
+        public override string ToString() {
+            return Content;
+        }
+    }
+}

Beam.Models/TableOfContentsData.cs

@@ -0,0 +1,14 @@
+using Beam.Models;
+
+namespace Beam.Models {
+    public record class TableOfContentsData : ArticleData {
+        /// <summary>
+        /// The link collection of the actual content
+        /// </summary>
+        public string[]? ContentLinks { get; set; }
+        /// <summary>
+        /// The link collection of all the Table Of Content pages for this specific resource.
+        /// </summary>
+        public string[]? PagesLinks { get; set; }
+    }
+}

Beam.Models/Template.cs

@@ -1,15 +0,0 @@
-using aeqw89.DataKeys;
-using Beam.Dynamic;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace Beam.Models {
-    public record class Template : IKeyed<Template> {
-        public required DataKey<Template> Key { get; set; }
-        public required StateChangerFactory Factory { get; set; }
-        public required SourceLinkBuilder Builder { get; set; }
-    }
-}

Beam.Models/Tracked.cs

@@ -1,11 +0,0 @@
-namespace Beam.Models {
-    internal class Tracked<T>(T obj) {
-        public T TrackedObject { get; set; } = obj;
-        public bool IsDirty { get; set; } = true;
-
-        public Tracked<T> SetDirty() {
-            IsDirty = true;
-            return this;
-        }
-    }
-}

Beam.Models/WebResource.cs

@@ -1,35 +0,0 @@
-using aeqw89.PersistentData;
-using aeqw89.DataKeys;
-using Beam.Dynamic;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace Beam.Models {
-    /// <summary>
-    /// Represents a specific resource accessible online; e.g. a book's contents.
-    /// </summary>
-    /// <remarks>
-    /// A resource should be one-to-one with a web request. 
-    /// </remarks>
-    /// <param name="key"></param>
-    public class WebResource(DataKey<WebResource> key) : IKeyed<WebResource> {
-        public DataKey<WebResource> Key { get; set; } = key;
-
-        public required DataKey<DataBindings> Bindings { get; set; }
-        public string? Name { get; set; }
-        public string? Domain { get; set; }
-        public string? Description { get; set; }
-
-
-        public WebResource() : this(new(string.Empty)) { }
-
-        //public Entity ToRecord(BeamDataDictionary sdd) {
-        //    return new Entity(this, sdd.Bindings[Bindings]);
-        //}
-
-        //public record class Entity(WebResource Resource, DataBindings Bindings);
-    }
-}