Initial commit

2026-06-17 07:19:25 +10:00
commit 363d894c79
26 changed files with 1805 additions and 0 deletions
@@ -0,0 +1,174 @@
+import { describe, test, expect, beforeEach, afterEach, mock, spyOn } from 'bun:test';
+import { mkdtempSync, rmSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { Cache } from '../src/services/cache.ts';
+import { fetchToc, fetchChapters, type DriverFactory } from '../src/services/scraper.ts';
+import type { NovelDefinition, AppConfig } from '../src/config/types.ts';
+
+const DEF: NovelDefinition = {
+  slug: 'test-novel',
+  type: 'web-scraped',
+  source: { url: 'https://example.com/novel' },
+  selectors: {
+    toc: { scope: '#chapters', linkPattern: 'https://example\\.com/chapter/.*' },
+    content: { scope: '#content' },
+  },
+  stealth: { enabled: true, waitMs: 0, addons: [] },
+  download: { parallelism: 2, minContentBytes: 5 },
+};
+
+const APP_CONFIG: AppConfig = {
+  server: { port: 8080 },
+  selenium: { remoteUrl: 'http://selenium:4444/wd/hub', browser: 'firefox' },
+  cache: { basePath: '/tmp' },
+  addons: { directory: './addons' },
+};
+
+let tmpDir: string;
+let cache: Cache;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), 'qa-scraper-test-'));
+  cache = new Cache(tmpDir);
+});
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function buildMockPage(links: string[]): string {
+  const anchors = links.map(href => `<a href="${href}">link</a>`).join('');
+  return `<html><body><div id="chapters">${anchors}</div></body></html>`;
+}
+
+function makeMockDriver(html: string): { driver: ReturnType<DriverFactory> extends Promise<infer T> ? T : never; driverFactory: DriverFactory; quitSpy: ReturnType<typeof mock> } {
+  const quitSpy = mock(() => Promise.resolve());
+  const driver = {
+    get: mock((_url: string) => Promise.resolve()),
+    sleep: mock((_ms: number) => Promise.resolve()),
+    executeScript: mock((_script: string) => Promise.resolve(html)),
+    quit: quitSpy,
+  } as unknown as Awaited<ReturnType<DriverFactory>>;
+  const driverFactory: DriverFactory = () => Promise.resolve(driver);
+  return { driver, driverFactory, quitSpy };
+}
+
+describe('fetchToc', () => {
+  test('returns cached TOC without opening a browser', async () => {
+    const urls = ['https://example.com/chapter/1', 'https://example.com/chapter/2'];
+    cache.saveToc(0, urls);
+
+    const driverFactory = mock(() => Promise.reject(new Error('should not be called')));
+    const result = await fetchToc(DEF, 0, cache, APP_CONFIG, driverFactory as unknown as DriverFactory);
+    expect(result).toEqual(urls);
+    expect(driverFactory).not.toHaveBeenCalled();
+  });
+
+  test('fetches TOC via browser when cache is cold', async () => {
+    const links = ['https://example.com/chapter/1', 'https://example.com/chapter/2'];
+    const { driverFactory, quitSpy } = makeMockDriver(buildMockPage(links));
+
+    const result = await fetchToc(DEF, 1, cache, APP_CONFIG, driverFactory);
+    expect(result).toEqual(links);
+    expect(quitSpy).toHaveBeenCalledTimes(1);
+  });
+
+  test('filters links by linkPattern', async () => {
+    const html = `<html><body><div id="chapters">
+      <a href="https://example.com/chapter/1">ch1</a>
+      <a href="https://other.com/chapter/2">other</a>
+      <a href="https://example.com/chapter/3">ch3</a>
+    </div></body></html>`;
+    const { driverFactory } = makeMockDriver(html);
+
+    const result = await fetchToc(DEF, 2, cache, APP_CONFIG, driverFactory);
+    expect(result).toEqual(['https://example.com/chapter/1', 'https://example.com/chapter/3']);
+  });
+
+  // Extended timeout: pRetry's default exponential backoff across 4 attempts can take ~7s
+  test('quits driver even on failure', async () => {
+    const { driverFactory, quitSpy } = makeMockDriver('<html><body></body></html>');
+    await expect(fetchToc(DEF, 3, cache, APP_CONFIG, driverFactory)).rejects.toThrow();
+    expect(quitSpy).toHaveBeenCalledTimes(1);
+  }, 15000);
+
+  test('finds anchors nested at arbitrary depth within the scope element', async () => {
+    // Simulates a TOC where links are buried inside ul > li > span > a
+    // rather than being direct children of the scope container.
+    const html = `<html><body><div id="chapters">
+      <ul>
+        <li><span><a href="https://example.com/chapter/1">deep link 1</a></span></li>
+        <li>
+          <div class="wrapper">
+            <div class="inner">
+              <a href="https://example.com/chapter/2">deep link 2</a>
+            </div>
+          </div>
+        </li>
+      </ul>
+    </div></body></html>`;
+    const { driverFactory } = makeMockDriver(html);
+    const result = await fetchToc(DEF, 5, cache, APP_CONFIG, driverFactory);
+    expect(result).toEqual([
+      'https://example.com/chapter/1',
+      'https://example.com/chapter/2',
+    ]);
+  });
+
+  test('saves fetched TOC to cache', async () => {
+    const links = ['https://example.com/chapter/1'];
+    const { driverFactory } = makeMockDriver(buildMockPage(links));
+    await fetchToc(DEF, 4, cache, APP_CONFIG, driverFactory);
+    expect(cache.tryGetToc(4)).toEqual(links);
+  });
+});
+
+describe('fetchChapters', () => {
+  test('returns cached view without fetching', async () => {
+    const content = JSON.stringify([{ body: 'Chapter content here.' }]);
+    cache.saveChapters(10, [{ index: 0, content }]);
+
+    const fetchSpy = spyOn(globalThis, 'fetch');
+    const view = await fetchChapters(DEF, 10, 0, ['https://example.com/chapter/1'], cache);
+    expect(view.chapters).toHaveLength(1);
+    expect(fetchSpy).not.toHaveBeenCalled();
+    fetchSpy.mockRestore();
+  });
+
+  test('downloads chapters and saves them', async () => {
+    // mockImplementation (not mockResolvedValue) so each call gets a fresh Response —
+    // reusing the same instance causes "Body already used" on retry reads.
+    const spy = spyOn(globalThis, 'fetch').mockImplementation(() =>
+      Promise.resolve(new Response('<html><body><div id="content">Chapter body text here.</div></body></html>', { status: 200 })),
+    );
+
+    const view = await fetchChapters(
+      DEF,
+      11,
+      0,
+      ['https://example.com/chapter/1', 'https://example.com/chapter/2'],
+      cache,
+    );
+
+    expect(view.min).toBe(0);
+    expect(view.max).toBe(1);
+    expect(view.chapters).toHaveLength(2);
+    expect(spy).toHaveBeenCalledTimes(2);
+    spy.mockRestore();
+  });
+
+  test('stores error fragment when chapter content is too short', async () => {
+    const spy = spyOn(globalThis, 'fetch').mockImplementation(() =>
+      Promise.resolve(new Response('<html><body><div id="content"></div></body></html>', { status: 200 })),
+    );
+
+    const view = await fetchChapters(DEF, 12, 0, ['https://example.com/chapter/1'], cache);
+    expect(view.chapters).toHaveLength(1);
+
+    // The saved content should be the error fragment JSON
+    const saved = cache.tryGetChapters(12, 0, 0);
+    expect(saved).not.toBeNull();
+    spy.mockRestore();
+  });
+});