Initial commit

2026-06-17 07:19:25 +10:00
commit 363d894c79
26 changed files with 1805 additions and 0 deletions
@@ -0,0 +1,101 @@
+import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
+import { mkdtempSync, rmSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { Cache } from '../src/services/cache.ts';
+import type { TextFragment } from '../src/types.ts';
+
+let tmpDir: string;
+let cache: Cache;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), 'qa-cache-test-'));
+  cache = new Cache(tmpDir);
+});
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe('cache — chapters', () => {
+  test('returns null on a cold cache', () => {
+    expect(cache.tryGetChapters(1, 0, 5)).toBeNull();
+  });
+
+  test('returns null when a chapter in range is missing', () => {
+    cache.saveChapters(1, [
+      { index: 0, content: 'Chapter zero content here' },
+      // index 1 is intentionally missing
+      { index: 2, content: 'Chapter two content here' },
+    ]);
+    expect(cache.tryGetChapters(1, 0, 2)).toBeNull();
+  });
+
+  test('returns null for invalid range (min > max)', () => {
+    expect(cache.tryGetChapters(1, 5, 2)).toBeNull();
+  });
+
+  test('saves and retrieves plain-text chapters', () => {
+    cache.saveChapters(1, [
+      { index: 0, content: 'Line one\nLine two' },
+      { index: 1, content: 'Chapter one content here' },
+    ]);
+    const view = cache.tryGetChapters(1, 0, 1);
+    expect(view).not.toBeNull();
+    expect(view!.id).toBe(1);
+    expect(view!.min).toBe(0);
+    expect(view!.max).toBe(1);
+    expect(view!.chapters).toHaveLength(2);
+    expect(view!.chapters[0]!.hasStyle).toBe(false);
+  });
+
+  test('detects styled (JSON fragment) chapters', () => {
+    const fragments: TextFragment[] = [{ body: 'Hello world', style: undefined }];
+    cache.saveChapters(1, [{ index: 0, content: JSON.stringify(fragments) }]);
+    const view = cache.tryGetChapters(1, 0, 0);
+    expect(view).not.toBeNull();
+    expect(view!.chapters[0]!.hasStyle).toBe(true);
+  });
+
+  test('returns null when a chapter file is too short', () => {
+    cache.saveChapters(1, [{ index: 0, content: 'hi' }]); // < 10 bytes
+    expect(cache.tryGetChapters(1, 0, 0)).toBeNull();
+  });
+
+  test('handles single-chapter range', () => {
+    cache.saveChapters(42, [{ index: 7, content: 'Single chapter content ok' }]);
+    const view = cache.tryGetChapters(42, 7, 7);
+    expect(view).not.toBeNull();
+    expect(view!.chapters).toHaveLength(1);
+  });
+});
+
+describe('cache — TOC', () => {
+  test('returns null on a cold cache', () => {
+    expect(cache.tryGetToc(1)).toBeNull();
+  });
+
+  test('round-trips a TOC', () => {
+    const urls = [
+      'https://example.com/chapter-1',
+      'https://example.com/chapter-2',
+      'https://example.com/chapter-3',
+    ];
+    cache.saveToc(1, urls);
+    expect(cache.tryGetToc(1)).toEqual(urls);
+  });
+
+  test('returns null for an empty TOC file', () => {
+    cache.saveToc(1, []);
+    expect(cache.tryGetToc(1)).toBeNull();
+  });
+
+  test('overwrites an existing TOC on re-save', () => {
+    cache.saveToc(1, ['https://example.com/old']);
+    cache.saveToc(1, ['https://example.com/new-1', 'https://example.com/new-2']);
+    expect(cache.tryGetToc(1)).toEqual([
+      'https://example.com/new-1',
+      'https://example.com/new-2',
+    ]);
+  });
+});
@@ -0,0 +1,143 @@
+import { describe, test, expect, beforeEach, afterEach, spyOn } from 'bun:test';
+import { mkdtempSync, rmSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { zipSync } from 'fflate';
+import { Cache } from '../src/services/cache.ts';
+import { fetchEpub } from '../src/services/epub.ts';
+
+const enc = new TextEncoder();
+
+function buildEpub(chapters: { title: string; content: string }[]): Uint8Array {
+  const files: Record<string, Uint8Array> = {};
+
+  files['META-INF/container.xml'] = enc.encode(
+    `<?xml version="1.0"?>
+<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
+  <rootfiles>
+    <rootfile full-path="content.opf" media-type="application/oebps-package+xml"/>
+  </rootfiles>
+</container>`,
+  );
+
+  const manifestItems = chapters
+    .map((_, i) => `<item id="ch${i}" href="ch${i}.xhtml" media-type="application/xhtml+xml"/>`)
+    .join('\n    ');
+  const spineItems = chapters.map((_, i) => `<itemref idref="ch${i}"/>`).join('\n    ');
+
+  files['content.opf'] = enc.encode(
+    `<?xml version="1.0"?>
+<package xmlns="http://www.idpf.org/2007/opf" version="3.0">
+  <manifest>
+    ${manifestItems}
+  </manifest>
+  <spine>
+    ${spineItems}
+  </spine>
+</package>`,
+  );
+
+  for (let i = 0; i < chapters.length; i++) {
+    const ch = chapters[i]!;
+    files[`ch${i}.xhtml`] = enc.encode(
+      `<html><body><div>${ch.content}</div></body></html>`,
+    );
+  }
+
+  return zipSync(files);
+}
+
+let tmpDir: string;
+let cache: Cache;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), 'qa-epub-test-'));
+  cache = new Cache(tmpDir);
+});
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function mockFetch(data: Uint8Array) {
+  return spyOn(globalThis, 'fetch').mockResolvedValue(
+    new Response(data, { status: 200 }),
+  );
+}
+
+describe('epub — parsing', () => {
+  test('extracts the correct number of chapters', async () => {
+    const epub = buildEpub([
+      { title: 'Ch 1', content: 'First chapter content here.' },
+      { title: 'Ch 2', content: 'Second chapter content here.' },
+      { title: 'Ch 3', content: 'Third chapter content here.' },
+    ]);
+    const spy = mockFetch(epub);
+
+    let rangeStart = -1, rangeEnd = -1;
+    const view = await fetchEpub('http://fake/book.epub', 1, 0, 2, cache, (s, e) => {
+      rangeStart = s; rangeEnd = e;
+    });
+
+    expect(view.chapters).toHaveLength(3);
+    expect(rangeStart).toBe(0);
+    expect(rangeEnd).toBe(2);
+    spy.mockRestore();
+  });
+
+  test('serves from cache on second call without re-fetching', async () => {
+    const epub = buildEpub([
+      { title: 'Ch 1', content: 'Chapter one long enough content.' },
+    ]);
+    const spy = mockFetch(epub);
+
+    await fetchEpub('http://fake/book.epub', 2, 0, 0, cache, () => {});
+    expect(spy).toHaveBeenCalledTimes(1);
+
+    // Second call — should hit cache
+    await fetchEpub('http://fake/book.epub', 2, 0, 0, cache, () => {});
+    expect(spy).toHaveBeenCalledTimes(1);
+
+    spy.mockRestore();
+  });
+
+  test('skips blank chapters', async () => {
+    const epub = buildEpub([
+      { title: 'Cover', content: '   ' },       // blank — should be skipped
+      { title: 'Ch 1', content: 'Real content here for this chapter.' },
+      { title: 'Ch 2', content: 'More real content for the next chapter.' },
+    ]);
+    const spy = mockFetch(epub);
+
+    let rangeEnd = -1;
+    await fetchEpub('http://fake/book.epub', 3, 0, 10, cache, (_, e) => { rangeEnd = e; });
+    expect(rangeEnd).toBe(1); // only 2 non-blank chapters (indices 0 and 1)
+    spy.mockRestore();
+  });
+
+  test('throws on HTTP error', async () => {
+    const spy = spyOn(globalThis, 'fetch').mockResolvedValue(
+      new Response('Not Found', { status: 404 }),
+    );
+    await expect(fetchEpub('http://fake/missing.epub', 4, 0, 0, cache, () => {})).rejects.toThrow(
+      'HTTP 404',
+    );
+    spy.mockRestore();
+  });
+
+  test('slices to the requested range', async () => {
+    const epub = buildEpub([
+      { title: 'Ch 0', content: 'Chapter zero has enough content here.' },
+      { title: 'Ch 1', content: 'Chapter one has enough content here.' },
+      { title: 'Ch 2', content: 'Chapter two has enough content here.' },
+      { title: 'Ch 3', content: 'Chapter three has enough content here.' },
+    ]);
+    const spy = mockFetch(epub);
+
+    const view = await fetchEpub('http://fake/book.epub', 5, 1, 2, cache, () => {});
+    expect(view.min).toBe(1);
+    expect(view.max).toBe(2);
+    expect(view.chapters).toHaveLength(2);
+    spy.mockRestore();
+  });
+});
@@ -0,0 +1,175 @@
+import { describe, test, expect, beforeEach } from 'bun:test';
+import { Hono } from 'hono';
+import { ResourceRegistry } from '../src/services/registry.ts';
+import { createNovelRouter } from '../src/routes/novel.ts';
+import type { ChapterView } from '../src/types.ts';
+import type { AppConfig } from '../src/config/types.ts';
+
+// Cache is not exercised by route tests — pass a stub that satisfies the type
+const stubCache = {} as import('../src/services/cache.ts').Cache;
+
+const stubAppConfig: AppConfig = {
+  server: { port: 8080 },
+  selenium: { remoteUrl: 'http://selenium:4444/wd/hub', browser: 'firefox' },
+  cache: { basePath: '/tmp' },
+  addons: { directory: './addons' },
+};
+
+function makeView(id: number, min: number, max: number): ChapterView {
+  return { id, min, max, chapters: [] };
+}
+
+let registry: ResourceRegistry;
+let app: Hono;
+
+beforeEach(() => {
+  registry = new ResourceRegistry();
+  const router = createNovelRouter(registry, stubCache, stubAppConfig);
+  app = new Hono();
+  app.route('/qa/requests/novel', router);
+});
+
+describe('GET /qa/requests/novel/available', () => {
+  test('returns empty object when nothing registered', async () => {
+    const res = await app.request('/qa/requests/novel/available');
+    expect(res.status).toBe(200);
+    const body = await res.json();
+    expect(body).toEqual({});
+  });
+
+  test('returns registered resources (without fetch fn)', async () => {
+    registry.register({
+      hash: 'abc123',
+      slug: 'my-novel',
+      startsAt: 0,
+      endsAt: 100,
+      fetch: async () => makeView(0, 0, 0),
+    });
+    const res = await app.request('/qa/requests/novel/available');
+    const body = await res.json() as Record<string, unknown>;
+    expect(Object.keys(body)).toHaveLength(1);
+    expect((body['0'] as Record<string, unknown>)['slug']).toBe('my-novel');
+    expect((body['0'] as Record<string, unknown>)['fetch']).toBeUndefined();
+  });
+});
+
+describe('GET /qa/requests/novel/:novel', () => {
+  beforeEach(() => {
+    registry.register({
+      hash: 'deadbeef',
+      slug: 'test-novel',
+      startsAt: 0,
+      endsAt: 9,
+      fetch: async (from, to) => makeView(0, from, to),
+    });
+  });
+
+  test('returns 400 when r param is missing', async () => {
+    const res = await app.request('/qa/requests/novel/0');
+    expect(res.status).toBe(400);
+    const body = await res.json() as { error: string };
+    expect(body.error).toContain('Missing');
+  });
+
+  test('returns 400 for malformed r param', async () => {
+    const res = await app.request('/qa/requests/novel/0?r=notanumber');
+    expect(res.status).toBe(400);
+  });
+
+  test('returns 400 when min > max', async () => {
+    const res = await app.request('/qa/requests/novel/0?r=10,5');
+    expect(res.status).toBe(400);
+  });
+
+  test('returns 422 for unknown novel id', async () => {
+    const res = await app.request('/qa/requests/novel/999?r=0,5');
+    expect(res.status).toBe(422);
+    const body = await res.json() as { error: string };
+    expect(body.error).toContain('Unknown');
+  });
+
+  test('returns 400 for non-integer novel id', async () => {
+    const res = await app.request('/qa/requests/novel/abc?r=0,5');
+    expect(res.status).toBe(400);
+  });
+
+  test('returns 200 html by default', async () => {
+    const res = await app.request('/qa/requests/novel/0?r=0,0');
+    // fetch returns empty chapters so HTML template would be rendered (template read from disk)
+    // We just verify it doesn't 5xx from the route layer itself
+    expect([200, 500]).toContain(res.status);
+  });
+
+  test('returns 200 json when f=json', async () => {
+    const res = await app.request('/qa/requests/novel/0?r=0,0&f=json');
+    expect([200, 500]).toContain(res.status);
+    if (res.status === 200) {
+      expect(res.headers.get('content-type')).toContain('application/json');
+    }
+  });
+
+  test('returns 500 when fetch throws', async () => {
+    registry.register({
+      hash: 'badfetch',
+      slug: 'broken-novel',
+      startsAt: 0,
+      endsAt: 0,
+      fetch: async () => { throw new Error('something went wrong internally'); },
+    });
+    const res = await app.request('/qa/requests/novel/1?r=0,0');
+    expect(res.status).toBe(500);
+    const body = await res.json() as { error: string; detail: string };
+    expect(body.error).toBeDefined();
+    expect(body.detail).toBe('something went wrong internally');
+  });
+});
+
+describe('POST /qa/requests/novel/new', () => {
+  test('returns 400 for non-JSON body', async () => {
+    const res = await app.request('/qa/requests/novel/new', {
+      method: 'POST',
+      body: 'not json',
+      headers: { 'Content-Type': 'text/plain' },
+    });
+    expect(res.status).toBe(400);
+  });
+
+  test('returns 400 when type is missing', async () => {
+    const res = await app.request('/qa/requests/novel/new', {
+      method: 'POST',
+      body: JSON.stringify({ slug: 'test', urls: ['http://x.com/book.epub'] }),
+      headers: { 'Content-Type': 'application/json' },
+    });
+    expect(res.status).toBe(400);
+  });
+
+  test('returns 400 for unsupported type', async () => {
+    const res = await app.request('/qa/requests/novel/new', {
+      method: 'POST',
+      body: JSON.stringify({ type: 'novel', slug: 'test', urls: ['http://x.com'] }),
+      headers: { 'Content-Type': 'application/json' },
+    });
+    expect(res.status).toBe(400);
+    const body = await res.json() as { error: string };
+    expect(body.error).toContain('Unsupported');
+  });
+
+  test('registers an epub and returns numeric id', async () => {
+    const res = await app.request('/qa/requests/novel/new', {
+      method: 'POST',
+      body: JSON.stringify({ type: 'epub', slug: 'my-book', urls: ['http://x.com/book.epub'] }),
+      headers: { 'Content-Type': 'application/json' },
+    });
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    expect(parseInt(text, 10)).toBeGreaterThanOrEqual(0);
+  });
+
+  test('returns 409 on duplicate registration', async () => {
+    const body = JSON.stringify({ type: 'epub', slug: 'dup', urls: ['http://x.com/dup.epub'] });
+    const headers = { 'Content-Type': 'application/json' };
+    await app.request('/qa/requests/novel/new', { method: 'POST', body, headers });
+    const res = await app.request('/qa/requests/novel/new', { method: 'POST', body, headers });
+    expect(res.status).toBe(409);
+  });
+});
@@ -0,0 +1,174 @@
+import { describe, test, expect, beforeEach, afterEach, mock, spyOn } from 'bun:test';
+import { mkdtempSync, rmSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { Cache } from '../src/services/cache.ts';
+import { fetchToc, fetchChapters, type DriverFactory } from '../src/services/scraper.ts';
+import type { NovelDefinition, AppConfig } from '../src/config/types.ts';
+
+const DEF: NovelDefinition = {
+  slug: 'test-novel',
+  type: 'web-scraped',
+  source: { url: 'https://example.com/novel' },
+  selectors: {
+    toc: { scope: '#chapters', linkPattern: 'https://example\\.com/chapter/.*' },
+    content: { scope: '#content' },
+  },
+  stealth: { enabled: true, waitMs: 0, addons: [] },
+  download: { parallelism: 2, minContentBytes: 5 },
+};
+
+const APP_CONFIG: AppConfig = {
+  server: { port: 8080 },
+  selenium: { remoteUrl: 'http://selenium:4444/wd/hub', browser: 'firefox' },
+  cache: { basePath: '/tmp' },
+  addons: { directory: './addons' },
+};
+
+let tmpDir: string;
+let cache: Cache;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), 'qa-scraper-test-'));
+  cache = new Cache(tmpDir);
+});
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function buildMockPage(links: string[]): string {
+  const anchors = links.map(href => `<a href="${href}">link</a>`).join('');
+  return `<html><body><div id="chapters">${anchors}</div></body></html>`;
+}
+
+function makeMockDriver(html: string): { driver: ReturnType<DriverFactory> extends Promise<infer T> ? T : never; driverFactory: DriverFactory; quitSpy: ReturnType<typeof mock> } {
+  const quitSpy = mock(() => Promise.resolve());
+  const driver = {
+    get: mock((_url: string) => Promise.resolve()),
+    sleep: mock((_ms: number) => Promise.resolve()),
+    executeScript: mock((_script: string) => Promise.resolve(html)),
+    quit: quitSpy,
+  } as unknown as Awaited<ReturnType<DriverFactory>>;
+  const driverFactory: DriverFactory = () => Promise.resolve(driver);
+  return { driver, driverFactory, quitSpy };
+}
+
+describe('fetchToc', () => {
+  test('returns cached TOC without opening a browser', async () => {
+    const urls = ['https://example.com/chapter/1', 'https://example.com/chapter/2'];
+    cache.saveToc(0, urls);
+
+    const driverFactory = mock(() => Promise.reject(new Error('should not be called')));
+    const result = await fetchToc(DEF, 0, cache, APP_CONFIG, driverFactory as unknown as DriverFactory);
+    expect(result).toEqual(urls);
+    expect(driverFactory).not.toHaveBeenCalled();
+  });
+
+  test('fetches TOC via browser when cache is cold', async () => {
+    const links = ['https://example.com/chapter/1', 'https://example.com/chapter/2'];
+    const { driverFactory, quitSpy } = makeMockDriver(buildMockPage(links));
+
+    const result = await fetchToc(DEF, 1, cache, APP_CONFIG, driverFactory);
+    expect(result).toEqual(links);
+    expect(quitSpy).toHaveBeenCalledTimes(1);
+  });
+
+  test('filters links by linkPattern', async () => {
+    const html = `<html><body><div id="chapters">
+      <a href="https://example.com/chapter/1">ch1</a>
+      <a href="https://other.com/chapter/2">other</a>
+      <a href="https://example.com/chapter/3">ch3</a>
+    </div></body></html>`;
+    const { driverFactory } = makeMockDriver(html);
+
+    const result = await fetchToc(DEF, 2, cache, APP_CONFIG, driverFactory);
+    expect(result).toEqual(['https://example.com/chapter/1', 'https://example.com/chapter/3']);
+  });
+
+  // Extended timeout: pRetry's default exponential backoff across 4 attempts can take ~7s
+  test('quits driver even on failure', async () => {
+    const { driverFactory, quitSpy } = makeMockDriver('<html><body></body></html>');
+    await expect(fetchToc(DEF, 3, cache, APP_CONFIG, driverFactory)).rejects.toThrow();
+    expect(quitSpy).toHaveBeenCalledTimes(1);
+  }, 15000);
+
+  test('finds anchors nested at arbitrary depth within the scope element', async () => {
+    // Simulates a TOC where links are buried inside ul > li > span > a
+    // rather than being direct children of the scope container.
+    const html = `<html><body><div id="chapters">
+      <ul>
+        <li><span><a href="https://example.com/chapter/1">deep link 1</a></span></li>
+        <li>
+          <div class="wrapper">
+            <div class="inner">
+              <a href="https://example.com/chapter/2">deep link 2</a>
+            </div>
+          </div>
+        </li>
+      </ul>
+    </div></body></html>`;
+    const { driverFactory } = makeMockDriver(html);
+    const result = await fetchToc(DEF, 5, cache, APP_CONFIG, driverFactory);
+    expect(result).toEqual([
+      'https://example.com/chapter/1',
+      'https://example.com/chapter/2',
+    ]);
+  });
+
+  test('saves fetched TOC to cache', async () => {
+    const links = ['https://example.com/chapter/1'];
+    const { driverFactory } = makeMockDriver(buildMockPage(links));
+    await fetchToc(DEF, 4, cache, APP_CONFIG, driverFactory);
+    expect(cache.tryGetToc(4)).toEqual(links);
+  });
+});
+
+describe('fetchChapters', () => {
+  test('returns cached view without fetching', async () => {
+    const content = JSON.stringify([{ body: 'Chapter content here.' }]);
+    cache.saveChapters(10, [{ index: 0, content }]);
+
+    const fetchSpy = spyOn(globalThis, 'fetch');
+    const view = await fetchChapters(DEF, 10, 0, ['https://example.com/chapter/1'], cache);
+    expect(view.chapters).toHaveLength(1);
+    expect(fetchSpy).not.toHaveBeenCalled();
+    fetchSpy.mockRestore();
+  });
+
+  test('downloads chapters and saves them', async () => {
+    // mockImplementation (not mockResolvedValue) so each call gets a fresh Response —
+    // reusing the same instance causes "Body already used" on retry reads.
+    const spy = spyOn(globalThis, 'fetch').mockImplementation(() =>
+      Promise.resolve(new Response('<html><body><div id="content">Chapter body text here.</div></body></html>', { status: 200 })),
+    );
+
+    const view = await fetchChapters(
+      DEF,
+      11,
+      0,
+      ['https://example.com/chapter/1', 'https://example.com/chapter/2'],
+      cache,
+    );
+
+    expect(view.min).toBe(0);
+    expect(view.max).toBe(1);
+    expect(view.chapters).toHaveLength(2);
+    expect(spy).toHaveBeenCalledTimes(2);
+    spy.mockRestore();
+  });
+
+  test('stores error fragment when chapter content is too short', async () => {
+    const spy = spyOn(globalThis, 'fetch').mockImplementation(() =>
+      Promise.resolve(new Response('<html><body><div id="content"></div></body></html>', { status: 200 })),
+    );
+
+    const view = await fetchChapters(DEF, 12, 0, ['https://example.com/chapter/1'], cache);
+    expect(view.chapters).toHaveLength(1);
+
+    // The saved content should be the error fragment JSON
+    const saved = cache.tryGetChapters(12, 0, 0);
+    expect(saved).not.toBeNull();
+    spy.mockRestore();
+  });
+});