import { describe, test, expect, beforeEach, afterEach, mock, spyOn } from 'bun:test';
import { mkdtempSync, rmSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { Cache } from '../src/services/cache.ts';
import { fetchToc, fetchChapters, type DriverFactory } from '../src/services/scraper.ts';
import type { NovelDefinition, AppConfig } from '../src/config/types.ts';
const DEF: NovelDefinition = {
slug: 'test-novel',
type: 'web-scraped',
source: { url: 'https://example.com/novel' },
selectors: {
toc: { scope: '#chapters', linkPattern: 'https://example\\.com/chapter/.*' },
content: { scope: '#content' },
},
stealth: { enabled: true, waitMs: 0, addons: [] },
download: { parallelism: 2, minContentBytes: 5 },
};
const APP_CONFIG: AppConfig = {
server: { port: 8080 },
selenium: { remoteUrl: 'http://selenium:4444/wd/hub', browser: 'firefox' },
cache: { basePath: '/tmp' },
addons: { directory: './addons' },
};
let tmpDir: string;
let cache: Cache;
beforeEach(() => {
tmpDir = mkdtempSync(join(tmpdir(), 'qa-scraper-test-'));
cache = new Cache(tmpDir);
});
afterEach(() => {
rmSync(tmpDir, { recursive: true, force: true });
});
function buildMockPage(links: string[]): string {
const anchors = links.map(href => `link`).join('');
return `${anchors}
`;
}
function makeMockDriver(html: string): { driver: ReturnType extends Promise ? T : never; driverFactory: DriverFactory; quitSpy: ReturnType } {
const quitSpy = mock(() => Promise.resolve());
const driver = {
get: mock((_url: string) => Promise.resolve()),
sleep: mock((_ms: number) => Promise.resolve()),
executeScript: mock((_script: string) => Promise.resolve(html)),
quit: quitSpy,
} as unknown as Awaited>;
const driverFactory: DriverFactory = () => Promise.resolve(driver);
return { driver, driverFactory, quitSpy };
}
describe('fetchToc', () => {
test('returns cached TOC without opening a browser', async () => {
const urls = ['https://example.com/chapter/1', 'https://example.com/chapter/2'];
cache.saveToc(0, urls);
const driverFactory = mock(() => Promise.reject(new Error('should not be called')));
const result = await fetchToc(DEF, 0, cache, APP_CONFIG, driverFactory as unknown as DriverFactory);
expect(result).toEqual(urls);
expect(driverFactory).not.toHaveBeenCalled();
});
test('fetches TOC via browser when cache is cold', async () => {
const links = ['https://example.com/chapter/1', 'https://example.com/chapter/2'];
const { driverFactory, quitSpy } = makeMockDriver(buildMockPage(links));
const result = await fetchToc(DEF, 1, cache, APP_CONFIG, driverFactory);
expect(result).toEqual(links);
expect(quitSpy).toHaveBeenCalledTimes(1);
});
test('filters links by linkPattern', async () => {
const html = ``;
const { driverFactory } = makeMockDriver(html);
const result = await fetchToc(DEF, 2, cache, APP_CONFIG, driverFactory);
expect(result).toEqual(['https://example.com/chapter/1', 'https://example.com/chapter/3']);
});
// Extended timeout: pRetry's default exponential backoff across 4 attempts can take ~7s
test('quits driver even on failure', async () => {
const { driverFactory, quitSpy } = makeMockDriver('');
await expect(fetchToc(DEF, 3, cache, APP_CONFIG, driverFactory)).rejects.toThrow();
expect(quitSpy).toHaveBeenCalledTimes(1);
}, 15000);
test('finds anchors nested at arbitrary depth within the scope element', async () => {
// Simulates a TOC where links are buried inside ul > li > span > a
// rather than being direct children of the scope container.
const html = ``;
const { driverFactory } = makeMockDriver(html);
const result = await fetchToc(DEF, 5, cache, APP_CONFIG, driverFactory);
expect(result).toEqual([
'https://example.com/chapter/1',
'https://example.com/chapter/2',
]);
});
test('saves fetched TOC to cache', async () => {
const links = ['https://example.com/chapter/1'];
const { driverFactory } = makeMockDriver(buildMockPage(links));
await fetchToc(DEF, 4, cache, APP_CONFIG, driverFactory);
expect(cache.tryGetToc(4)).toEqual(links);
});
});
describe('fetchChapters', () => {
test('returns cached view without fetching', async () => {
const content = JSON.stringify([{ body: 'Chapter content here.' }]);
cache.saveChapters(10, [{ index: 0, content }]);
const fetchSpy = spyOn(globalThis, 'fetch');
const view = await fetchChapters(DEF, 10, 0, ['https://example.com/chapter/1'], cache);
expect(view.chapters).toHaveLength(1);
expect(fetchSpy).not.toHaveBeenCalled();
fetchSpy.mockRestore();
});
test('downloads chapters and saves them', async () => {
// mockImplementation (not mockResolvedValue) so each call gets a fresh Response —
// reusing the same instance causes "Body already used" on retry reads.
const spy = spyOn(globalThis, 'fetch').mockImplementation(() =>
Promise.resolve(new Response('Chapter body text here.
', { status: 200 })),
);
const view = await fetchChapters(
DEF,
11,
0,
['https://example.com/chapter/1', 'https://example.com/chapter/2'],
cache,
);
expect(view.min).toBe(0);
expect(view.max).toBe(1);
expect(view.chapters).toHaveLength(2);
expect(spy).toHaveBeenCalledTimes(2);
spy.mockRestore();
});
test('stores error fragment when chapter content is too short', async () => {
const spy = spyOn(globalThis, 'fetch').mockImplementation(() =>
Promise.resolve(new Response('', { status: 200 })),
);
const view = await fetchChapters(DEF, 12, 0, ['https://example.com/chapter/1'], cache);
expect(view.chapters).toHaveLength(1);
// The saved content should be the error fragment JSON
const saved = cache.tryGetChapters(12, 0, 0);
expect(saved).not.toBeNull();
spy.mockRestore();
});
});