Fixed error message logging, added detailed diagnostics.

src/services/analyzer.ts

@@ -0,0 +1,212 @@
+import * as cheerio from 'cheerio';
+
+export interface Finding {
+  check: string;
+  passed: boolean;
+  detail: string;
+}
+
+export interface PageDiagnosis {
+  pageTitle: string;
+  bodyBytes: number;
+  actualUrl?: string;
+  findings: Finding[];
+  likelyCause: string;
+}
+
+function pass(check: string, detail: string): Finding {
+  return { check, passed: true, detail };
+}
+
+function fail(check: string, detail: string): Finding {
+  return { check, passed: false, detail };
+}
+
+export function analyzeTocPage(
+  html: string,
+  scope: string,
+  linkPattern: string,
+  actualUrl?: string,
+): PageDiagnosis {
+  const $ = cheerio.load(html);
+  const pageTitle = $('title').text().trim();
+  const bodyText = $('body').text().toLowerCase();
+  const bodyBytes = html.length;
+  const findings: Finding[] = [];
+
+  // ── 1. Page loaded at all ────────────────────────────────────────────────────
+  findings.push(
+    bodyBytes < 500
+      ? fail('Page content', `Only ${bodyBytes} bytes — page likely failed to load (expected hundreds of KB)`)
+      : pass('Page content', `${bodyBytes.toLocaleString()} bytes`),
+  );
+
+  // ── 2. Cloudflare ────────────────────────────────────────────────────────────
+  const cfByTitle =
+    pageTitle.toLowerCase().includes('just a moment') ||
+    pageTitle.toLowerCase().includes('attention required');
+  const cfByElement =
+    $('[data-cf-settings]').length > 0 ||
+    $('#cf-wrapper, #challenge-form, #challenge-running, #challenge-stage').length > 0 ||
+    $('script[src*="challenges.cloudflare.com"]').length > 0;
+  const isCf = cfByTitle || cfByElement;
+  findings.push(
+    isCf
+      ? fail('Cloudflare challenge', `Detected — title: "${pageTitle}"`)
+      : pass('Cloudflare challenge', 'Not detected'),
+  );
+
+  // ── 3. Generic security / bot block ─────────────────────────────────────────
+  const blockKeywords = [
+    'access denied', 'you have been blocked', 'ddos-guard',
+    'bot protection', 'security check', 'please verify',
+  ];
+  const foundKeyword = blockKeywords.find(
+    k => bodyText.includes(k) || pageTitle.toLowerCase().includes(k),
+  );
+  findings.push(
+    foundKeyword
+      ? fail('Security / bot block', `Keyword "${foundKeyword}" found on page`)
+      : pass('Security / bot block', 'Not detected'),
+  );
+
+  // ── 4. CAPTCHA ───────────────────────────────────────────────────────────────
+  const hasCaptcha =
+    $('iframe[src*="recaptcha"], div.g-recaptcha, [class*="captcha"], [id*="captcha"]').length > 0 ||
+    bodyText.includes('captcha');
+  findings.push(
+    hasCaptcha
+      ? fail('CAPTCHA', 'CAPTCHA element found — manual solving required')
+      : pass('CAPTCHA', 'Not detected'),
+  );
+
+  // ── 5. Login wall ────────────────────────────────────────────────────────────
+  const hasLogin =
+    $('input[type="password"]').length > 0 ||
+    $('form[action*="login"], form[action*="signin"]').length > 0 ||
+    pageTitle.toLowerCase().includes('login') ||
+    pageTitle.toLowerCase().includes('sign in');
+  findings.push(
+    hasLogin
+      ? fail('Login wall', `Login elements present — title: "${pageTitle}"`)
+      : pass('Login wall', 'Not detected'),
+  );
+
+  // ── 6. Progressive scope selector breakdown ──────────────────────────────────
+  // Builds the compound selector segment-by-segment so we can pinpoint exactly
+  // which part of "a b c" breaks down.
+  const scopeSegments = scope.trim().split(/\s+/).filter(Boolean);
+  let builtSelector = '';
+  let lastFoundSeg = '';
+  let firstMissingSeg = '';
+
+  for (const seg of scopeSegments) {
+    const candidate = builtSelector ? `${builtSelector} ${seg}` : seg;
+    if ($(candidate).length > 0) {
+      lastFoundSeg = seg;
+      builtSelector = candidate;
+    } else {
+      firstMissingSeg = seg;
+      break;
+    }
+  }
+
+  if (!firstMissingSeg) {
+    findings.push(pass('TOC scope selector', `"${scope}" matched — unexpected, re-check logic`));
+  } else if (!lastFoundSeg) {
+    const root = scopeSegments[0] ?? scope;
+    findings.push(
+      fail('TOC scope selector', `Root "${root}" not found anywhere on page — scope: "${scope}"`),
+    );
+  } else {
+    const existsElsewhere = $(firstMissingSeg).length > 0;
+    findings.push(
+      fail(
+        'TOC scope selector',
+        `"${lastFoundSeg}" found, but "${firstMissingSeg}" is missing within it` +
+          (existsElsewhere
+            ? ` (it exists elsewhere on the page — wrong DOM hierarchy)`
+            : ` (not found anywhere on page)`) +
+          ` — full scope: "${scope}"`,
+      ),
+    );
+  }
+
+  // ── 7. Link pattern anywhere on page ────────────────────────────────────────
+  const pattern = new RegExp(linkPattern);
+  const allHrefs: string[] = [];
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (href) allHrefs.push(href);
+  });
+  const matchingLinks = allHrefs.filter(h => pattern.test(h));
+
+  if (matchingLinks.length > 0) {
+    findings.push(
+      pass(
+        'Link pattern (anywhere)',
+        `${matchingLinks.length} link(s) matching "${linkPattern}" exist on page — they are just outside the scope selector`,
+      ),
+    );
+  } else {
+    const sample = allHrefs.slice(0, 5).join(', ') || 'none';
+    findings.push(
+      fail(
+        'Link pattern (anywhere)',
+        `No links match "${linkPattern}". Total links: ${allHrefs.length}. Sample: [${sample}]`,
+      ),
+    );
+  }
+
+  // ── Determine likely cause ───────────────────────────────────────────────────
+  // Priority: security blocks first (actionable externally), then scope/structure
+  // issues (actionable in config), then unknown.
+  // "links found outside scope" beats "scope root missing" because it tells the
+  // user the data is there — just the selector needs updating — rather than
+  // implying the whole site was redesigned.
+  let likelyCause: string;
+  if (isCf) {
+    likelyCause =
+      'Cloudflare challenge page — uBlock may not be installed correctly, ' +
+      'or the wait time is too short for the JS challenge to resolve';
+  } else if (foundKeyword) {
+    likelyCause = `Security block ("${foundKeyword}") — the site is actively blocking automated access`;
+  } else if (hasCaptcha) {
+    likelyCause = 'CAPTCHA challenge — automated solving is not supported';
+  } else if (hasLogin) {
+    likelyCause = 'Login wall — the content may have moved behind an account requirement';
+  } else if (matchingLinks.length > 0) {
+    likelyCause =
+      'Chapter links exist on the page but are outside the scope selector — ' +
+      'the DOM structure shifted; update the scope selector in novels.yaml';
+  } else if (!lastFoundSeg) {
+    likelyCause =
+      `Root selector "${scopeSegments[0] ?? scope}" is completely absent — ` +
+      'the site has likely been redesigned; the scope in novels.yaml needs updating';
+  } else if (firstMissingSeg) {
+    likelyCause =
+      `Inner selector "${firstMissingSeg}" is missing within "${lastFoundSeg}" — ` +
+      'a partial page redesign; update the scope selector in novels.yaml';
+  } else {
+    likelyCause = `Unable to determine cause automatically. Page title: "${pageTitle}"`;
+  }
+
+  return { pageTitle, bodyBytes, actualUrl, findings, likelyCause };
+}
+
+export function formatDiagnosis(diagnosis: PageDiagnosis): string {
+  const lines: string[] = [
+    'TOC fetch failed — page diagnosis:',
+    `  Title     : "${diagnosis.pageTitle}"`,
+    `  Body size : ${diagnosis.bodyBytes.toLocaleString()} bytes`,
+  ];
+  if (diagnosis.actualUrl) {
+    lines.push(`  Landed on : ${diagnosis.actualUrl}`);
+  }
+  lines.push('  Checks:');
+  for (const f of diagnosis.findings) {
+    lines.push(`    [${f.passed ? '✓' : '✗'}] ${f.check}: ${f.detail}`);
+  }
+  lines.push(`  Likely cause: ${diagnosis.likelyCause}`);
+  return lines.join('\n');
+}

src/services/scraper.ts

@@ -7,6 +7,7 @@ import path from 'path';
 import type { AppConfig, NovelDefinition } from '../config/types.ts';
 import type { Cache } from './cache.ts';
 import type { ChapterView, TextFragment, Style } from '../types.ts';
+import { analyzeTocPage, formatDiagnosis } from './analyzer.ts';
 
 export type DriverFactory = () => Promise<WebDriver>;
 
@@ -52,24 +53,30 @@ export async function fetchToc(
   const createDriver = driverFactory ?? buildDefaultDriverFactory(def, appConfig);
   const driver = await createDriver();
 
+  // Captured on every attempt so the analyzer always has the most recent page.
+  let lastHtml = '';
+  let lastActualUrl: string | undefined;
+
   try {
     const urls = await pRetry(
       async () => {
         await driver.get(sourceUrl);
         await driver.sleep(def.stealth!.waitMs);
 
-        // Get the live rendered DOM (post-JS, post-uBlock)
+        // Read the live rendered DOM (post-JS, post-uBlock) and the URL we
-        const html = (await driver.executeScript(
+        // actually landed on (may differ from sourceUrl after redirects).
+        lastHtml = (await driver.executeScript(
           'return document.documentElement.outerHTML',
         )) as string;
+        lastActualUrl = (await driver.executeScript(
+          'return window.location.href',
+        )) as string;
 
-        const $ = cheerio.load(html);
+        const $ = cheerio.load(lastHtml);
         const container = $(tocCfg.scope);
 
         if (!container.length) {
-          throw new Error(
+          throw new Error(`TOC container "${tocCfg.scope}" not found`);
-            `TOC container "${tocCfg.scope}" not found on page — page may not have loaded correctly`,
-          );
         }
 
         const pattern = new RegExp(tocCfg.linkPattern);
@@ -81,7 +88,7 @@ export async function fetchToc(
 
         if (found.length === 0) {
           throw new Error(
-            `No TOC links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`,
+            `No links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`,
           );
         }
 
@@ -89,15 +96,32 @@ export async function fetchToc(
       },
       {
         retries: 3,
-        onFailedAttempt: err =>
+        onFailedAttempt: err => {
+          // p-retry types this as RetryContext; the original thrown value is at
+          // err.error. Error.message is non-enumerable so JSON.stringify shows {}.
+          const cause = (err as unknown as { error?: unknown }).error;
+          const msg = cause instanceof Error ? cause.message : String(cause ?? err);
           console.warn(
-            `[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${err.message}`,
+            `[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${msg}`,
-          ),
+          );
+        },
       },
     );
 
     cache.saveToc(id, urls);
     return urls;
+  } catch (err) {
+    // All retries exhausted — run the page analyzer and surface a useful error.
+    const diagnosis = lastHtml
+      ? analyzeTocPage(lastHtml, tocCfg.scope, tocCfg.linkPattern, lastActualUrl)
+      : null;
+
+    const baseMessage = err instanceof Error ? err.message : String(err);
+    const fullMessage = diagnosis
+      ? formatDiagnosis(diagnosis)
+      : `TOC fetch failed (no page was captured): ${baseMessage}`;
+
+    throw new Error(fullMessage);
   } finally {
     await driver.quit();
   }

tests/analyzer.test.ts

@@ -0,0 +1,148 @@
+import { describe, test, expect } from 'bun:test';
+import { analyzeTocPage } from '../src/services/analyzer.ts';
+
+const SCOPE = '#list-chapter .chapter-archive-grid';
+const PATTERN = 'https://novelbin\\.com/b/the-mech-touch/.*';
+
+function makeHtml(opts: {
+  title?: string;
+  body?: string;
+  links?: string[];
+  extraHead?: string;
+}) {
+  const links = (opts.links ?? [])
+    .map(href => `<a href="${href}">link</a>`)
+    .join('');
+  return `<html>
+    <head>
+      <title>${opts.title ?? 'Test Page'}</title>
+      ${opts.extraHead ?? ''}
+    </head>
+    <body>${opts.body ?? ''}${links}</body>
+  </html>`;
+}
+
+describe('analyzer — Cloudflare detection', () => {
+  test('flags "Just a moment" title as Cloudflare', () => {
+    const html = makeHtml({ title: 'Just a moment...' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!;
+    expect(cf.passed).toBe(false);
+    expect(d.likelyCause).toContain('Cloudflare');
+  });
+
+  test('flags #challenge-form element as Cloudflare', () => {
+    const html = makeHtml({ body: '<form id="challenge-form"></form>' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!;
+    expect(cf.passed).toBe(false);
+  });
+
+  test('does not flag a clean page as Cloudflare', () => {
+    const html = makeHtml({ title: 'The Mech Touch — Chapters', body: '<div id="list-chapter"><div class="chapter-archive-grid"></div></div>' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!;
+    expect(cf.passed).toBe(true);
+  });
+});
+
+describe('analyzer — security block detection', () => {
+  test('flags "you have been blocked" body text', () => {
+    const html = makeHtml({ body: '<p>You have been blocked from accessing this site.</p>' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'Security / bot block')!;
+    expect(check.passed).toBe(false);
+    expect(d.likelyCause).toContain('Security block');
+  });
+});
+
+describe('analyzer — CAPTCHA detection', () => {
+  test('flags g-recaptcha div', () => {
+    const html = makeHtml({ body: '<div class="g-recaptcha" data-sitekey="abc"></div>' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'CAPTCHA')!;
+    expect(check.passed).toBe(false);
+  });
+});
+
+describe('analyzer — login wall detection', () => {
+  test('flags password input', () => {
+    const html = makeHtml({ body: '<form><input type="password"/></form>' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'Login wall')!;
+    expect(check.passed).toBe(false);
+    expect(d.likelyCause).toContain('Login');
+  });
+});
+
+describe('analyzer — scope selector breakdown', () => {
+  test('reports root missing when neither segment exists', () => {
+    const html = makeHtml({ body: '<div id="something-else"></div>' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'TOC scope selector')!;
+    expect(check.passed).toBe(false);
+    expect(check.detail).toContain('#list-chapter');
+    expect(check.detail).toContain('not found anywhere');
+    expect(d.likelyCause).toContain('redesigned');
+  });
+
+  test('reports inner selector missing when root exists but child does not', () => {
+    const html = makeHtml({ body: '<div id="list-chapter"><ul><li>no grid here</li></ul></div>' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'TOC scope selector')!;
+    expect(check.passed).toBe(false);
+    expect(check.detail).toContain('#list-chapter');
+    expect(check.detail).toContain('.chapter-archive-grid');
+    expect(d.likelyCause).toContain('Inner selector');
+  });
+
+  test('notes when inner selector exists elsewhere on the page (wrong hierarchy)', () => {
+    const html = makeHtml({
+      body: '<div id="list-chapter"></div><div class="chapter-archive-grid"></div>',
+    });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'TOC scope selector')!;
+    expect(check.passed).toBe(false);
+    expect(check.detail).toContain('exists elsewhere');
+  });
+
+  test('passes when full scope matches', () => {
+    const html = makeHtml({
+      body: '<div id="list-chapter"><div class="chapter-archive-grid"></div></div>',
+    });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'TOC scope selector')!;
+    expect(check.passed).toBe(true);
+  });
+});
+
+describe('analyzer — link pattern check', () => {
+  test('finds matching links outside the scope when scope is missing', () => {
+    const links = [
+      'https://novelbin.com/b/the-mech-touch/chapter-1',
+      'https://novelbin.com/b/the-mech-touch/chapter-2',
+    ];
+    const html = makeHtml({ body: '<div id="other">' + links.map(l => `<a href="${l}">ch</a>`).join('') + '</div>' });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'Link pattern (anywhere)')!;
+    expect(check.passed).toBe(true);
+    expect(check.detail).toContain('2 link');
+    expect(d.likelyCause).toContain('scope selector');
+  });
+
+  test('reports no matching links and samples what is there', () => {
+    const html = makeHtml({ links: ['https://other.com/a', 'https://other.com/b'] });
+    const d = analyzeTocPage(html, SCOPE, PATTERN);
+    const check = d.findings.find(f => f.check === 'Link pattern (anywhere)')!;
+    expect(check.passed).toBe(false);
+    expect(check.detail).toContain('other.com');
+  });
+});
+
+describe('analyzer — actualUrl', () => {
+  test('includes the landed URL in the diagnosis', () => {
+    const html = makeHtml({});
+    const d = analyzeTocPage(html, SCOPE, PATTERN, 'https://novelbin.com/redirected');
+    expect(d.actualUrl).toBe('https://novelbin.com/redirected');
+  });
+});