Fixed error message logging, added detailed diagnostics.

2026-06-17 07:43:13 +10:00
parent fdd1d1131d
commit a5145c6902
3 changed files with 394 additions and 10 deletions
@@ -0,0 +1,212 @@
+import * as cheerio from 'cheerio';
+
+export interface Finding {
+  check: string;
+  passed: boolean;
+  detail: string;
+}
+
+export interface PageDiagnosis {
+  pageTitle: string;
+  bodyBytes: number;
+  actualUrl?: string;
+  findings: Finding[];
+  likelyCause: string;
+}
+
+function pass(check: string, detail: string): Finding {
+  return { check, passed: true, detail };
+}
+
+function fail(check: string, detail: string): Finding {
+  return { check, passed: false, detail };
+}
+
+export function analyzeTocPage(
+  html: string,
+  scope: string,
+  linkPattern: string,
+  actualUrl?: string,
+): PageDiagnosis {
+  const $ = cheerio.load(html);
+  const pageTitle = $('title').text().trim();
+  const bodyText = $('body').text().toLowerCase();
+  const bodyBytes = html.length;
+  const findings: Finding[] = [];
+
+  // ── 1. Page loaded at all ────────────────────────────────────────────────────
+  findings.push(
+    bodyBytes < 500
+      ? fail('Page content', `Only ${bodyBytes} bytes — page likely failed to load (expected hundreds of KB)`)
+      : pass('Page content', `${bodyBytes.toLocaleString()} bytes`),
+  );
+
+  // ── 2. Cloudflare ────────────────────────────────────────────────────────────
+  const cfByTitle =
+    pageTitle.toLowerCase().includes('just a moment') ||
+    pageTitle.toLowerCase().includes('attention required');
+  const cfByElement =
+    $('[data-cf-settings]').length > 0 ||
+    $('#cf-wrapper, #challenge-form, #challenge-running, #challenge-stage').length > 0 ||
+    $('script[src*="challenges.cloudflare.com"]').length > 0;
+  const isCf = cfByTitle || cfByElement;
+  findings.push(
+    isCf
+      ? fail('Cloudflare challenge', `Detected — title: "${pageTitle}"`)
+      : pass('Cloudflare challenge', 'Not detected'),
+  );
+
+  // ── 3. Generic security / bot block ─────────────────────────────────────────
+  const blockKeywords = [
+    'access denied', 'you have been blocked', 'ddos-guard',
+    'bot protection', 'security check', 'please verify',
+  ];
+  const foundKeyword = blockKeywords.find(
+    k => bodyText.includes(k) || pageTitle.toLowerCase().includes(k),
+  );
+  findings.push(
+    foundKeyword
+      ? fail('Security / bot block', `Keyword "${foundKeyword}" found on page`)
+      : pass('Security / bot block', 'Not detected'),
+  );
+
+  // ── 4. CAPTCHA ───────────────────────────────────────────────────────────────
+  const hasCaptcha =
+    $('iframe[src*="recaptcha"], div.g-recaptcha, [class*="captcha"], [id*="captcha"]').length > 0 ||
+    bodyText.includes('captcha');
+  findings.push(
+    hasCaptcha
+      ? fail('CAPTCHA', 'CAPTCHA element found — manual solving required')
+      : pass('CAPTCHA', 'Not detected'),
+  );
+
+  // ── 5. Login wall ────────────────────────────────────────────────────────────
+  const hasLogin =
+    $('input[type="password"]').length > 0 ||
+    $('form[action*="login"], form[action*="signin"]').length > 0 ||
+    pageTitle.toLowerCase().includes('login') ||
+    pageTitle.toLowerCase().includes('sign in');
+  findings.push(
+    hasLogin
+      ? fail('Login wall', `Login elements present — title: "${pageTitle}"`)
+      : pass('Login wall', 'Not detected'),
+  );
+
+  // ── 6. Progressive scope selector breakdown ──────────────────────────────────
+  // Builds the compound selector segment-by-segment so we can pinpoint exactly
+  // which part of "a b c" breaks down.
+  const scopeSegments = scope.trim().split(/\s+/).filter(Boolean);
+  let builtSelector = '';
+  let lastFoundSeg = '';
+  let firstMissingSeg = '';
+
+  for (const seg of scopeSegments) {
+    const candidate = builtSelector ? `${builtSelector} ${seg}` : seg;
+    if ($(candidate).length > 0) {
+      lastFoundSeg = seg;
+      builtSelector = candidate;
+    } else {
+      firstMissingSeg = seg;
+      break;
+    }
+  }
+
+  if (!firstMissingSeg) {
+    findings.push(pass('TOC scope selector', `"${scope}" matched — unexpected, re-check logic`));
+  } else if (!lastFoundSeg) {
+    const root = scopeSegments[0] ?? scope;
+    findings.push(
+      fail('TOC scope selector', `Root "${root}" not found anywhere on page — scope: "${scope}"`),
+    );
+  } else {
+    const existsElsewhere = $(firstMissingSeg).length > 0;
+    findings.push(
+      fail(
+        'TOC scope selector',
+        `"${lastFoundSeg}" found, but "${firstMissingSeg}" is missing within it` +
+          (existsElsewhere
+            ? ` (it exists elsewhere on the page — wrong DOM hierarchy)`
+            : ` (not found anywhere on page)`) +
+          ` — full scope: "${scope}"`,
+      ),
+    );
+  }
+
+  // ── 7. Link pattern anywhere on page ────────────────────────────────────────
+  const pattern = new RegExp(linkPattern);
+  const allHrefs: string[] = [];
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (href) allHrefs.push(href);
+  });
+  const matchingLinks = allHrefs.filter(h => pattern.test(h));
+
+  if (matchingLinks.length > 0) {
+    findings.push(
+      pass(
+        'Link pattern (anywhere)',
+        `${matchingLinks.length} link(s) matching "${linkPattern}" exist on page — they are just outside the scope selector`,
+      ),
+    );
+  } else {
+    const sample = allHrefs.slice(0, 5).join(', ') || 'none';
+    findings.push(
+      fail(
+        'Link pattern (anywhere)',
+        `No links match "${linkPattern}". Total links: ${allHrefs.length}. Sample: [${sample}]`,
+      ),
+    );
+  }
+
+  // ── Determine likely cause ───────────────────────────────────────────────────
+  // Priority: security blocks first (actionable externally), then scope/structure
+  // issues (actionable in config), then unknown.
+  // "links found outside scope" beats "scope root missing" because it tells the
+  // user the data is there — just the selector needs updating — rather than
+  // implying the whole site was redesigned.
+  let likelyCause: string;
+  if (isCf) {
+    likelyCause =
+      'Cloudflare challenge page — uBlock may not be installed correctly, ' +
+      'or the wait time is too short for the JS challenge to resolve';
+  } else if (foundKeyword) {
+    likelyCause = `Security block ("${foundKeyword}") — the site is actively blocking automated access`;
+  } else if (hasCaptcha) {
+    likelyCause = 'CAPTCHA challenge — automated solving is not supported';
+  } else if (hasLogin) {
+    likelyCause = 'Login wall — the content may have moved behind an account requirement';
+  } else if (matchingLinks.length > 0) {
+    likelyCause =
+      'Chapter links exist on the page but are outside the scope selector — ' +
+      'the DOM structure shifted; update the scope selector in novels.yaml';
+  } else if (!lastFoundSeg) {
+    likelyCause =
+      `Root selector "${scopeSegments[0] ?? scope}" is completely absent — ` +
+      'the site has likely been redesigned; the scope in novels.yaml needs updating';
+  } else if (firstMissingSeg) {
+    likelyCause =
+      `Inner selector "${firstMissingSeg}" is missing within "${lastFoundSeg}" — ` +
+      'a partial page redesign; update the scope selector in novels.yaml';
+  } else {
+    likelyCause = `Unable to determine cause automatically. Page title: "${pageTitle}"`;
+  }
+
+  return { pageTitle, bodyBytes, actualUrl, findings, likelyCause };
+}
+
+export function formatDiagnosis(diagnosis: PageDiagnosis): string {
+  const lines: string[] = [
+    'TOC fetch failed — page diagnosis:',
+    `  Title     : "${diagnosis.pageTitle}"`,
+    `  Body size : ${diagnosis.bodyBytes.toLocaleString()} bytes`,
+  ];
+  if (diagnosis.actualUrl) {
+    lines.push(`  Landed on : ${diagnosis.actualUrl}`);
+  }
+  lines.push('  Checks:');
+  for (const f of diagnosis.findings) {
+    lines.push(`    [${f.passed ? '✓' : '✗'}] ${f.check}: ${f.detail}`);
+  }
+  lines.push(`  Likely cause: ${diagnosis.likelyCause}`);
+  return lines.join('\n');
+}
@@ -7,6 +7,7 @@ import path from 'path';
 import type { AppConfig, NovelDefinition } from '../config/types.ts';
 import type { Cache } from './cache.ts';
 import type { ChapterView, TextFragment, Style } from '../types.ts';
+import { analyzeTocPage, formatDiagnosis } from './analyzer.ts';

 export type DriverFactory = () => Promise<WebDriver>;

@@ -52,24 +53,30 @@ export async function fetchToc(
  const createDriver = driverFactory ?? buildDefaultDriverFactory(def, appConfig);
  const driver = await createDriver();

+  // Captured on every attempt so the analyzer always has the most recent page.
+  let lastHtml = '';
+  let lastActualUrl: string | undefined;
+
  try {
    const urls = await pRetry(
      async () => {
        await driver.get(sourceUrl);
        await driver.sleep(def.stealth!.waitMs);

-        // Get the live rendered DOM (post-JS, post-uBlock)
-        const html = (await driver.executeScript(
+        // Read the live rendered DOM (post-JS, post-uBlock) and the URL we
+        // actually landed on (may differ from sourceUrl after redirects).
+        lastHtml = (await driver.executeScript(
          'return document.documentElement.outerHTML',
        )) as string;
+        lastActualUrl = (await driver.executeScript(
+          'return window.location.href',
+        )) as string;

-        const $ = cheerio.load(html);
+        const $ = cheerio.load(lastHtml);
        const container = $(tocCfg.scope);

        if (!container.length) {
-          throw new Error(
-            `TOC container "${tocCfg.scope}" not found on page — page may not have loaded correctly`,
-          );
+          throw new Error(`TOC container "${tocCfg.scope}" not found`);
        }

        const pattern = new RegExp(tocCfg.linkPattern);
@@ -81,7 +88,7 @@ export async function fetchToc(

        if (found.length === 0) {
          throw new Error(
-            `No TOC links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`,
+            `No links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`,
          );
        }

@@ -89,15 +96,32 @@ export async function fetchToc(
      },
      {
        retries: 3,
-        onFailedAttempt: err =>
+        onFailedAttempt: err => {
+          // p-retry types this as RetryContext; the original thrown value is at
+          // err.error. Error.message is non-enumerable so JSON.stringify shows {}.
+          const cause = (err as unknown as { error?: unknown }).error;
+          const msg = cause instanceof Error ? cause.message : String(cause ?? err);
          console.warn(
-            `[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${err.message}`,
-          ),
+            `[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${msg}`,
+          );
+        },
      },
    );

    cache.saveToc(id, urls);
    return urls;
+  } catch (err) {
+    // All retries exhausted — run the page analyzer and surface a useful error.
+    const diagnosis = lastHtml
+      ? analyzeTocPage(lastHtml, tocCfg.scope, tocCfg.linkPattern, lastActualUrl)
+      : null;
+
+    const baseMessage = err instanceof Error ? err.message : String(err);
+    const fullMessage = diagnosis
+      ? formatDiagnosis(diagnosis)
+      : `TOC fetch failed (no page was captured): ${baseMessage}`;
+
+    throw new Error(fullMessage);
  } finally {
    await driver.quit();
  }