From a5145c6902962028830f7c8c441e37b84fa28263 Mon Sep 17 00:00:00 2001 From: qwsdcvghyu89 <61093706+qwsdcvghyu89@users.noreply.github.com> Date: Wed, 17 Jun 2026 07:43:13 +1000 Subject: [PATCH] Fixed error message logging, added detailed diagnostics. --- src/services/analyzer.ts | 212 +++++++++++++++++++++++++++++++++++++++ src/services/scraper.ts | 44 ++++++-- tests/analyzer.test.ts | 148 +++++++++++++++++++++++++++ 3 files changed, 394 insertions(+), 10 deletions(-) create mode 100644 src/services/analyzer.ts create mode 100644 tests/analyzer.test.ts diff --git a/src/services/analyzer.ts b/src/services/analyzer.ts new file mode 100644 index 0000000..8255713 --- /dev/null +++ b/src/services/analyzer.ts @@ -0,0 +1,212 @@ +import * as cheerio from 'cheerio'; + +export interface Finding { + check: string; + passed: boolean; + detail: string; +} + +export interface PageDiagnosis { + pageTitle: string; + bodyBytes: number; + actualUrl?: string; + findings: Finding[]; + likelyCause: string; +} + +function pass(check: string, detail: string): Finding { + return { check, passed: true, detail }; +} + +function fail(check: string, detail: string): Finding { + return { check, passed: false, detail }; +} + +export function analyzeTocPage( + html: string, + scope: string, + linkPattern: string, + actualUrl?: string, +): PageDiagnosis { + const $ = cheerio.load(html); + const pageTitle = $('title').text().trim(); + const bodyText = $('body').text().toLowerCase(); + const bodyBytes = html.length; + const findings: Finding[] = []; + + // ── 1. Page loaded at all ──────────────────────────────────────────────────── + findings.push( + bodyBytes < 500 + ? fail('Page content', `Only ${bodyBytes} bytes — page likely failed to load (expected hundreds of KB)`) + : pass('Page content', `${bodyBytes.toLocaleString()} bytes`), + ); + + // ── 2. Cloudflare ──────────────────────────────────────────────────────────── + const cfByTitle = + pageTitle.toLowerCase().includes('just a moment') || + pageTitle.toLowerCase().includes('attention required'); + const cfByElement = + $('[data-cf-settings]').length > 0 || + $('#cf-wrapper, #challenge-form, #challenge-running, #challenge-stage').length > 0 || + $('script[src*="challenges.cloudflare.com"]').length > 0; + const isCf = cfByTitle || cfByElement; + findings.push( + isCf + ? fail('Cloudflare challenge', `Detected — title: "${pageTitle}"`) + : pass('Cloudflare challenge', 'Not detected'), + ); + + // ── 3. Generic security / bot block ───────────────────────────────────────── + const blockKeywords = [ + 'access denied', 'you have been blocked', 'ddos-guard', + 'bot protection', 'security check', 'please verify', + ]; + const foundKeyword = blockKeywords.find( + k => bodyText.includes(k) || pageTitle.toLowerCase().includes(k), + ); + findings.push( + foundKeyword + ? fail('Security / bot block', `Keyword "${foundKeyword}" found on page`) + : pass('Security / bot block', 'Not detected'), + ); + + // ── 4. CAPTCHA ─────────────────────────────────────────────────────────────── + const hasCaptcha = + $('iframe[src*="recaptcha"], div.g-recaptcha, [class*="captcha"], [id*="captcha"]').length > 0 || + bodyText.includes('captcha'); + findings.push( + hasCaptcha + ? fail('CAPTCHA', 'CAPTCHA element found — manual solving required') + : pass('CAPTCHA', 'Not detected'), + ); + + // ── 5. Login wall ──────────────────────────────────────────────────────────── + const hasLogin = + $('input[type="password"]').length > 0 || + $('form[action*="login"], form[action*="signin"]').length > 0 || + pageTitle.toLowerCase().includes('login') || + pageTitle.toLowerCase().includes('sign in'); + findings.push( + hasLogin + ? fail('Login wall', `Login elements present — title: "${pageTitle}"`) + : pass('Login wall', 'Not detected'), + ); + + // ── 6. Progressive scope selector breakdown ────────────────────────────────── + // Builds the compound selector segment-by-segment so we can pinpoint exactly + // which part of "a b c" breaks down. + const scopeSegments = scope.trim().split(/\s+/).filter(Boolean); + let builtSelector = ''; + let lastFoundSeg = ''; + let firstMissingSeg = ''; + + for (const seg of scopeSegments) { + const candidate = builtSelector ? `${builtSelector} ${seg}` : seg; + if ($(candidate).length > 0) { + lastFoundSeg = seg; + builtSelector = candidate; + } else { + firstMissingSeg = seg; + break; + } + } + + if (!firstMissingSeg) { + findings.push(pass('TOC scope selector', `"${scope}" matched — unexpected, re-check logic`)); + } else if (!lastFoundSeg) { + const root = scopeSegments[0] ?? scope; + findings.push( + fail('TOC scope selector', `Root "${root}" not found anywhere on page — scope: "${scope}"`), + ); + } else { + const existsElsewhere = $(firstMissingSeg).length > 0; + findings.push( + fail( + 'TOC scope selector', + `"${lastFoundSeg}" found, but "${firstMissingSeg}" is missing within it` + + (existsElsewhere + ? ` (it exists elsewhere on the page — wrong DOM hierarchy)` + : ` (not found anywhere on page)`) + + ` — full scope: "${scope}"`, + ), + ); + } + + // ── 7. Link pattern anywhere on page ──────────────────────────────────────── + const pattern = new RegExp(linkPattern); + const allHrefs: string[] = []; + $('a[href]').each((_, el) => { + const href = $(el).attr('href'); + if (href) allHrefs.push(href); + }); + const matchingLinks = allHrefs.filter(h => pattern.test(h)); + + if (matchingLinks.length > 0) { + findings.push( + pass( + 'Link pattern (anywhere)', + `${matchingLinks.length} link(s) matching "${linkPattern}" exist on page — they are just outside the scope selector`, + ), + ); + } else { + const sample = allHrefs.slice(0, 5).join(', ') || 'none'; + findings.push( + fail( + 'Link pattern (anywhere)', + `No links match "${linkPattern}". Total links: ${allHrefs.length}. Sample: [${sample}]`, + ), + ); + } + + // ── Determine likely cause ─────────────────────────────────────────────────── + // Priority: security blocks first (actionable externally), then scope/structure + // issues (actionable in config), then unknown. + // "links found outside scope" beats "scope root missing" because it tells the + // user the data is there — just the selector needs updating — rather than + // implying the whole site was redesigned. + let likelyCause: string; + if (isCf) { + likelyCause = + 'Cloudflare challenge page — uBlock may not be installed correctly, ' + + 'or the wait time is too short for the JS challenge to resolve'; + } else if (foundKeyword) { + likelyCause = `Security block ("${foundKeyword}") — the site is actively blocking automated access`; + } else if (hasCaptcha) { + likelyCause = 'CAPTCHA challenge — automated solving is not supported'; + } else if (hasLogin) { + likelyCause = 'Login wall — the content may have moved behind an account requirement'; + } else if (matchingLinks.length > 0) { + likelyCause = + 'Chapter links exist on the page but are outside the scope selector — ' + + 'the DOM structure shifted; update the scope selector in novels.yaml'; + } else if (!lastFoundSeg) { + likelyCause = + `Root selector "${scopeSegments[0] ?? scope}" is completely absent — ` + + 'the site has likely been redesigned; the scope in novels.yaml needs updating'; + } else if (firstMissingSeg) { + likelyCause = + `Inner selector "${firstMissingSeg}" is missing within "${lastFoundSeg}" — ` + + 'a partial page redesign; update the scope selector in novels.yaml'; + } else { + likelyCause = `Unable to determine cause automatically. Page title: "${pageTitle}"`; + } + + return { pageTitle, bodyBytes, actualUrl, findings, likelyCause }; +} + +export function formatDiagnosis(diagnosis: PageDiagnosis): string { + const lines: string[] = [ + 'TOC fetch failed — page diagnosis:', + ` Title : "${diagnosis.pageTitle}"`, + ` Body size : ${diagnosis.bodyBytes.toLocaleString()} bytes`, + ]; + if (diagnosis.actualUrl) { + lines.push(` Landed on : ${diagnosis.actualUrl}`); + } + lines.push(' Checks:'); + for (const f of diagnosis.findings) { + lines.push(` [${f.passed ? '✓' : '✗'}] ${f.check}: ${f.detail}`); + } + lines.push(` Likely cause: ${diagnosis.likelyCause}`); + return lines.join('\n'); +} diff --git a/src/services/scraper.ts b/src/services/scraper.ts index ddfc918..5cefc2c 100644 --- a/src/services/scraper.ts +++ b/src/services/scraper.ts @@ -7,6 +7,7 @@ import path from 'path'; import type { AppConfig, NovelDefinition } from '../config/types.ts'; import type { Cache } from './cache.ts'; import type { ChapterView, TextFragment, Style } from '../types.ts'; +import { analyzeTocPage, formatDiagnosis } from './analyzer.ts'; export type DriverFactory = () => Promise; @@ -52,24 +53,30 @@ export async function fetchToc( const createDriver = driverFactory ?? buildDefaultDriverFactory(def, appConfig); const driver = await createDriver(); + // Captured on every attempt so the analyzer always has the most recent page. + let lastHtml = ''; + let lastActualUrl: string | undefined; + try { const urls = await pRetry( async () => { await driver.get(sourceUrl); await driver.sleep(def.stealth!.waitMs); - // Get the live rendered DOM (post-JS, post-uBlock) - const html = (await driver.executeScript( + // Read the live rendered DOM (post-JS, post-uBlock) and the URL we + // actually landed on (may differ from sourceUrl after redirects). + lastHtml = (await driver.executeScript( 'return document.documentElement.outerHTML', )) as string; + lastActualUrl = (await driver.executeScript( + 'return window.location.href', + )) as string; - const $ = cheerio.load(html); + const $ = cheerio.load(lastHtml); const container = $(tocCfg.scope); if (!container.length) { - throw new Error( - `TOC container "${tocCfg.scope}" not found on page — page may not have loaded correctly`, - ); + throw new Error(`TOC container "${tocCfg.scope}" not found`); } const pattern = new RegExp(tocCfg.linkPattern); @@ -81,7 +88,7 @@ export async function fetchToc( if (found.length === 0) { throw new Error( - `No TOC links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`, + `No links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`, ); } @@ -89,15 +96,32 @@ export async function fetchToc( }, { retries: 3, - onFailedAttempt: err => + onFailedAttempt: err => { + // p-retry types this as RetryContext; the original thrown value is at + // err.error. Error.message is non-enumerable so JSON.stringify shows {}. + const cause = (err as unknown as { error?: unknown }).error; + const msg = cause instanceof Error ? cause.message : String(cause ?? err); console.warn( - `[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${err.message}`, - ), + `[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${msg}`, + ); + }, }, ); cache.saveToc(id, urls); return urls; + } catch (err) { + // All retries exhausted — run the page analyzer and surface a useful error. + const diagnosis = lastHtml + ? analyzeTocPage(lastHtml, tocCfg.scope, tocCfg.linkPattern, lastActualUrl) + : null; + + const baseMessage = err instanceof Error ? err.message : String(err); + const fullMessage = diagnosis + ? formatDiagnosis(diagnosis) + : `TOC fetch failed (no page was captured): ${baseMessage}`; + + throw new Error(fullMessage); } finally { await driver.quit(); } diff --git a/tests/analyzer.test.ts b/tests/analyzer.test.ts new file mode 100644 index 0000000..34375e2 --- /dev/null +++ b/tests/analyzer.test.ts @@ -0,0 +1,148 @@ +import { describe, test, expect } from 'bun:test'; +import { analyzeTocPage } from '../src/services/analyzer.ts'; + +const SCOPE = '#list-chapter .chapter-archive-grid'; +const PATTERN = 'https://novelbin\\.com/b/the-mech-touch/.*'; + +function makeHtml(opts: { + title?: string; + body?: string; + links?: string[]; + extraHead?: string; +}) { + const links = (opts.links ?? []) + .map(href => `link`) + .join(''); + return ` + + ${opts.title ?? 'Test Page'} + ${opts.extraHead ?? ''} + + ${opts.body ?? ''}${links} + `; +} + +describe('analyzer — Cloudflare detection', () => { + test('flags "Just a moment" title as Cloudflare', () => { + const html = makeHtml({ title: 'Just a moment...' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!; + expect(cf.passed).toBe(false); + expect(d.likelyCause).toContain('Cloudflare'); + }); + + test('flags #challenge-form element as Cloudflare', () => { + const html = makeHtml({ body: '
' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!; + expect(cf.passed).toBe(false); + }); + + test('does not flag a clean page as Cloudflare', () => { + const html = makeHtml({ title: 'The Mech Touch — Chapters', body: '
' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!; + expect(cf.passed).toBe(true); + }); +}); + +describe('analyzer — security block detection', () => { + test('flags "you have been blocked" body text', () => { + const html = makeHtml({ body: '

You have been blocked from accessing this site.

' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'Security / bot block')!; + expect(check.passed).toBe(false); + expect(d.likelyCause).toContain('Security block'); + }); +}); + +describe('analyzer — CAPTCHA detection', () => { + test('flags g-recaptcha div', () => { + const html = makeHtml({ body: '
' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'CAPTCHA')!; + expect(check.passed).toBe(false); + }); +}); + +describe('analyzer — login wall detection', () => { + test('flags password input', () => { + const html = makeHtml({ body: '
' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'Login wall')!; + expect(check.passed).toBe(false); + expect(d.likelyCause).toContain('Login'); + }); +}); + +describe('analyzer — scope selector breakdown', () => { + test('reports root missing when neither segment exists', () => { + const html = makeHtml({ body: '
' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'TOC scope selector')!; + expect(check.passed).toBe(false); + expect(check.detail).toContain('#list-chapter'); + expect(check.detail).toContain('not found anywhere'); + expect(d.likelyCause).toContain('redesigned'); + }); + + test('reports inner selector missing when root exists but child does not', () => { + const html = makeHtml({ body: '
' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'TOC scope selector')!; + expect(check.passed).toBe(false); + expect(check.detail).toContain('#list-chapter'); + expect(check.detail).toContain('.chapter-archive-grid'); + expect(d.likelyCause).toContain('Inner selector'); + }); + + test('notes when inner selector exists elsewhere on the page (wrong hierarchy)', () => { + const html = makeHtml({ + body: '
', + }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'TOC scope selector')!; + expect(check.passed).toBe(false); + expect(check.detail).toContain('exists elsewhere'); + }); + + test('passes when full scope matches', () => { + const html = makeHtml({ + body: '
', + }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'TOC scope selector')!; + expect(check.passed).toBe(true); + }); +}); + +describe('analyzer — link pattern check', () => { + test('finds matching links outside the scope when scope is missing', () => { + const links = [ + 'https://novelbin.com/b/the-mech-touch/chapter-1', + 'https://novelbin.com/b/the-mech-touch/chapter-2', + ]; + const html = makeHtml({ body: '
' + links.map(l => `ch`).join('') + '
' }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'Link pattern (anywhere)')!; + expect(check.passed).toBe(true); + expect(check.detail).toContain('2 link'); + expect(d.likelyCause).toContain('scope selector'); + }); + + test('reports no matching links and samples what is there', () => { + const html = makeHtml({ links: ['https://other.com/a', 'https://other.com/b'] }); + const d = analyzeTocPage(html, SCOPE, PATTERN); + const check = d.findings.find(f => f.check === 'Link pattern (anywhere)')!; + expect(check.passed).toBe(false); + expect(check.detail).toContain('other.com'); + }); +}); + +describe('analyzer — actualUrl', () => { + test('includes the landed URL in the diagnosis', () => { + const html = makeHtml({}); + const d = analyzeTocPage(html, SCOPE, PATTERN, 'https://novelbin.com/redirected'); + expect(d.actualUrl).toBe('https://novelbin.com/redirected'); + }); +});