Fixed error message logging, added detailed diagnostics.
This commit is contained in:
qwsdcvghyu89
@@ -0,0 +1,212 @@
|
||||
import * as cheerio from 'cheerio';
|
||||
|
||||
export interface Finding {
|
||||
check: string;
|
||||
passed: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface PageDiagnosis {
|
||||
pageTitle: string;
|
||||
bodyBytes: number;
|
||||
actualUrl?: string;
|
||||
findings: Finding[];
|
||||
likelyCause: string;
|
||||
}
|
||||
|
||||
function pass(check: string, detail: string): Finding {
|
||||
return { check, passed: true, detail };
|
||||
}
|
||||
|
||||
function fail(check: string, detail: string): Finding {
|
||||
return { check, passed: false, detail };
|
||||
}
|
||||
|
||||
export function analyzeTocPage(
|
||||
html: string,
|
||||
scope: string,
|
||||
linkPattern: string,
|
||||
actualUrl?: string,
|
||||
): PageDiagnosis {
|
||||
const $ = cheerio.load(html);
|
||||
const pageTitle = $('title').text().trim();
|
||||
const bodyText = $('body').text().toLowerCase();
|
||||
const bodyBytes = html.length;
|
||||
const findings: Finding[] = [];
|
||||
|
||||
// ── 1. Page loaded at all ────────────────────────────────────────────────────
|
||||
findings.push(
|
||||
bodyBytes < 500
|
||||
? fail('Page content', `Only ${bodyBytes} bytes — page likely failed to load (expected hundreds of KB)`)
|
||||
: pass('Page content', `${bodyBytes.toLocaleString()} bytes`),
|
||||
);
|
||||
|
||||
// ── 2. Cloudflare ────────────────────────────────────────────────────────────
|
||||
const cfByTitle =
|
||||
pageTitle.toLowerCase().includes('just a moment') ||
|
||||
pageTitle.toLowerCase().includes('attention required');
|
||||
const cfByElement =
|
||||
$('[data-cf-settings]').length > 0 ||
|
||||
$('#cf-wrapper, #challenge-form, #challenge-running, #challenge-stage').length > 0 ||
|
||||
$('script[src*="challenges.cloudflare.com"]').length > 0;
|
||||
const isCf = cfByTitle || cfByElement;
|
||||
findings.push(
|
||||
isCf
|
||||
? fail('Cloudflare challenge', `Detected — title: "${pageTitle}"`)
|
||||
: pass('Cloudflare challenge', 'Not detected'),
|
||||
);
|
||||
|
||||
// ── 3. Generic security / bot block ─────────────────────────────────────────
|
||||
const blockKeywords = [
|
||||
'access denied', 'you have been blocked', 'ddos-guard',
|
||||
'bot protection', 'security check', 'please verify',
|
||||
];
|
||||
const foundKeyword = blockKeywords.find(
|
||||
k => bodyText.includes(k) || pageTitle.toLowerCase().includes(k),
|
||||
);
|
||||
findings.push(
|
||||
foundKeyword
|
||||
? fail('Security / bot block', `Keyword "${foundKeyword}" found on page`)
|
||||
: pass('Security / bot block', 'Not detected'),
|
||||
);
|
||||
|
||||
// ── 4. CAPTCHA ───────────────────────────────────────────────────────────────
|
||||
const hasCaptcha =
|
||||
$('iframe[src*="recaptcha"], div.g-recaptcha, [class*="captcha"], [id*="captcha"]').length > 0 ||
|
||||
bodyText.includes('captcha');
|
||||
findings.push(
|
||||
hasCaptcha
|
||||
? fail('CAPTCHA', 'CAPTCHA element found — manual solving required')
|
||||
: pass('CAPTCHA', 'Not detected'),
|
||||
);
|
||||
|
||||
// ── 5. Login wall ────────────────────────────────────────────────────────────
|
||||
const hasLogin =
|
||||
$('input[type="password"]').length > 0 ||
|
||||
$('form[action*="login"], form[action*="signin"]').length > 0 ||
|
||||
pageTitle.toLowerCase().includes('login') ||
|
||||
pageTitle.toLowerCase().includes('sign in');
|
||||
findings.push(
|
||||
hasLogin
|
||||
? fail('Login wall', `Login elements present — title: "${pageTitle}"`)
|
||||
: pass('Login wall', 'Not detected'),
|
||||
);
|
||||
|
||||
// ── 6. Progressive scope selector breakdown ──────────────────────────────────
|
||||
// Builds the compound selector segment-by-segment so we can pinpoint exactly
|
||||
// which part of "a b c" breaks down.
|
||||
const scopeSegments = scope.trim().split(/\s+/).filter(Boolean);
|
||||
let builtSelector = '';
|
||||
let lastFoundSeg = '';
|
||||
let firstMissingSeg = '';
|
||||
|
||||
for (const seg of scopeSegments) {
|
||||
const candidate = builtSelector ? `${builtSelector} ${seg}` : seg;
|
||||
if ($(candidate).length > 0) {
|
||||
lastFoundSeg = seg;
|
||||
builtSelector = candidate;
|
||||
} else {
|
||||
firstMissingSeg = seg;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!firstMissingSeg) {
|
||||
findings.push(pass('TOC scope selector', `"${scope}" matched — unexpected, re-check logic`));
|
||||
} else if (!lastFoundSeg) {
|
||||
const root = scopeSegments[0] ?? scope;
|
||||
findings.push(
|
||||
fail('TOC scope selector', `Root "${root}" not found anywhere on page — scope: "${scope}"`),
|
||||
);
|
||||
} else {
|
||||
const existsElsewhere = $(firstMissingSeg).length > 0;
|
||||
findings.push(
|
||||
fail(
|
||||
'TOC scope selector',
|
||||
`"${lastFoundSeg}" found, but "${firstMissingSeg}" is missing within it` +
|
||||
(existsElsewhere
|
||||
? ` (it exists elsewhere on the page — wrong DOM hierarchy)`
|
||||
: ` (not found anywhere on page)`) +
|
||||
` — full scope: "${scope}"`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// ── 7. Link pattern anywhere on page ────────────────────────────────────────
|
||||
const pattern = new RegExp(linkPattern);
|
||||
const allHrefs: string[] = [];
|
||||
$('a[href]').each((_, el) => {
|
||||
const href = $(el).attr('href');
|
||||
if (href) allHrefs.push(href);
|
||||
});
|
||||
const matchingLinks = allHrefs.filter(h => pattern.test(h));
|
||||
|
||||
if (matchingLinks.length > 0) {
|
||||
findings.push(
|
||||
pass(
|
||||
'Link pattern (anywhere)',
|
||||
`${matchingLinks.length} link(s) matching "${linkPattern}" exist on page — they are just outside the scope selector`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
const sample = allHrefs.slice(0, 5).join(', ') || 'none';
|
||||
findings.push(
|
||||
fail(
|
||||
'Link pattern (anywhere)',
|
||||
`No links match "${linkPattern}". Total links: ${allHrefs.length}. Sample: [${sample}]`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// ── Determine likely cause ───────────────────────────────────────────────────
|
||||
// Priority: security blocks first (actionable externally), then scope/structure
|
||||
// issues (actionable in config), then unknown.
|
||||
// "links found outside scope" beats "scope root missing" because it tells the
|
||||
// user the data is there — just the selector needs updating — rather than
|
||||
// implying the whole site was redesigned.
|
||||
let likelyCause: string;
|
||||
if (isCf) {
|
||||
likelyCause =
|
||||
'Cloudflare challenge page — uBlock may not be installed correctly, ' +
|
||||
'or the wait time is too short for the JS challenge to resolve';
|
||||
} else if (foundKeyword) {
|
||||
likelyCause = `Security block ("${foundKeyword}") — the site is actively blocking automated access`;
|
||||
} else if (hasCaptcha) {
|
||||
likelyCause = 'CAPTCHA challenge — automated solving is not supported';
|
||||
} else if (hasLogin) {
|
||||
likelyCause = 'Login wall — the content may have moved behind an account requirement';
|
||||
} else if (matchingLinks.length > 0) {
|
||||
likelyCause =
|
||||
'Chapter links exist on the page but are outside the scope selector — ' +
|
||||
'the DOM structure shifted; update the scope selector in novels.yaml';
|
||||
} else if (!lastFoundSeg) {
|
||||
likelyCause =
|
||||
`Root selector "${scopeSegments[0] ?? scope}" is completely absent — ` +
|
||||
'the site has likely been redesigned; the scope in novels.yaml needs updating';
|
||||
} else if (firstMissingSeg) {
|
||||
likelyCause =
|
||||
`Inner selector "${firstMissingSeg}" is missing within "${lastFoundSeg}" — ` +
|
||||
'a partial page redesign; update the scope selector in novels.yaml';
|
||||
} else {
|
||||
likelyCause = `Unable to determine cause automatically. Page title: "${pageTitle}"`;
|
||||
}
|
||||
|
||||
return { pageTitle, bodyBytes, actualUrl, findings, likelyCause };
|
||||
}
|
||||
|
||||
export function formatDiagnosis(diagnosis: PageDiagnosis): string {
|
||||
const lines: string[] = [
|
||||
'TOC fetch failed — page diagnosis:',
|
||||
` Title : "${diagnosis.pageTitle}"`,
|
||||
` Body size : ${diagnosis.bodyBytes.toLocaleString()} bytes`,
|
||||
];
|
||||
if (diagnosis.actualUrl) {
|
||||
lines.push(` Landed on : ${diagnosis.actualUrl}`);
|
||||
}
|
||||
lines.push(' Checks:');
|
||||
for (const f of diagnosis.findings) {
|
||||
lines.push(` [${f.passed ? '✓' : '✗'}] ${f.check}: ${f.detail}`);
|
||||
}
|
||||
lines.push(` Likely cause: ${diagnosis.likelyCause}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
+34
-10
@@ -7,6 +7,7 @@ import path from 'path';
|
||||
import type { AppConfig, NovelDefinition } from '../config/types.ts';
|
||||
import type { Cache } from './cache.ts';
|
||||
import type { ChapterView, TextFragment, Style } from '../types.ts';
|
||||
import { analyzeTocPage, formatDiagnosis } from './analyzer.ts';
|
||||
|
||||
export type DriverFactory = () => Promise<WebDriver>;
|
||||
|
||||
@@ -52,24 +53,30 @@ export async function fetchToc(
|
||||
const createDriver = driverFactory ?? buildDefaultDriverFactory(def, appConfig);
|
||||
const driver = await createDriver();
|
||||
|
||||
// Captured on every attempt so the analyzer always has the most recent page.
|
||||
let lastHtml = '';
|
||||
let lastActualUrl: string | undefined;
|
||||
|
||||
try {
|
||||
const urls = await pRetry(
|
||||
async () => {
|
||||
await driver.get(sourceUrl);
|
||||
await driver.sleep(def.stealth!.waitMs);
|
||||
|
||||
// Get the live rendered DOM (post-JS, post-uBlock)
|
||||
const html = (await driver.executeScript(
|
||||
// Read the live rendered DOM (post-JS, post-uBlock) and the URL we
|
||||
// actually landed on (may differ from sourceUrl after redirects).
|
||||
lastHtml = (await driver.executeScript(
|
||||
'return document.documentElement.outerHTML',
|
||||
)) as string;
|
||||
lastActualUrl = (await driver.executeScript(
|
||||
'return window.location.href',
|
||||
)) as string;
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
const $ = cheerio.load(lastHtml);
|
||||
const container = $(tocCfg.scope);
|
||||
|
||||
if (!container.length) {
|
||||
throw new Error(
|
||||
`TOC container "${tocCfg.scope}" not found on page — page may not have loaded correctly`,
|
||||
);
|
||||
throw new Error(`TOC container "${tocCfg.scope}" not found`);
|
||||
}
|
||||
|
||||
const pattern = new RegExp(tocCfg.linkPattern);
|
||||
@@ -81,7 +88,7 @@ export async function fetchToc(
|
||||
|
||||
if (found.length === 0) {
|
||||
throw new Error(
|
||||
`No TOC links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`,
|
||||
`No links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -89,15 +96,32 @@ export async function fetchToc(
|
||||
},
|
||||
{
|
||||
retries: 3,
|
||||
onFailedAttempt: err =>
|
||||
onFailedAttempt: err => {
|
||||
// p-retry types this as RetryContext; the original thrown value is at
|
||||
// err.error. Error.message is non-enumerable so JSON.stringify shows {}.
|
||||
const cause = (err as unknown as { error?: unknown }).error;
|
||||
const msg = cause instanceof Error ? cause.message : String(cause ?? err);
|
||||
console.warn(
|
||||
`[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${err.message}`,
|
||||
),
|
||||
`[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${msg}`,
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
cache.saveToc(id, urls);
|
||||
return urls;
|
||||
} catch (err) {
|
||||
// All retries exhausted — run the page analyzer and surface a useful error.
|
||||
const diagnosis = lastHtml
|
||||
? analyzeTocPage(lastHtml, tocCfg.scope, tocCfg.linkPattern, lastActualUrl)
|
||||
: null;
|
||||
|
||||
const baseMessage = err instanceof Error ? err.message : String(err);
|
||||
const fullMessage = diagnosis
|
||||
? formatDiagnosis(diagnosis)
|
||||
: `TOC fetch failed (no page was captured): ${baseMessage}`;
|
||||
|
||||
throw new Error(fullMessage);
|
||||
} finally {
|
||||
await driver.quit();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { analyzeTocPage } from '../src/services/analyzer.ts';
|
||||
|
||||
const SCOPE = '#list-chapter .chapter-archive-grid';
|
||||
const PATTERN = 'https://novelbin\\.com/b/the-mech-touch/.*';
|
||||
|
||||
function makeHtml(opts: {
|
||||
title?: string;
|
||||
body?: string;
|
||||
links?: string[];
|
||||
extraHead?: string;
|
||||
}) {
|
||||
const links = (opts.links ?? [])
|
||||
.map(href => `<a href="${href}">link</a>`)
|
||||
.join('');
|
||||
return `<html>
|
||||
<head>
|
||||
<title>${opts.title ?? 'Test Page'}</title>
|
||||
${opts.extraHead ?? ''}
|
||||
</head>
|
||||
<body>${opts.body ?? ''}${links}</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
describe('analyzer — Cloudflare detection', () => {
|
||||
test('flags "Just a moment" title as Cloudflare', () => {
|
||||
const html = makeHtml({ title: 'Just a moment...' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!;
|
||||
expect(cf.passed).toBe(false);
|
||||
expect(d.likelyCause).toContain('Cloudflare');
|
||||
});
|
||||
|
||||
test('flags #challenge-form element as Cloudflare', () => {
|
||||
const html = makeHtml({ body: '<form id="challenge-form"></form>' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!;
|
||||
expect(cf.passed).toBe(false);
|
||||
});
|
||||
|
||||
test('does not flag a clean page as Cloudflare', () => {
|
||||
const html = makeHtml({ title: 'The Mech Touch — Chapters', body: '<div id="list-chapter"><div class="chapter-archive-grid"></div></div>' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const cf = d.findings.find(f => f.check === 'Cloudflare challenge')!;
|
||||
expect(cf.passed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyzer — security block detection', () => {
|
||||
test('flags "you have been blocked" body text', () => {
|
||||
const html = makeHtml({ body: '<p>You have been blocked from accessing this site.</p>' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'Security / bot block')!;
|
||||
expect(check.passed).toBe(false);
|
||||
expect(d.likelyCause).toContain('Security block');
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyzer — CAPTCHA detection', () => {
|
||||
test('flags g-recaptcha div', () => {
|
||||
const html = makeHtml({ body: '<div class="g-recaptcha" data-sitekey="abc"></div>' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'CAPTCHA')!;
|
||||
expect(check.passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyzer — login wall detection', () => {
|
||||
test('flags password input', () => {
|
||||
const html = makeHtml({ body: '<form><input type="password"/></form>' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'Login wall')!;
|
||||
expect(check.passed).toBe(false);
|
||||
expect(d.likelyCause).toContain('Login');
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyzer — scope selector breakdown', () => {
|
||||
test('reports root missing when neither segment exists', () => {
|
||||
const html = makeHtml({ body: '<div id="something-else"></div>' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'TOC scope selector')!;
|
||||
expect(check.passed).toBe(false);
|
||||
expect(check.detail).toContain('#list-chapter');
|
||||
expect(check.detail).toContain('not found anywhere');
|
||||
expect(d.likelyCause).toContain('redesigned');
|
||||
});
|
||||
|
||||
test('reports inner selector missing when root exists but child does not', () => {
|
||||
const html = makeHtml({ body: '<div id="list-chapter"><ul><li>no grid here</li></ul></div>' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'TOC scope selector')!;
|
||||
expect(check.passed).toBe(false);
|
||||
expect(check.detail).toContain('#list-chapter');
|
||||
expect(check.detail).toContain('.chapter-archive-grid');
|
||||
expect(d.likelyCause).toContain('Inner selector');
|
||||
});
|
||||
|
||||
test('notes when inner selector exists elsewhere on the page (wrong hierarchy)', () => {
|
||||
const html = makeHtml({
|
||||
body: '<div id="list-chapter"></div><div class="chapter-archive-grid"></div>',
|
||||
});
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'TOC scope selector')!;
|
||||
expect(check.passed).toBe(false);
|
||||
expect(check.detail).toContain('exists elsewhere');
|
||||
});
|
||||
|
||||
test('passes when full scope matches', () => {
|
||||
const html = makeHtml({
|
||||
body: '<div id="list-chapter"><div class="chapter-archive-grid"></div></div>',
|
||||
});
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'TOC scope selector')!;
|
||||
expect(check.passed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyzer — link pattern check', () => {
|
||||
test('finds matching links outside the scope when scope is missing', () => {
|
||||
const links = [
|
||||
'https://novelbin.com/b/the-mech-touch/chapter-1',
|
||||
'https://novelbin.com/b/the-mech-touch/chapter-2',
|
||||
];
|
||||
const html = makeHtml({ body: '<div id="other">' + links.map(l => `<a href="${l}">ch</a>`).join('') + '</div>' });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'Link pattern (anywhere)')!;
|
||||
expect(check.passed).toBe(true);
|
||||
expect(check.detail).toContain('2 link');
|
||||
expect(d.likelyCause).toContain('scope selector');
|
||||
});
|
||||
|
||||
test('reports no matching links and samples what is there', () => {
|
||||
const html = makeHtml({ links: ['https://other.com/a', 'https://other.com/b'] });
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN);
|
||||
const check = d.findings.find(f => f.check === 'Link pattern (anywhere)')!;
|
||||
expect(check.passed).toBe(false);
|
||||
expect(check.detail).toContain('other.com');
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyzer — actualUrl', () => {
|
||||
test('includes the landed URL in the diagnosis', () => {
|
||||
const html = makeHtml({});
|
||||
const d = analyzeTocPage(html, SCOPE, PATTERN, 'https://novelbin.com/redirected');
|
||||
expect(d.actualUrl).toBe('https://novelbin.com/redirected');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user