Fixed failure message logging - added wait step.

This commit is contained in:
qwsdcvghyu89
2026-06-17 07:56:11 +10:00
parent a5145c6902
commit a10437c217
4 changed files with 60 additions and 13 deletions
+9 -1
View File
@@ -23,10 +23,18 @@ export interface ContentSelector {
scope: string;
}
export type StealthStep =
| { type: 'waitMs'; ms: number }
| { type: 'waitForElement'; selector: string; timeoutMs?: number }
| { type: 'click'; selector: string }
| { type: 'scrollToBottom' };
export interface StealthConfig {
enabled: boolean;
waitMs: number;
/** @deprecated Use `steps` instead. Kept for backward compatibility. */
waitMs?: number;
addons: string[];
steps?: StealthStep[];
}
export interface DownloadConfig {
+3
View File
@@ -27,4 +27,7 @@ console.log(`[init] Listening on port ${appConfig.server.port}`);
export default {
port: appConfig.server.port,
fetch: app.fetch,
// TOC scraping can take 15-30s; raise above Bun's 10s default so the first
// request doesn't time out before the cache is populated.
idleTimeout: 120,
};
+44 -11
View File
@@ -1,16 +1,41 @@
import { Builder, type WebDriver } from 'selenium-webdriver';
import { Builder, By, until, type WebDriver } from 'selenium-webdriver';
import { Options as FirefoxOptions } from 'selenium-webdriver/firefox.js';
import * as cheerio from 'cheerio';
import pLimit from 'p-limit';
import pRetry from 'p-retry';
import path from 'path';
import type { AppConfig, NovelDefinition } from '../config/types.ts';
import type { AppConfig, NovelDefinition, StealthConfig } from '../config/types.ts';
import type { Cache } from './cache.ts';
import type { ChapterView, TextFragment, Style } from '../types.ts';
import { analyzeTocPage, formatDiagnosis } from './analyzer.ts';
export type DriverFactory = () => Promise<WebDriver>;
async function executeStealthSteps(driver: WebDriver, stealth: StealthConfig): Promise<void> {
if (stealth.steps && stealth.steps.length > 0) {
for (const step of stealth.steps) {
switch (step.type) {
case 'waitMs':
await driver.sleep(step.ms);
break;
case 'waitForElement':
await driver.wait(until.elementLocated(By.css(step.selector)), step.timeoutMs ?? 10_000);
break;
case 'click': {
const el = await driver.findElement(By.css(step.selector));
await el.click();
break;
}
case 'scrollToBottom':
await driver.executeScript('window.scrollTo(0, document.body.scrollHeight)');
break;
}
}
} else if (stealth.waitMs != null) {
await driver.sleep(stealth.waitMs);
}
}
function buildDefaultDriverFactory(def: NovelDefinition, appConfig: AppConfig): DriverFactory {
return async () => {
const options = new FirefoxOptions();
@@ -61,7 +86,7 @@ export async function fetchToc(
const urls = await pRetry(
async () => {
await driver.get(sourceUrl);
await driver.sleep(def.stealth!.waitMs);
await executeStealthSteps(driver, def.stealth!);
// Read the live rendered DOM (post-JS, post-uBlock) and the URL we
// actually landed on (may differ from sourceUrl after redirects).
@@ -97,10 +122,16 @@ export async function fetchToc(
{
retries: 3,
onFailedAttempt: err => {
// p-retry types this as RetryContext; the original thrown value is at
// err.error. Error.message is non-enumerable so JSON.stringify shows {}.
const cause = (err as unknown as { error?: unknown }).error;
const msg = cause instanceof Error ? cause.message : String(cause ?? err);
// p-retry passes a plain frozen context {error, attemptNumber, retriesLeft, ...}
// NOT an Error subclass, so err.message is always undefined.
// err.error is the original thrown value.
const cause: unknown = (err as { error?: unknown }).error;
const msg =
cause instanceof Error
? (cause.message || cause.constructor.name)
: cause != null
? String(cause)
: `attempt ${err.attemptNumber} failed`;
console.warn(
`[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${msg}`,
);
@@ -184,10 +215,12 @@ export async function fetchChapters(
},
{
retries: 2,
onFailedAttempt: err =>
console.warn(
`[scraper] Chapter ${index} attempt ${err.attemptNumber} failed: ${err.message}`,
),
onFailedAttempt: err => {
const cause: unknown = (err as { error?: unknown }).error;
const msg =
cause instanceof Error ? cause.message || cause.constructor.name : String(cause ?? err);
console.warn(`[scraper] Chapter ${index} attempt ${err.attemptNumber} failed: ${msg}`);
},
},
);
} catch (err) {