From a10437c217c2d88f568061acca7f676e7373469d Mon Sep 17 00:00:00 2001
From: qwsdcvghyu89 <61093706+qwsdcvghyu89@users.noreply.github.com>
Date: Wed, 17 Jun 2026 07:56:11 +1000
Subject: [PATCH] Fixed failure message logging - added wait step.

---
 config/novels.yaml      |  5 +++-
 src/config/types.ts     | 10 +++++++-
 src/index.ts            |  3 +++
 src/services/scraper.ts | 55 ++++++++++++++++++++++++++++++++---------
 4 files changed, 60 insertions(+), 13 deletions(-)

diff --git a/config/novels.yaml b/config/novels.yaml
index d1384fc..d434cf8 100644
--- a/config/novels.yaml
+++ b/config/novels.yaml
@@ -13,9 +13,12 @@ novels:
         scope: "#chr-content"
     stealth:
       enabled: true
-      waitMs: 2000
       addons:
         - "uBlock0_1.67.0.firefox.signed.xpi"
+      steps:
+        - type: waitForElement
+          selector: "#list-chapter .chapter-archive-grid"
+          timeoutMs: 15000
     download:
       parallelism: 4
       minContentBytes: 10
diff --git a/src/config/types.ts b/src/config/types.ts
index acb26c0..d9dca0c 100644
--- a/src/config/types.ts
+++ b/src/config/types.ts
@@ -23,10 +23,18 @@ export interface ContentSelector {
   scope: string;
 }
 
+export type StealthStep =
+  | { type: 'waitMs'; ms: number }
+  | { type: 'waitForElement'; selector: string; timeoutMs?: number }
+  | { type: 'click'; selector: string }
+  | { type: 'scrollToBottom' };
+
 export interface StealthConfig {
   enabled: boolean;
-  waitMs: number;
+  /** @deprecated Use `steps` instead. Kept for backward compatibility. */
+  waitMs?: number;
   addons: string[];
+  steps?: StealthStep[];
 }
 
 export interface DownloadConfig {
diff --git a/src/index.ts b/src/index.ts
index 4682e25..476f549 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -27,4 +27,7 @@ console.log(`[init] Listening on port ${appConfig.server.port}`);
 export default {
   port: appConfig.server.port,
   fetch: app.fetch,
+  // TOC scraping can take 15-30s; raise above Bun's 10s default so the first
+  // request doesn't time out before the cache is populated.
+  idleTimeout: 120,
 };
diff --git a/src/services/scraper.ts b/src/services/scraper.ts
index 5cefc2c..d09b22a 100644
--- a/src/services/scraper.ts
+++ b/src/services/scraper.ts
@@ -1,16 +1,41 @@
-import { Builder, type WebDriver } from 'selenium-webdriver';
+import { Builder, By, until, type WebDriver } from 'selenium-webdriver';
 import { Options as FirefoxOptions } from 'selenium-webdriver/firefox.js';
 import * as cheerio from 'cheerio';
 import pLimit from 'p-limit';
 import pRetry from 'p-retry';
 import path from 'path';
-import type { AppConfig, NovelDefinition } from '../config/types.ts';
+import type { AppConfig, NovelDefinition, StealthConfig } from '../config/types.ts';
 import type { Cache } from './cache.ts';
 import type { ChapterView, TextFragment, Style } from '../types.ts';
 import { analyzeTocPage, formatDiagnosis } from './analyzer.ts';
 
 export type DriverFactory = () => Promise<WebDriver>;
 
+async function executeStealthSteps(driver: WebDriver, stealth: StealthConfig): Promise<void> {
+  if (stealth.steps && stealth.steps.length > 0) {
+    for (const step of stealth.steps) {
+      switch (step.type) {
+        case 'waitMs':
+          await driver.sleep(step.ms);
+          break;
+        case 'waitForElement':
+          await driver.wait(until.elementLocated(By.css(step.selector)), step.timeoutMs ?? 10_000);
+          break;
+        case 'click': {
+          const el = await driver.findElement(By.css(step.selector));
+          await el.click();
+          break;
+        }
+        case 'scrollToBottom':
+          await driver.executeScript('window.scrollTo(0, document.body.scrollHeight)');
+          break;
+      }
+    }
+  } else if (stealth.waitMs != null) {
+    await driver.sleep(stealth.waitMs);
+  }
+}
+
 function buildDefaultDriverFactory(def: NovelDefinition, appConfig: AppConfig): DriverFactory {
   return async () => {
     const options = new FirefoxOptions();
@@ -61,7 +86,7 @@ export async function fetchToc(
     const urls = await pRetry(
       async () => {
         await driver.get(sourceUrl);
-        await driver.sleep(def.stealth!.waitMs);
+        await executeStealthSteps(driver, def.stealth!);
 
         // Read the live rendered DOM (post-JS, post-uBlock) and the URL we
         // actually landed on (may differ from sourceUrl after redirects).
@@ -97,10 +122,16 @@ export async function fetchToc(
       {
         retries: 3,
         onFailedAttempt: err => {
-          // p-retry types this as RetryContext; the original thrown value is at
-          // err.error. Error.message is non-enumerable so JSON.stringify shows {}.
-          const cause = (err as unknown as { error?: unknown }).error;
-          const msg = cause instanceof Error ? cause.message : String(cause ?? err);
+          // p-retry passes a plain frozen context {error, attemptNumber, retriesLeft, ...}
+          // NOT an Error subclass, so err.message is always undefined.
+          // err.error is the original thrown value.
+          const cause: unknown = (err as { error?: unknown }).error;
+          const msg =
+            cause instanceof Error
+              ? (cause.message || cause.constructor.name)
+              : cause != null
+                ? String(cause)
+                : `attempt ${err.attemptNumber} failed`;
           console.warn(
             `[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${msg}`,
           );
@@ -184,10 +215,12 @@ export async function fetchChapters(
             },
             {
               retries: 2,
-              onFailedAttempt: err =>
-                console.warn(
-                  `[scraper] Chapter ${index} attempt ${err.attemptNumber} failed: ${err.message}`,
-                ),
+              onFailedAttempt: err => {
+                const cause: unknown = (err as { error?: unknown }).error;
+                const msg =
+                  cause instanceof Error ? cause.message || cause.constructor.name : String(cause ?? err);
+                console.warn(`[scraper] Chapter ${index} attempt ${err.attemptNumber} failed: ${msg}`);
+              },
             },
           );
         } catch (err) {