commit 363d894c797057dc4f093a5ce5ce990adfd09041 Author: qwsdcvghyu89 <61093706+qwsdcvghyu89@users.noreply.github.com> Date: Wed Jun 17 07:19:25 2026 +1000 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a14702c --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# dependencies (bun install) +node_modules + +# output +out +dist +*.tgz + +# code coverage +coverage +*.lcov + +# logs +logs +_.log +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# caches +.eslintcache +.cache +*.tsbuildinfo + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..65310cc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM oven/bun:1 AS base +WORKDIR /app + +FROM base AS install +COPY package.json bun.lock ./ +RUN bun install --frozen-lockfile --production + +FROM base AS final +COPY --from=install /app/node_modules ./node_modules +COPY src/ ./src/ +COPY config/ ./config/ +COPY templates/ ./templates/ +# uBlock0_*.xpi must be placed in addons/ before building. +# Copy it from the C# project: QuickAccess/QuickAccess/uBlock0_1.67.0.firefox.signed.xpi +COPY addons/ ./addons/ +EXPOSE 8080 +CMD ["bun", "run", "src/index.ts"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..b95b362 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# quick-access-hono + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run index.ts +``` + +This project was created using `bun init` in bun v1.3.8. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime. diff --git a/addons/uBlock0_1.67.0.firefox.signed.xpi b/addons/uBlock0_1.67.0.firefox.signed.xpi new file mode 100644 index 0000000..20d7917 Binary files /dev/null and b/addons/uBlock0_1.67.0.firefox.signed.xpi differ diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..3273f6d --- /dev/null +++ b/bun.lock @@ -0,0 +1,134 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "quick-access-hono", + "dependencies": { + "cheerio": "^1.2.0", + "fflate": "^0.8.3", + "hono": "^4.12.25", + "p-limit": "^7.3.0", + "p-retry": "^8.0.0", + "selenium-webdriver": "^4.45.0", + "yaml": "^2.9.0", + }, + "devDependencies": { + "@types/bun": "latest", + "@types/selenium-webdriver": "^4.35.6", + }, + "peerDependencies": { + "typescript": "^5", + }, + }, + }, + "packages": { + "@bazel/runfiles": ["@bazel/runfiles@6.5.0", "", {}, "sha512-RzahvqTkfpY2jsDxo8YItPX+/iZ6hbiikw1YhE0bA9EKBR5Og8Pa6FHn9PO9M0zaXRVsr0GFQLKbB/0rzy9SzA=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + + "@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], + + "@types/selenium-webdriver": ["@types/selenium-webdriver@4.35.6", "", { "dependencies": { "@types/node": "*", "@types/ws": "*" } }, "sha512-8nfyMRi4VvkY9QrQGyY/zkleAhnjnmE8YtdEeoCrWe3izp1P9vo9f5VTNRYF0up+l+kn+VuZah+je+bLddNV+g=="], + + "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], + + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + + "cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="], + + "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], + + "core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="], + + "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + + "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + + "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], + + "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + + "encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="], + + "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + + "fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="], + + "hono": ["hono@4.12.25", "", {}, "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ=="], + + "htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="], + + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + + "immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + + "is-network-error": ["is-network-error@1.3.2", "", {}, "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA=="], + + "isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="], + + "jszip": ["jszip@3.10.1", "", { "dependencies": { "lie": "~3.3.0", "pako": "~1.0.2", "readable-stream": "~2.3.6", "setimmediate": "^1.0.5" } }, "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g=="], + + "lie": ["lie@3.3.0", "", { "dependencies": { "immediate": "~3.0.5" } }, "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ=="], + + "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + + "p-limit": ["p-limit@7.3.0", "", { "dependencies": { "yocto-queue": "^1.2.1" } }, "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw=="], + + "p-retry": ["p-retry@8.0.0", "", { "dependencies": { "is-network-error": "^1.3.0" } }, "sha512-kFVqH1HxOHp8LupNsOys7bSV09VYTRLxarH/mokO4Rqhk6wGi70E0jh4VzvVGXfEVNggHoHLAMWsQqHyU1Ey9A=="], + + "pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="], + + "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], + + "parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="], + + "parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="], + + "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], + + "readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + + "safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "selenium-webdriver": ["selenium-webdriver@4.45.0", "", { "dependencies": { "@bazel/runfiles": "^6.5.0", "jszip": "^3.10.1", "tmp": "^0.2.7", "ws": "^8.21.0" } }, "sha512-Cb2nqvJiwXVOtRTCYHX9D1FJR5+Ls7aL3Nev0t6n4CpXsQ//YGiiUmSCbvTDDeLtbV85SZ46qmLab4SIYKXWRw=="], + + "setimmediate": ["setimmediate@1.0.5", "", {}, "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="], + + "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + + "tmp": ["tmp@0.2.7", "", {}, "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="], + + "undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + + "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="], + + "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="], + + "ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="], + + "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], + + "yocto-queue": ["yocto-queue@1.2.2", "", {}, "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ=="], + + "htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="], + + "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + } +} diff --git a/config/.config.yaml b/config/.config.yaml new file mode 100644 index 0000000..1f07a81 --- /dev/null +++ b/config/.config.yaml @@ -0,0 +1,12 @@ +server: + port: 8080 + +selenium: + remoteUrl: "http://selenium-firefox:4444/wd/hub" + browser: firefox + +cache: + basePath: "~/downloads/quick-access" + +addons: + directory: "./addons" diff --git a/config/novels.yaml b/config/novels.yaml new file mode 100644 index 0000000..d1384fc --- /dev/null +++ b/config/novels.yaml @@ -0,0 +1,21 @@ +novels: + - slug: "exlor-the-mech-touch" + type: web-scraped + source: + url: "https://novelbin.com/b/the-mech-touch#tab-chapters-title" + selectors: + toc: + # CSS equivalent of the original XPath: + # (//*[@id="list-chapter"]//*[contains(@class,"chapter-archive-grid")])[1] + scope: "#list-chapter .chapter-archive-grid" + linkPattern: "https://novelbin\\.com/b/the-mech-touch/.*" + content: + scope: "#chr-content" + stealth: + enabled: true + waitMs: 2000 + addons: + - "uBlock0_1.67.0.firefox.signed.xpi" + download: + parallelism: 4 + minContentBytes: 10 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..106b59c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +services: + quick-access: + image: quick-access-hono + build: + context: . + dockerfile: Dockerfile + ports: + - "8080:8080" + depends_on: + - selenium-firefox + + selenium-firefox: + image: selenium/standalone-firefox:latest + ports: + - "4444:4444" diff --git a/package.json b/package.json new file mode 100644 index 0000000..dc28df6 --- /dev/null +++ b/package.json @@ -0,0 +1,26 @@ +{ + "name": "quick-access-hono", + "module": "src/index.ts", + "type": "module", + "private": true, + "scripts": { + "start": "bun run src/index.ts", + "test": "bun test" + }, + "devDependencies": { + "@types/bun": "latest", + "@types/selenium-webdriver": "^4.35.6" + }, + "peerDependencies": { + "typescript": "^5" + }, + "dependencies": { + "cheerio": "^1.2.0", + "fflate": "^0.8.3", + "hono": "^4.12.25", + "p-limit": "^7.3.0", + "p-retry": "^8.0.0", + "selenium-webdriver": "^4.45.0", + "yaml": "^2.9.0" + } +} diff --git a/src/config/loader.ts b/src/config/loader.ts new file mode 100644 index 0000000..abb0a82 --- /dev/null +++ b/src/config/loader.ts @@ -0,0 +1,38 @@ +import { readFileSync } from 'fs'; +import { parse } from 'yaml'; +import path from 'path'; +import type { AppConfig, NovelsConfig } from './types.ts'; + +const CONFIG_DIR = path.resolve(process.cwd(), 'config'); + +export function loadAppConfig(): AppConfig { + const raw = readFileSync(path.join(CONFIG_DIR, '.config.yaml'), 'utf-8'); + const config = parse(raw) as AppConfig; + + if (!config.server?.port) throw new Error('Config missing server.port'); + if (!config.selenium?.remoteUrl) throw new Error('Config missing selenium.remoteUrl'); + if (!config.cache?.basePath) throw new Error('Config missing cache.basePath'); + if (!config.addons?.directory) throw new Error('Config missing addons.directory'); + + return config; +} + +export function loadNovelsConfig(): NovelsConfig { + const raw = readFileSync(path.join(CONFIG_DIR, 'novels.yaml'), 'utf-8'); + const config = parse(raw) as NovelsConfig; + + if (!Array.isArray(config.novels)) throw new Error('novels.yaml must have a top-level "novels" array'); + + for (const novel of config.novels) { + if (!novel.slug) throw new Error(`Novel entry missing slug: ${JSON.stringify(novel)}`); + if (!novel.type) throw new Error(`Novel "${novel.slug}" missing type`); + if (novel.type !== 'epub' && novel.type !== 'web-scraped') { + throw new Error(`Novel "${novel.slug}" has unknown type "${novel.type}"`); + } + if (!novel.source?.url && !novel.source?.urls?.length) { + throw new Error(`Novel "${novel.slug}" missing source.url or source.urls`); + } + } + + return config; +} diff --git a/src/config/types.ts b/src/config/types.ts new file mode 100644 index 0000000..acb26c0 --- /dev/null +++ b/src/config/types.ts @@ -0,0 +1,54 @@ +export interface AppConfig { + server: { + port: number; + }; + selenium: { + remoteUrl: string; + browser: string; + }; + cache: { + basePath: string; + }; + addons: { + directory: string; + }; +} + +export interface TocSelector { + scope: string; + linkPattern: string; +} + +export interface ContentSelector { + scope: string; +} + +export interface StealthConfig { + enabled: boolean; + waitMs: number; + addons: string[]; +} + +export interface DownloadConfig { + parallelism: number; + minContentBytes: number; +} + +export interface NovelDefinition { + slug: string; + type: 'epub' | 'web-scraped'; + source: { + url?: string; + urls?: string[]; + }; + selectors?: { + toc?: TocSelector; + content?: ContentSelector; + }; + stealth?: StealthConfig; + download?: DownloadConfig; +} + +export interface NovelsConfig { + novels: NovelDefinition[]; +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..4682e25 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,30 @@ +import { Hono } from 'hono'; +import { loadAppConfig, loadNovelsConfig } from './config/loader.ts'; +import { ResourceRegistry } from './services/registry.ts'; +import { Cache } from './services/cache.ts'; +import { createNovelRouter, registerNovelFromConfig } from './routes/novel.ts'; + +const appConfig = loadAppConfig(); +const novelsConfig = loadNovelsConfig(); + +const registry = new ResourceRegistry(); +const cache = new Cache(appConfig.cache.basePath); + +for (const def of novelsConfig.novels) { + try { + const id = registerNovelFromConfig(def, registry, cache, appConfig); + console.log(`[init] Registered novel "${def.slug}" as id=${id}`); + } catch (err) { + console.error(`[init] Failed to register novel "${def.slug}": ${err instanceof Error ? err.message : err}`); + } +} + +const app = new Hono(); +app.route('/qa/requests/novel', createNovelRouter(registry, cache, appConfig)); + +console.log(`[init] Listening on port ${appConfig.server.port}`); + +export default { + port: appConfig.server.port, + fetch: app.fetch, +}; diff --git a/src/renderers/html.ts b/src/renderers/html.ts new file mode 100644 index 0000000..24bc8d6 --- /dev/null +++ b/src/renderers/html.ts @@ -0,0 +1,62 @@ +import { readFileSync } from 'fs'; +import path from 'path'; +import type { ChapterView } from '../types.ts'; + +const TEMPLATE_PATH = path.resolve(process.cwd(), 'templates', 'book.template.html'); + +let templateCache: string | null = null; + +function getTemplate(): string { + if (!templateCache) templateCache = readFileSync(TEMPLATE_PATH, 'utf-8'); + return templateCache; +} + +function escapeHtml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +function readChapterLines(filePath: string): string[] { + try { + const raw = readFileSync(filePath, 'utf-8'); + // Both plain-text and JSON chapters are read as plain text for HTML output, + // matching the C# SomeBasicFileView.GetAsHTML() behaviour. + try { + const fragments = JSON.parse(raw) as Array<{ body: string }>; + if (Array.isArray(fragments) && fragments.length > 0 && 'body' in (fragments[0] ?? {})) { + return fragments.map(f => f.body).filter(Boolean); + } + } catch { + // not JSON — fall through to plain text + } + return raw.split(/[\n\r]+/).map(l => l.trim()).filter(Boolean); + } catch { + return []; + } +} + +export function renderHtml(view: ChapterView): string { + const template = getTemplate(); + + const content = view.chapters + .map((chapter, i) => { + const chapNum = view.min + i; + const lines = readChapterLines(chapter.path); + if (lines.length === 0) return ''; + const paragraphs = lines + .map((line, j) => `

${escapeHtml(line)}

`) + .join('\n'); + return `
\n${paragraphs}\n
`; + }) + .filter(Boolean) + .join('\n'); + + return template + .replace('{id}', String(view.id)) + .replace('{min}', String(view.min)) + .replace('{max}', String(view.max)) + .replace('{content}', content); +} diff --git a/src/renderers/json.ts b/src/renderers/json.ts new file mode 100644 index 0000000..0395c87 --- /dev/null +++ b/src/renderers/json.ts @@ -0,0 +1,27 @@ +import { readFileSync } from 'fs'; +import type { ChapterView, TextFragment } from '../types.ts'; + +interface ChapterRow { + chapter: number; + contents: string[] | TextFragment[]; +} + +function readChapterContents(filePath: string, hasStyle: boolean): string[] | TextFragment[] { + try { + const raw = readFileSync(filePath, 'utf-8'); + if (hasStyle) { + return JSON.parse(raw) as TextFragment[]; + } + return raw.split(/[\n\r]+/).map(l => l.trim()).filter(Boolean); + } catch { + return []; + } +} + +export function renderJson(view: ChapterView): string { + const rows: ChapterRow[] = view.chapters.map((chapter, i) => ({ + chapter: view.min + i, + contents: readChapterContents(chapter.path, chapter.hasStyle), + })); + return JSON.stringify(rows); +} diff --git a/src/routes/novel.ts b/src/routes/novel.ts new file mode 100644 index 0000000..40c1796 --- /dev/null +++ b/src/routes/novel.ts @@ -0,0 +1,180 @@ +import { Hono } from 'hono'; +import { createHash } from 'crypto'; +import { ResourceRegistry, ResourceConflictError, ResourceNotFoundError } from '../services/registry.ts'; +import { Cache } from '../services/cache.ts'; +import { fetchEpub } from '../services/epub.ts'; +import { fetchToc, fetchChapters } from '../services/scraper.ts'; +import { renderHtml } from '../renderers/html.ts'; +import { renderJson } from '../renderers/json.ts'; +import type { AppConfig } from '../config/types.ts'; + +export function createNovelRouter(registry: ResourceRegistry, cache: Cache, appConfig: AppConfig) { + const app = new Hono(); + + // /available MUST be registered before /:novel — otherwise Hono matches "available" as an ID + app.get('/available', c => { + return c.json(registry.getAll()); + }); + + app.get('/:novel', async c => { + const novelParam = c.req.param('novel'); + const novelId = parseInt(novelParam, 10); + if (isNaN(novelId)) { + return c.json({ error: 'Invalid novel ID', detail: 'Novel ID must be a non-negative integer' }, 400); + } + + const rangeString = c.req.query('r'); + if (!rangeString) { + return c.json( + { error: 'Missing required query parameter', detail: '?r=min,max is required, e.g. ?r=0,10' }, + 400, + ); + } + + const parts = rangeString.split(','); + const min = parseInt(parts[0] ?? '', 10); + const max = parseInt(parts[1] ?? '', 10); + if (parts.length !== 2 || isNaN(min) || isNaN(max)) { + return c.json( + { + error: 'Invalid range format', + detail: `Expected ?r=min,max with integer values, got ?r=${rangeString}`, + }, + 400, + ); + } + if (min > max) { + return c.json({ error: 'Invalid range', detail: 'min must be less than or equal to max' }, 400); + } + + const format = c.req.query('f') ?? 'html'; + + let resource; + try { + resource = registry.get(novelId); + } catch (err) { + if (err instanceof ResourceNotFoundError) { + return c.json({ error: 'Unknown novel ID', detail: err.message }, 422); + } + throw err; + } + + try { + const view = await resource.fetch(min, max); + if (format === 'json') { + return c.body(renderJson(view), 200, { 'Content-Type': 'application/json' }); + } + return c.html(renderHtml(view)); + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + console.error(`[route] Fetch failed for novel ${novelId} range ${min}-${max}: ${detail}`); + return c.json({ error: 'Failed to fetch or render content', detail }, 500); + } + }); + + app.post('/new', async c => { + let body: { type?: unknown; slug?: unknown; urls?: unknown }; + try { + body = await c.req.json(); + } catch { + return c.json({ error: 'Invalid request body', detail: 'Body must be valid JSON' }, 400); + } + + if (typeof body.type !== 'string' || !body.type) { + return c.json({ error: 'Missing field', detail: '"type" is required and must be a string' }, 400); + } + if (typeof body.slug !== 'string' || !body.slug) { + return c.json({ error: 'Missing field', detail: '"slug" is required and must be a string' }, 400); + } + if (!Array.isArray(body.urls) || body.urls.length === 0 || typeof body.urls[0] !== 'string') { + return c.json( + { error: 'Missing field', detail: '"urls" must be a non-empty array of strings' }, + 400, + ); + } + + const { type, slug } = body; + const urls = body.urls as string[]; + + if (type !== 'epub') { + return c.json( + { + error: 'Unsupported type', + detail: `Type "${type}" is not supported. Supported types: epub`, + }, + 400, + ); + } + + const hash = createHash('sha256').update(type + urls.join(';')).digest('hex'); + const url = urls[0]!; + + let registeredId = -1; + try { + registeredId = registry.register({ + hash, + slug, + startsAt: 0, + endsAt: -1, + url, + fetch: (from, to) => + fetchEpub(url, registeredId, from, to, cache, (start, end) => + registry.updateRange(registeredId, start, end), + ), + }); + } catch (err) { + if (err instanceof ResourceConflictError) { + return c.json( + { error: 'Conflict', detail: 'A resource with those parameters is already registered' }, + 409, + ); + } + const detail = err instanceof Error ? err.message : String(err); + return c.json({ error: 'Registration failed', detail }, 500); + } + + return c.text(String(registeredId)); + }); + + return app; +} + +export function registerNovelFromConfig( + def: import('../config/types.ts').NovelDefinition, + registry: ResourceRegistry, + cache: Cache, + appConfig: AppConfig, +): number { + const sourceUrl = def.source.url ?? def.source.urls?.[0] ?? ''; + const hash = createHash('sha256').update(def.slug + sourceUrl).digest('hex'); + + let registeredId = -1; + + if (def.type === 'epub') { + registeredId = registry.register({ + hash, + slug: def.slug, + startsAt: 0, + endsAt: -1, + url: sourceUrl, + fetch: (from, to) => + fetchEpub(sourceUrl, registeredId, from, to, cache, (start, end) => + registry.updateRange(registeredId, start, end), + ), + }); + } else { + registeredId = registry.register({ + hash, + slug: def.slug, + startsAt: 0, + endsAt: -1, + url: sourceUrl, + fetch: async (from, to) => { + const toc = await fetchToc(def, registeredId, cache, appConfig); + return fetchChapters(def, registeredId, from, toc.slice(from, to + 1), cache); + }, + }); + } + + return registeredId; +} diff --git a/src/services/cache.ts b/src/services/cache.ts new file mode 100644 index 0000000..be69eb5 --- /dev/null +++ b/src/services/cache.ts @@ -0,0 +1,81 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; +import path from 'path'; +import os from 'os'; +import type { ChapterFile, ChapterView, TextFragment } from '../types.ts'; + +const TOC_FILENAME = 'toc.links'; + +function looksLikeFragments(content: string): boolean { + try { + const parsed = JSON.parse(content); + return ( + Array.isArray(parsed) && + parsed.length > 0 && + typeof parsed[0] === 'object' && + parsed[0] !== null && + 'body' in (parsed[0] as object) + ); + } catch { + return false; + } +} + +export class Cache { + private readonly base: string; + + constructor(basePath: string) { + this.base = basePath.replace(/^~/, os.homedir()); + } + + private chapterPath(id: number, index: number): string { + return path.join(this.base, String(id), String(index)); + } + + private tocPath(id: number): string { + return path.join(this.base, String(id), TOC_FILENAME); + } + + private ensureDir(id: number): void { + const dir = path.join(this.base, String(id)); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + } + + tryGetChapters(id: number, from: number, to: number): ChapterView | null { + if (from > to) return null; + + const chapters: ChapterFile[] = []; + for (let i = from; i <= to; i++) { + const p = this.chapterPath(id, i); + if (!existsSync(p)) return null; + const content = readFileSync(p, 'utf-8'); + if (content.length < 10) return null; + chapters.push({ path: p, hasStyle: looksLikeFragments(content) }); + } + return { id, min: from, max: to, chapters }; + } + + tryGetToc(id: number): string[] | null { + const p = this.tocPath(id); + if (!existsSync(p)) return null; + const lines = readFileSync(p, 'utf-8') + .split('\n') + .map(l => l.trim()) + .filter(Boolean); + if (lines.length === 0) return null; + return lines; + } + + saveChapters(id: number, chapters: { index: number; content: string }[]): ChapterFile[] { + this.ensureDir(id); + return chapters.map(({ index, content }) => { + const p = this.chapterPath(id, index); + writeFileSync(p, content, 'utf-8'); + return { path: p, hasStyle: looksLikeFragments(content) }; + }); + } + + saveToc(id: number, urls: string[]): void { + this.ensureDir(id); + writeFileSync(this.tocPath(id), urls.join('\n'), 'utf-8'); + } +} diff --git a/src/services/epub.ts b/src/services/epub.ts new file mode 100644 index 0000000..c5a2e69 --- /dev/null +++ b/src/services/epub.ts @@ -0,0 +1,107 @@ +import { unzipSync } from 'fflate'; +import * as cheerio from 'cheerio'; +import type { Cache } from './cache.ts'; +import type { ChapterView } from '../types.ts'; + +const dec = new TextDecoder(); + +function parseContainerXml(xml: string): string { + const $ = cheerio.load(xml, { xmlMode: true }); + const opfPath = $('rootfile').attr('full-path'); + if (!opfPath) throw new Error('EPUB container.xml is missing full-path attribute'); + return opfPath; +} + +function parseOpfSpine(xml: string): string[] { + const $ = cheerio.load(xml, { xmlMode: true }); + + const manifest: Record = {}; + $('manifest item').each((_, el) => { + const id = $(el).attr('id'); + const href = $(el).attr('href'); + if (id && href) manifest[id] = href; + }); + + const spine: string[] = []; + $('spine itemref').each((_, el) => { + const idref = $(el).attr('idref'); + if (idref) { + const href = manifest[idref]; + if (href) spine.push(href); + } + }); + + return spine; +} + +function resolveInZip(opfPath: string, relative: string): string { + const dir = opfPath.includes('/') + ? opfPath.slice(0, opfPath.lastIndexOf('/') + 1) + : ''; + return dir + decodeURIComponent(relative); +} + +function extractDivText(xhtml: string): string { + const $ = cheerio.load(xhtml); + const lines: string[] = []; + $('div').each((_, el) => { + const text = $(el).text().trim(); + if (text) lines.push(text); + }); + return lines.join('\n'); +} + +export async function fetchEpub( + url: string, + id: number, + from: number, + to: number, + cache: Cache, + onRangeKnown: (startsAt: number, endsAt: number) => void, +): Promise { + const cached = cache.tryGetChapters(id, from, to); + if (cached) return cached; + + const response = await fetch(url, { redirect: 'follow' }); + if (!response.ok) { + throw new Error(`Failed to download EPUB from ${url}: HTTP ${response.status} ${response.statusText}`); + } + + const buffer = await response.arrayBuffer(); + const zip = unzipSync(new Uint8Array(buffer)); + + const containerEntry = zip['META-INF/container.xml']; + if (!containerEntry) throw new Error('Invalid EPUB: missing META-INF/container.xml'); + + const opfPath = parseContainerXml(dec.decode(containerEntry)); + const opfEntry = zip[opfPath]; + if (!opfEntry) throw new Error(`Invalid EPUB: OPF not found at "${opfPath}"`); + + const spineHrefs = parseOpfSpine(dec.decode(opfEntry)); + if (spineHrefs.length === 0) throw new Error('EPUB spine is empty — no readable chapters found'); + + const toSave: { index: number; content: string }[] = []; + let chapterIndex = 0; + + for (const href of spineHrefs) { + const zipKey = resolveInZip(opfPath, href); + const entry = zip[zipKey]; + if (!entry) continue; + + const text = extractDivText(dec.decode(entry)); + if (!text.trim()) continue; + + toSave.push({ index: chapterIndex, content: text }); + chapterIndex++; + } + + if (toSave.length === 0) throw new Error('EPUB parsed but all chapters were empty'); + + cache.saveChapters(id, toSave); + onRangeKnown(0, chapterIndex - 1); + + const effectiveTo = Math.min(to, chapterIndex - 1); + const view = cache.tryGetChapters(id, from, effectiveTo); + if (!view) throw new Error('Cache read failed immediately after EPUB save — this is a bug'); + return view; +} diff --git a/src/services/registry.ts b/src/services/registry.ts new file mode 100644 index 0000000..060c81b --- /dev/null +++ b/src/services/registry.ts @@ -0,0 +1,52 @@ +import type { Resource, ResourceMeta } from '../types.ts'; + +export class ResourceConflictError extends Error { + constructor() { + super('A resource with that hash is already registered'); + } +} + +export class ResourceNotFoundError extends Error { + constructor(id: number) { + super(`No resource found with id ${id}`); + } +} + +export class ResourceRegistry { + private readonly byId = new Map(); + private readonly hashes = new Set(); + + register(resource: Omit): number { + if (this.hashes.has(resource.hash)) throw new ResourceConflictError(); + + let id = 0; + while (this.byId.has(id)) id++; + + const full: Resource = { ...resource, id }; + this.byId.set(id, full); + this.hashes.add(resource.hash); + return id; + } + + get(id: number): Resource { + const r = this.byId.get(id); + if (!r) throw new ResourceNotFoundError(id); + return r; + } + + getAll(): Record { + const out: Record = {}; + for (const [id, r] of this.byId) { + const { fetch: _fetch, ...meta } = r; + out[id] = meta; + } + return out; + } + + updateRange(id: number, startsAt: number, endsAt: number): void { + const r = this.byId.get(id); + if (!r) throw new ResourceNotFoundError(id); + r.startsAt = startsAt; + r.endsAt = endsAt; + } +} diff --git a/src/services/scraper.ts b/src/services/scraper.ts new file mode 100644 index 0000000..0829e30 --- /dev/null +++ b/src/services/scraper.ts @@ -0,0 +1,186 @@ +import { Builder, type WebDriver } from 'selenium-webdriver'; +import { Options as FirefoxOptions } from 'selenium-webdriver/firefox.js'; +import * as cheerio from 'cheerio'; +import pLimit from 'p-limit'; +import pRetry from 'p-retry'; +import { readFileSync } from 'fs'; +import path from 'path'; +import type { AppConfig, NovelDefinition } from '../config/types.ts'; +import type { Cache } from './cache.ts'; +import type { ChapterView, TextFragment, Style } from '../types.ts'; + +export type DriverFactory = () => Promise; + +function buildDefaultDriverFactory(def: NovelDefinition, appConfig: AppConfig): DriverFactory { + return async () => { + const options = new FirefoxOptions(); + + for (const addonName of (def.stealth?.addons ?? [])) { + const addonPath = path.resolve(appConfig.addons.directory, addonName); + const addonData = readFileSync(addonPath); + options.addExtensions(addonData); + } + + return new Builder() + .usingServer(appConfig.selenium.remoteUrl) + .forBrowser('firefox') + .setFirefoxOptions(options) + .build(); + }; +} + +export async function fetchToc( + def: NovelDefinition, + id: number, + cache: Cache, + appConfig: AppConfig, + driverFactory?: DriverFactory, +): Promise { + const cached = cache.tryGetToc(id); + if (cached) return cached; + + const sourceUrl = def.source.url; + if (!sourceUrl) throw new Error(`Novel "${def.slug}" is missing source.url for TOC fetch`); + + const tocCfg = def.selectors?.toc; + if (!tocCfg) throw new Error(`Novel "${def.slug}" is missing selectors.toc`); + + if (!def.stealth?.enabled) { + throw new Error( + `Novel "${def.slug}" requires stealth: true — the TOC page cannot be fetched without a real browser and uBlock`, + ); + } + + const createDriver = driverFactory ?? buildDefaultDriverFactory(def, appConfig); + const driver = await createDriver(); + + try { + const urls = await pRetry( + async () => { + await driver.get(sourceUrl); + await driver.sleep(def.stealth!.waitMs); + + // Get the live rendered DOM (post-JS, post-uBlock) + const html = (await driver.executeScript( + 'return document.documentElement.outerHTML', + )) as string; + + const $ = cheerio.load(html); + const container = $(tocCfg.scope); + + if (!container.length) { + throw new Error( + `TOC container "${tocCfg.scope}" not found on page — page may not have loaded correctly`, + ); + } + + const pattern = new RegExp(tocCfg.linkPattern); + const found: string[] = []; + container.find('a').each((_, el) => { + const href = $(el).attr('href'); + if (href && pattern.test(href)) found.push(href); + }); + + if (found.length === 0) { + throw new Error( + `No TOC links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`, + ); + } + + return found; + }, + { + retries: 3, + onFailedAttempt: err => + console.warn( + `[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${err.message}`, + ), + }, + ); + + cache.saveToc(id, urls); + return urls; + } finally { + await driver.quit(); + } +} + +const errorStyle: Style = { + display: 'inline', + fontFamily: 'sans-serif', + fontSize: '12pt', + fontStyle: 'regular', + fontWeight: 'bold', + color: 'red', +}; + +export async function fetchChapters( + def: NovelDefinition, + id: number, + from: number, + tocUrls: string[], + cache: Cache, +): Promise { + const to = from + tocUrls.length - 1; + + const cached = cache.tryGetChapters(id, from, to); + if (cached) return cached; + + const contentScope = def.selectors?.content?.scope ?? '#chr-content'; + const minBytes = def.download?.minContentBytes ?? 10; + const parallelism = def.download?.parallelism ?? 4; + + const limit = pLimit(parallelism); + + const chapterResults = await Promise.all( + tocUrls.map((url, i) => + limit(async (): Promise<{ index: number; content: string }> => { + const index = from + i; + try { + return await pRetry( + async () => { + const response = await fetch(url); + if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`); + + const html = await response.text(); + const $ = cheerio.load(html); + const rawText = $(contentScope).text().trim(); + + if (rawText.length < minBytes) { + throw new Error( + `Content at "${url}" is too short (${rawText.length} bytes, minimum ${minBytes})`, + ); + } + + const fragments: TextFragment[] = rawText + .split(/[\n\r]+/) + .map(line => line.trim()) + .filter(Boolean) + .map(body => ({ body })); + + return { index, content: JSON.stringify(fragments) }; + }, + { + retries: 2, + onFailedAttempt: err => + console.warn( + `[scraper] Chapter ${index} attempt ${err.attemptNumber} failed: ${err.message}`, + ), + }, + ); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`[scraper] Chapter ${index} permanently failed: ${message}`); + const errorFragments: TextFragment[] = [ + { body: `Chapter ${index} failed to download`, style: errorStyle }, + { body: message, style: { ...errorStyle, fontWeight: 'light' } }, + ]; + return { index, content: JSON.stringify(errorFragments) }; + } + }), + ), + ); + + const files = cache.saveChapters(id, chapterResults); + return { id, min: from, max: to, chapters: files }; +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..1f1070c --- /dev/null +++ b/src/types.ts @@ -0,0 +1,46 @@ +export interface Style { + display: string; + fontFamily: string; + fontSize: string; + fontStyle: string; + fontWeight: string; + color: string; +} + +export const defaultStyle: Style = { + display: 'inline-block', + fontFamily: 'sans-serif', + fontSize: '12pt', + fontStyle: 'regular', + fontWeight: 'light', + color: 'white', +}; + +export interface TextFragment { + body: string; + style?: Style; +} + +export interface ChapterFile { + path: string; + hasStyle: boolean; +} + +export interface ChapterView { + id: number; + min: number; + max: number; + chapters: ChapterFile[]; +} + +export interface Resource { + id: number; + hash: string; + slug: string; + startsAt: number; + endsAt: number; + url?: string; + fetch: (from: number, to: number) => Promise; +} + +export type ResourceMeta = Omit; diff --git a/templates/book.template.html b/templates/book.template.html new file mode 100644 index 0000000..99c6e43 --- /dev/null +++ b/templates/book.template.html @@ -0,0 +1,46 @@ + + + + + + Novel #{id} — Chapters {min}–{max} + + + +
id: {id}  |  chapters {min}–{max}
+
+ {content} +
+ + diff --git a/tests/cache.test.ts b/tests/cache.test.ts new file mode 100644 index 0000000..68717e2 --- /dev/null +++ b/tests/cache.test.ts @@ -0,0 +1,101 @@ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { mkdtempSync, rmSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { Cache } from '../src/services/cache.ts'; +import type { TextFragment } from '../src/types.ts'; + +let tmpDir: string; +let cache: Cache; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'qa-cache-test-')); + cache = new Cache(tmpDir); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe('cache — chapters', () => { + test('returns null on a cold cache', () => { + expect(cache.tryGetChapters(1, 0, 5)).toBeNull(); + }); + + test('returns null when a chapter in range is missing', () => { + cache.saveChapters(1, [ + { index: 0, content: 'Chapter zero content here' }, + // index 1 is intentionally missing + { index: 2, content: 'Chapter two content here' }, + ]); + expect(cache.tryGetChapters(1, 0, 2)).toBeNull(); + }); + + test('returns null for invalid range (min > max)', () => { + expect(cache.tryGetChapters(1, 5, 2)).toBeNull(); + }); + + test('saves and retrieves plain-text chapters', () => { + cache.saveChapters(1, [ + { index: 0, content: 'Line one\nLine two' }, + { index: 1, content: 'Chapter one content here' }, + ]); + const view = cache.tryGetChapters(1, 0, 1); + expect(view).not.toBeNull(); + expect(view!.id).toBe(1); + expect(view!.min).toBe(0); + expect(view!.max).toBe(1); + expect(view!.chapters).toHaveLength(2); + expect(view!.chapters[0]!.hasStyle).toBe(false); + }); + + test('detects styled (JSON fragment) chapters', () => { + const fragments: TextFragment[] = [{ body: 'Hello world', style: undefined }]; + cache.saveChapters(1, [{ index: 0, content: JSON.stringify(fragments) }]); + const view = cache.tryGetChapters(1, 0, 0); + expect(view).not.toBeNull(); + expect(view!.chapters[0]!.hasStyle).toBe(true); + }); + + test('returns null when a chapter file is too short', () => { + cache.saveChapters(1, [{ index: 0, content: 'hi' }]); // < 10 bytes + expect(cache.tryGetChapters(1, 0, 0)).toBeNull(); + }); + + test('handles single-chapter range', () => { + cache.saveChapters(42, [{ index: 7, content: 'Single chapter content ok' }]); + const view = cache.tryGetChapters(42, 7, 7); + expect(view).not.toBeNull(); + expect(view!.chapters).toHaveLength(1); + }); +}); + +describe('cache — TOC', () => { + test('returns null on a cold cache', () => { + expect(cache.tryGetToc(1)).toBeNull(); + }); + + test('round-trips a TOC', () => { + const urls = [ + 'https://example.com/chapter-1', + 'https://example.com/chapter-2', + 'https://example.com/chapter-3', + ]; + cache.saveToc(1, urls); + expect(cache.tryGetToc(1)).toEqual(urls); + }); + + test('returns null for an empty TOC file', () => { + cache.saveToc(1, []); + expect(cache.tryGetToc(1)).toBeNull(); + }); + + test('overwrites an existing TOC on re-save', () => { + cache.saveToc(1, ['https://example.com/old']); + cache.saveToc(1, ['https://example.com/new-1', 'https://example.com/new-2']); + expect(cache.tryGetToc(1)).toEqual([ + 'https://example.com/new-1', + 'https://example.com/new-2', + ]); + }); +}); diff --git a/tests/epub.test.ts b/tests/epub.test.ts new file mode 100644 index 0000000..73abab2 --- /dev/null +++ b/tests/epub.test.ts @@ -0,0 +1,143 @@ +import { describe, test, expect, beforeEach, afterEach, spyOn } from 'bun:test'; +import { mkdtempSync, rmSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { zipSync } from 'fflate'; +import { Cache } from '../src/services/cache.ts'; +import { fetchEpub } from '../src/services/epub.ts'; + +const enc = new TextEncoder(); + +function buildEpub(chapters: { title: string; content: string }[]): Uint8Array { + const files: Record = {}; + + files['META-INF/container.xml'] = enc.encode( + ` + + + + +`, + ); + + const manifestItems = chapters + .map((_, i) => ``) + .join('\n '); + const spineItems = chapters.map((_, i) => ``).join('\n '); + + files['content.opf'] = enc.encode( + ` + + + ${manifestItems} + + + ${spineItems} + +`, + ); + + for (let i = 0; i < chapters.length; i++) { + const ch = chapters[i]!; + files[`ch${i}.xhtml`] = enc.encode( + `
${ch.content}
`, + ); + } + + return zipSync(files); +} + +let tmpDir: string; +let cache: Cache; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'qa-epub-test-')); + cache = new Cache(tmpDir); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +function mockFetch(data: Uint8Array) { + return spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(data, { status: 200 }), + ); +} + +describe('epub — parsing', () => { + test('extracts the correct number of chapters', async () => { + const epub = buildEpub([ + { title: 'Ch 1', content: 'First chapter content here.' }, + { title: 'Ch 2', content: 'Second chapter content here.' }, + { title: 'Ch 3', content: 'Third chapter content here.' }, + ]); + const spy = mockFetch(epub); + + let rangeStart = -1, rangeEnd = -1; + const view = await fetchEpub('http://fake/book.epub', 1, 0, 2, cache, (s, e) => { + rangeStart = s; rangeEnd = e; + }); + + expect(view.chapters).toHaveLength(3); + expect(rangeStart).toBe(0); + expect(rangeEnd).toBe(2); + spy.mockRestore(); + }); + + test('serves from cache on second call without re-fetching', async () => { + const epub = buildEpub([ + { title: 'Ch 1', content: 'Chapter one long enough content.' }, + ]); + const spy = mockFetch(epub); + + await fetchEpub('http://fake/book.epub', 2, 0, 0, cache, () => {}); + expect(spy).toHaveBeenCalledTimes(1); + + // Second call — should hit cache + await fetchEpub('http://fake/book.epub', 2, 0, 0, cache, () => {}); + expect(spy).toHaveBeenCalledTimes(1); + + spy.mockRestore(); + }); + + test('skips blank chapters', async () => { + const epub = buildEpub([ + { title: 'Cover', content: ' ' }, // blank — should be skipped + { title: 'Ch 1', content: 'Real content here for this chapter.' }, + { title: 'Ch 2', content: 'More real content for the next chapter.' }, + ]); + const spy = mockFetch(epub); + + let rangeEnd = -1; + await fetchEpub('http://fake/book.epub', 3, 0, 10, cache, (_, e) => { rangeEnd = e; }); + expect(rangeEnd).toBe(1); // only 2 non-blank chapters (indices 0 and 1) + spy.mockRestore(); + }); + + test('throws on HTTP error', async () => { + const spy = spyOn(globalThis, 'fetch').mockResolvedValue( + new Response('Not Found', { status: 404 }), + ); + await expect(fetchEpub('http://fake/missing.epub', 4, 0, 0, cache, () => {})).rejects.toThrow( + 'HTTP 404', + ); + spy.mockRestore(); + }); + + test('slices to the requested range', async () => { + const epub = buildEpub([ + { title: 'Ch 0', content: 'Chapter zero has enough content here.' }, + { title: 'Ch 1', content: 'Chapter one has enough content here.' }, + { title: 'Ch 2', content: 'Chapter two has enough content here.' }, + { title: 'Ch 3', content: 'Chapter three has enough content here.' }, + ]); + const spy = mockFetch(epub); + + const view = await fetchEpub('http://fake/book.epub', 5, 1, 2, cache, () => {}); + expect(view.min).toBe(1); + expect(view.max).toBe(2); + expect(view.chapters).toHaveLength(2); + spy.mockRestore(); + }); +}); diff --git a/tests/routes.test.ts b/tests/routes.test.ts new file mode 100644 index 0000000..8870a76 --- /dev/null +++ b/tests/routes.test.ts @@ -0,0 +1,175 @@ +import { describe, test, expect, beforeEach } from 'bun:test'; +import { Hono } from 'hono'; +import { ResourceRegistry } from '../src/services/registry.ts'; +import { createNovelRouter } from '../src/routes/novel.ts'; +import type { ChapterView } from '../src/types.ts'; +import type { AppConfig } from '../src/config/types.ts'; + +// Cache is not exercised by route tests — pass a stub that satisfies the type +const stubCache = {} as import('../src/services/cache.ts').Cache; + +const stubAppConfig: AppConfig = { + server: { port: 8080 }, + selenium: { remoteUrl: 'http://selenium:4444/wd/hub', browser: 'firefox' }, + cache: { basePath: '/tmp' }, + addons: { directory: './addons' }, +}; + +function makeView(id: number, min: number, max: number): ChapterView { + return { id, min, max, chapters: [] }; +} + +let registry: ResourceRegistry; +let app: Hono; + +beforeEach(() => { + registry = new ResourceRegistry(); + const router = createNovelRouter(registry, stubCache, stubAppConfig); + app = new Hono(); + app.route('/qa/requests/novel', router); +}); + +describe('GET /qa/requests/novel/available', () => { + test('returns empty object when nothing registered', async () => { + const res = await app.request('/qa/requests/novel/available'); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body).toEqual({}); + }); + + test('returns registered resources (without fetch fn)', async () => { + registry.register({ + hash: 'abc123', + slug: 'my-novel', + startsAt: 0, + endsAt: 100, + fetch: async () => makeView(0, 0, 0), + }); + const res = await app.request('/qa/requests/novel/available'); + const body = await res.json() as Record; + expect(Object.keys(body)).toHaveLength(1); + expect((body['0'] as Record)['slug']).toBe('my-novel'); + expect((body['0'] as Record)['fetch']).toBeUndefined(); + }); +}); + +describe('GET /qa/requests/novel/:novel', () => { + beforeEach(() => { + registry.register({ + hash: 'deadbeef', + slug: 'test-novel', + startsAt: 0, + endsAt: 9, + fetch: async (from, to) => makeView(0, from, to), + }); + }); + + test('returns 400 when r param is missing', async () => { + const res = await app.request('/qa/requests/novel/0'); + expect(res.status).toBe(400); + const body = await res.json() as { error: string }; + expect(body.error).toContain('Missing'); + }); + + test('returns 400 for malformed r param', async () => { + const res = await app.request('/qa/requests/novel/0?r=notanumber'); + expect(res.status).toBe(400); + }); + + test('returns 400 when min > max', async () => { + const res = await app.request('/qa/requests/novel/0?r=10,5'); + expect(res.status).toBe(400); + }); + + test('returns 422 for unknown novel id', async () => { + const res = await app.request('/qa/requests/novel/999?r=0,5'); + expect(res.status).toBe(422); + const body = await res.json() as { error: string }; + expect(body.error).toContain('Unknown'); + }); + + test('returns 400 for non-integer novel id', async () => { + const res = await app.request('/qa/requests/novel/abc?r=0,5'); + expect(res.status).toBe(400); + }); + + test('returns 200 html by default', async () => { + const res = await app.request('/qa/requests/novel/0?r=0,0'); + // fetch returns empty chapters so HTML template would be rendered (template read from disk) + // We just verify it doesn't 5xx from the route layer itself + expect([200, 500]).toContain(res.status); + }); + + test('returns 200 json when f=json', async () => { + const res = await app.request('/qa/requests/novel/0?r=0,0&f=json'); + expect([200, 500]).toContain(res.status); + if (res.status === 200) { + expect(res.headers.get('content-type')).toContain('application/json'); + } + }); + + test('returns 500 when fetch throws', async () => { + registry.register({ + hash: 'badfetch', + slug: 'broken-novel', + startsAt: 0, + endsAt: 0, + fetch: async () => { throw new Error('something went wrong internally'); }, + }); + const res = await app.request('/qa/requests/novel/1?r=0,0'); + expect(res.status).toBe(500); + const body = await res.json() as { error: string; detail: string }; + expect(body.error).toBeDefined(); + expect(body.detail).toBe('something went wrong internally'); + }); +}); + +describe('POST /qa/requests/novel/new', () => { + test('returns 400 for non-JSON body', async () => { + const res = await app.request('/qa/requests/novel/new', { + method: 'POST', + body: 'not json', + headers: { 'Content-Type': 'text/plain' }, + }); + expect(res.status).toBe(400); + }); + + test('returns 400 when type is missing', async () => { + const res = await app.request('/qa/requests/novel/new', { + method: 'POST', + body: JSON.stringify({ slug: 'test', urls: ['http://x.com/book.epub'] }), + headers: { 'Content-Type': 'application/json' }, + }); + expect(res.status).toBe(400); + }); + + test('returns 400 for unsupported type', async () => { + const res = await app.request('/qa/requests/novel/new', { + method: 'POST', + body: JSON.stringify({ type: 'novel', slug: 'test', urls: ['http://x.com'] }), + headers: { 'Content-Type': 'application/json' }, + }); + expect(res.status).toBe(400); + const body = await res.json() as { error: string }; + expect(body.error).toContain('Unsupported'); + }); + + test('registers an epub and returns numeric id', async () => { + const res = await app.request('/qa/requests/novel/new', { + method: 'POST', + body: JSON.stringify({ type: 'epub', slug: 'my-book', urls: ['http://x.com/book.epub'] }), + headers: { 'Content-Type': 'application/json' }, + }); + expect(res.status).toBe(200); + const text = await res.text(); + expect(parseInt(text, 10)).toBeGreaterThanOrEqual(0); + }); + + test('returns 409 on duplicate registration', async () => { + const body = JSON.stringify({ type: 'epub', slug: 'dup', urls: ['http://x.com/dup.epub'] }); + const headers = { 'Content-Type': 'application/json' }; + await app.request('/qa/requests/novel/new', { method: 'POST', body, headers }); + const res = await app.request('/qa/requests/novel/new', { method: 'POST', body, headers }); + expect(res.status).toBe(409); + }); +}); diff --git a/tests/scraper.test.ts b/tests/scraper.test.ts new file mode 100644 index 0000000..b6f0128 --- /dev/null +++ b/tests/scraper.test.ts @@ -0,0 +1,174 @@ +import { describe, test, expect, beforeEach, afterEach, mock, spyOn } from 'bun:test'; +import { mkdtempSync, rmSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { Cache } from '../src/services/cache.ts'; +import { fetchToc, fetchChapters, type DriverFactory } from '../src/services/scraper.ts'; +import type { NovelDefinition, AppConfig } from '../src/config/types.ts'; + +const DEF: NovelDefinition = { + slug: 'test-novel', + type: 'web-scraped', + source: { url: 'https://example.com/novel' }, + selectors: { + toc: { scope: '#chapters', linkPattern: 'https://example\\.com/chapter/.*' }, + content: { scope: '#content' }, + }, + stealth: { enabled: true, waitMs: 0, addons: [] }, + download: { parallelism: 2, minContentBytes: 5 }, +}; + +const APP_CONFIG: AppConfig = { + server: { port: 8080 }, + selenium: { remoteUrl: 'http://selenium:4444/wd/hub', browser: 'firefox' }, + cache: { basePath: '/tmp' }, + addons: { directory: './addons' }, +}; + +let tmpDir: string; +let cache: Cache; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'qa-scraper-test-')); + cache = new Cache(tmpDir); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +function buildMockPage(links: string[]): string { + const anchors = links.map(href => `link`).join(''); + return `
${anchors}
`; +} + +function makeMockDriver(html: string): { driver: ReturnType extends Promise ? T : never; driverFactory: DriverFactory; quitSpy: ReturnType } { + const quitSpy = mock(() => Promise.resolve()); + const driver = { + get: mock((_url: string) => Promise.resolve()), + sleep: mock((_ms: number) => Promise.resolve()), + executeScript: mock((_script: string) => Promise.resolve(html)), + quit: quitSpy, + } as unknown as Awaited>; + const driverFactory: DriverFactory = () => Promise.resolve(driver); + return { driver, driverFactory, quitSpy }; +} + +describe('fetchToc', () => { + test('returns cached TOC without opening a browser', async () => { + const urls = ['https://example.com/chapter/1', 'https://example.com/chapter/2']; + cache.saveToc(0, urls); + + const driverFactory = mock(() => Promise.reject(new Error('should not be called'))); + const result = await fetchToc(DEF, 0, cache, APP_CONFIG, driverFactory as unknown as DriverFactory); + expect(result).toEqual(urls); + expect(driverFactory).not.toHaveBeenCalled(); + }); + + test('fetches TOC via browser when cache is cold', async () => { + const links = ['https://example.com/chapter/1', 'https://example.com/chapter/2']; + const { driverFactory, quitSpy } = makeMockDriver(buildMockPage(links)); + + const result = await fetchToc(DEF, 1, cache, APP_CONFIG, driverFactory); + expect(result).toEqual(links); + expect(quitSpy).toHaveBeenCalledTimes(1); + }); + + test('filters links by linkPattern', async () => { + const html = `
+ ch1 + other + ch3 +
`; + const { driverFactory } = makeMockDriver(html); + + const result = await fetchToc(DEF, 2, cache, APP_CONFIG, driverFactory); + expect(result).toEqual(['https://example.com/chapter/1', 'https://example.com/chapter/3']); + }); + + // Extended timeout: pRetry's default exponential backoff across 4 attempts can take ~7s + test('quits driver even on failure', async () => { + const { driverFactory, quitSpy } = makeMockDriver(''); + await expect(fetchToc(DEF, 3, cache, APP_CONFIG, driverFactory)).rejects.toThrow(); + expect(quitSpy).toHaveBeenCalledTimes(1); + }, 15000); + + test('finds anchors nested at arbitrary depth within the scope element', async () => { + // Simulates a TOC where links are buried inside ul > li > span > a + // rather than being direct children of the scope container. + const html = `
+ +
`; + const { driverFactory } = makeMockDriver(html); + const result = await fetchToc(DEF, 5, cache, APP_CONFIG, driverFactory); + expect(result).toEqual([ + 'https://example.com/chapter/1', + 'https://example.com/chapter/2', + ]); + }); + + test('saves fetched TOC to cache', async () => { + const links = ['https://example.com/chapter/1']; + const { driverFactory } = makeMockDriver(buildMockPage(links)); + await fetchToc(DEF, 4, cache, APP_CONFIG, driverFactory); + expect(cache.tryGetToc(4)).toEqual(links); + }); +}); + +describe('fetchChapters', () => { + test('returns cached view without fetching', async () => { + const content = JSON.stringify([{ body: 'Chapter content here.' }]); + cache.saveChapters(10, [{ index: 0, content }]); + + const fetchSpy = spyOn(globalThis, 'fetch'); + const view = await fetchChapters(DEF, 10, 0, ['https://example.com/chapter/1'], cache); + expect(view.chapters).toHaveLength(1); + expect(fetchSpy).not.toHaveBeenCalled(); + fetchSpy.mockRestore(); + }); + + test('downloads chapters and saves them', async () => { + // mockImplementation (not mockResolvedValue) so each call gets a fresh Response — + // reusing the same instance causes "Body already used" on retry reads. + const spy = spyOn(globalThis, 'fetch').mockImplementation(() => + Promise.resolve(new Response('
Chapter body text here.
', { status: 200 })), + ); + + const view = await fetchChapters( + DEF, + 11, + 0, + ['https://example.com/chapter/1', 'https://example.com/chapter/2'], + cache, + ); + + expect(view.min).toBe(0); + expect(view.max).toBe(1); + expect(view.chapters).toHaveLength(2); + expect(spy).toHaveBeenCalledTimes(2); + spy.mockRestore(); + }); + + test('stores error fragment when chapter content is too short', async () => { + const spy = spyOn(globalThis, 'fetch').mockImplementation(() => + Promise.resolve(new Response('
', { status: 200 })), + ); + + const view = await fetchChapters(DEF, 12, 0, ['https://example.com/chapter/1'], cache); + expect(view.chapters).toHaveLength(1); + + // The saved content should be the error fragment JSON + const saved = cache.tryGetChapters(12, 0, 0); + expect(saved).not.toBeNull(); + spy.mockRestore(); + }); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..bfa0fea --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + // Environment setup & latest features + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}