Initial commit
This commit is contained in:
qwsdcvghyu89
+34
@@ -0,0 +1,34 @@
|
||||
# dependencies (bun install)
|
||||
node_modules
|
||||
|
||||
# output
|
||||
out
|
||||
dist
|
||||
*.tgz
|
||||
|
||||
# code coverage
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# logs
|
||||
logs
|
||||
_.log
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# caches
|
||||
.eslintcache
|
||||
.cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
# Finder (MacOS) folder config
|
||||
.DS_Store
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
FROM oven/bun:1 AS base
|
||||
WORKDIR /app
|
||||
|
||||
FROM base AS install
|
||||
COPY package.json bun.lock ./
|
||||
RUN bun install --frozen-lockfile --production
|
||||
|
||||
FROM base AS final
|
||||
COPY --from=install /app/node_modules ./node_modules
|
||||
COPY src/ ./src/
|
||||
COPY config/ ./config/
|
||||
COPY templates/ ./templates/
|
||||
# uBlock0_*.xpi must be placed in addons/ before building.
|
||||
# Copy it from the C# project: QuickAccess/QuickAccess/uBlock0_1.67.0.firefox.signed.xpi
|
||||
COPY addons/ ./addons/
|
||||
EXPOSE 8080
|
||||
CMD ["bun", "run", "src/index.ts"]
|
||||
@@ -0,0 +1,15 @@
|
||||
# quick-access-hono
|
||||
|
||||
To install dependencies:
|
||||
|
||||
```bash
|
||||
bun install
|
||||
```
|
||||
|
||||
To run:
|
||||
|
||||
```bash
|
||||
bun run index.ts
|
||||
```
|
||||
|
||||
This project was created using `bun init` in bun v1.3.8. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.
|
||||
Binary file not shown.
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "quick-access-hono",
|
||||
"dependencies": {
|
||||
"cheerio": "^1.2.0",
|
||||
"fflate": "^0.8.3",
|
||||
"hono": "^4.12.25",
|
||||
"p-limit": "^7.3.0",
|
||||
"p-retry": "^8.0.0",
|
||||
"selenium-webdriver": "^4.45.0",
|
||||
"yaml": "^2.9.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/selenium-webdriver": "^4.35.6",
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@bazel/runfiles": ["@bazel/runfiles@6.5.0", "", {}, "sha512-RzahvqTkfpY2jsDxo8YItPX+/iZ6hbiikw1YhE0bA9EKBR5Og8Pa6FHn9PO9M0zaXRVsr0GFQLKbB/0rzy9SzA=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="],
|
||||
|
||||
"@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="],
|
||||
|
||||
"@types/selenium-webdriver": ["@types/selenium-webdriver@4.35.6", "", { "dependencies": { "@types/node": "*", "@types/ws": "*" } }, "sha512-8nfyMRi4VvkY9QrQGyY/zkleAhnjnmE8YtdEeoCrWe3izp1P9vo9f5VTNRYF0up+l+kn+VuZah+je+bLddNV+g=="],
|
||||
|
||||
"@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="],
|
||||
|
||||
"cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="],
|
||||
|
||||
"cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="],
|
||||
|
||||
"core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="],
|
||||
|
||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||
|
||||
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||
|
||||
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||
|
||||
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||
|
||||
"domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
|
||||
|
||||
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||
|
||||
"encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="],
|
||||
|
||||
"entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
|
||||
"fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="],
|
||||
|
||||
"hono": ["hono@4.12.25", "", {}, "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
|
||||
|
||||
"immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="],
|
||||
|
||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
||||
|
||||
"is-network-error": ["is-network-error@1.3.2", "", {}, "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA=="],
|
||||
|
||||
"isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="],
|
||||
|
||||
"jszip": ["jszip@3.10.1", "", { "dependencies": { "lie": "~3.3.0", "pako": "~1.0.2", "readable-stream": "~2.3.6", "setimmediate": "^1.0.5" } }, "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g=="],
|
||||
|
||||
"lie": ["lie@3.3.0", "", { "dependencies": { "immediate": "~3.0.5" } }, "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ=="],
|
||||
|
||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||
|
||||
"p-limit": ["p-limit@7.3.0", "", { "dependencies": { "yocto-queue": "^1.2.1" } }, "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw=="],
|
||||
|
||||
"p-retry": ["p-retry@8.0.0", "", { "dependencies": { "is-network-error": "^1.3.0" } }, "sha512-kFVqH1HxOHp8LupNsOys7bSV09VYTRLxarH/mokO4Rqhk6wGi70E0jh4VzvVGXfEVNggHoHLAMWsQqHyU1Ey9A=="],
|
||||
|
||||
"pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="],
|
||||
|
||||
"parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
|
||||
|
||||
"parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="],
|
||||
|
||||
"parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="],
|
||||
|
||||
"process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="],
|
||||
|
||||
"readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"selenium-webdriver": ["selenium-webdriver@4.45.0", "", { "dependencies": { "@bazel/runfiles": "^6.5.0", "jszip": "^3.10.1", "tmp": "^0.2.7", "ws": "^8.21.0" } }, "sha512-Cb2nqvJiwXVOtRTCYHX9D1FJR5+Ls7aL3Nev0t6n4CpXsQ//YGiiUmSCbvTDDeLtbV85SZ46qmLab4SIYKXWRw=="],
|
||||
|
||||
"setimmediate": ["setimmediate@1.0.5", "", {}, "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="],
|
||||
|
||||
"string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"tmp": ["tmp@0.2.7", "", {}, "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="],
|
||||
|
||||
"undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="],
|
||||
|
||||
"util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="],
|
||||
|
||||
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||
|
||||
"ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="],
|
||||
|
||||
"yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="],
|
||||
|
||||
"yocto-queue": ["yocto-queue@1.2.2", "", {}, "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ=="],
|
||||
|
||||
"htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="],
|
||||
|
||||
"parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
server:
|
||||
port: 8080
|
||||
|
||||
selenium:
|
||||
remoteUrl: "http://selenium-firefox:4444/wd/hub"
|
||||
browser: firefox
|
||||
|
||||
cache:
|
||||
basePath: "~/downloads/quick-access"
|
||||
|
||||
addons:
|
||||
directory: "./addons"
|
||||
@@ -0,0 +1,21 @@
|
||||
novels:
|
||||
- slug: "exlor-the-mech-touch"
|
||||
type: web-scraped
|
||||
source:
|
||||
url: "https://novelbin.com/b/the-mech-touch#tab-chapters-title"
|
||||
selectors:
|
||||
toc:
|
||||
# CSS equivalent of the original XPath:
|
||||
# (//*[@id="list-chapter"]//*[contains(@class,"chapter-archive-grid")])[1]
|
||||
scope: "#list-chapter .chapter-archive-grid"
|
||||
linkPattern: "https://novelbin\\.com/b/the-mech-touch/.*"
|
||||
content:
|
||||
scope: "#chr-content"
|
||||
stealth:
|
||||
enabled: true
|
||||
waitMs: 2000
|
||||
addons:
|
||||
- "uBlock0_1.67.0.firefox.signed.xpi"
|
||||
download:
|
||||
parallelism: 4
|
||||
minContentBytes: 10
|
||||
@@ -0,0 +1,15 @@
|
||||
services:
|
||||
quick-access:
|
||||
image: quick-access-hono
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- "8080:8080"
|
||||
depends_on:
|
||||
- selenium-firefox
|
||||
|
||||
selenium-firefox:
|
||||
image: selenium/standalone-firefox:latest
|
||||
ports:
|
||||
- "4444:4444"
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"name": "quick-access-hono",
|
||||
"module": "src/index.ts",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "bun run src/index.ts",
|
||||
"test": "bun test"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/selenium-webdriver": "^4.35.6"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5"
|
||||
},
|
||||
"dependencies": {
|
||||
"cheerio": "^1.2.0",
|
||||
"fflate": "^0.8.3",
|
||||
"hono": "^4.12.25",
|
||||
"p-limit": "^7.3.0",
|
||||
"p-retry": "^8.0.0",
|
||||
"selenium-webdriver": "^4.45.0",
|
||||
"yaml": "^2.9.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import { parse } from 'yaml';
|
||||
import path from 'path';
|
||||
import type { AppConfig, NovelsConfig } from './types.ts';
|
||||
|
||||
const CONFIG_DIR = path.resolve(process.cwd(), 'config');
|
||||
|
||||
export function loadAppConfig(): AppConfig {
|
||||
const raw = readFileSync(path.join(CONFIG_DIR, '.config.yaml'), 'utf-8');
|
||||
const config = parse(raw) as AppConfig;
|
||||
|
||||
if (!config.server?.port) throw new Error('Config missing server.port');
|
||||
if (!config.selenium?.remoteUrl) throw new Error('Config missing selenium.remoteUrl');
|
||||
if (!config.cache?.basePath) throw new Error('Config missing cache.basePath');
|
||||
if (!config.addons?.directory) throw new Error('Config missing addons.directory');
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
export function loadNovelsConfig(): NovelsConfig {
|
||||
const raw = readFileSync(path.join(CONFIG_DIR, 'novels.yaml'), 'utf-8');
|
||||
const config = parse(raw) as NovelsConfig;
|
||||
|
||||
if (!Array.isArray(config.novels)) throw new Error('novels.yaml must have a top-level "novels" array');
|
||||
|
||||
for (const novel of config.novels) {
|
||||
if (!novel.slug) throw new Error(`Novel entry missing slug: ${JSON.stringify(novel)}`);
|
||||
if (!novel.type) throw new Error(`Novel "${novel.slug}" missing type`);
|
||||
if (novel.type !== 'epub' && novel.type !== 'web-scraped') {
|
||||
throw new Error(`Novel "${novel.slug}" has unknown type "${novel.type}"`);
|
||||
}
|
||||
if (!novel.source?.url && !novel.source?.urls?.length) {
|
||||
throw new Error(`Novel "${novel.slug}" missing source.url or source.urls`);
|
||||
}
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
export interface AppConfig {
|
||||
server: {
|
||||
port: number;
|
||||
};
|
||||
selenium: {
|
||||
remoteUrl: string;
|
||||
browser: string;
|
||||
};
|
||||
cache: {
|
||||
basePath: string;
|
||||
};
|
||||
addons: {
|
||||
directory: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TocSelector {
|
||||
scope: string;
|
||||
linkPattern: string;
|
||||
}
|
||||
|
||||
export interface ContentSelector {
|
||||
scope: string;
|
||||
}
|
||||
|
||||
export interface StealthConfig {
|
||||
enabled: boolean;
|
||||
waitMs: number;
|
||||
addons: string[];
|
||||
}
|
||||
|
||||
export interface DownloadConfig {
|
||||
parallelism: number;
|
||||
minContentBytes: number;
|
||||
}
|
||||
|
||||
export interface NovelDefinition {
|
||||
slug: string;
|
||||
type: 'epub' | 'web-scraped';
|
||||
source: {
|
||||
url?: string;
|
||||
urls?: string[];
|
||||
};
|
||||
selectors?: {
|
||||
toc?: TocSelector;
|
||||
content?: ContentSelector;
|
||||
};
|
||||
stealth?: StealthConfig;
|
||||
download?: DownloadConfig;
|
||||
}
|
||||
|
||||
export interface NovelsConfig {
|
||||
novels: NovelDefinition[];
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Hono } from 'hono';
|
||||
import { loadAppConfig, loadNovelsConfig } from './config/loader.ts';
|
||||
import { ResourceRegistry } from './services/registry.ts';
|
||||
import { Cache } from './services/cache.ts';
|
||||
import { createNovelRouter, registerNovelFromConfig } from './routes/novel.ts';
|
||||
|
||||
const appConfig = loadAppConfig();
|
||||
const novelsConfig = loadNovelsConfig();
|
||||
|
||||
const registry = new ResourceRegistry();
|
||||
const cache = new Cache(appConfig.cache.basePath);
|
||||
|
||||
for (const def of novelsConfig.novels) {
|
||||
try {
|
||||
const id = registerNovelFromConfig(def, registry, cache, appConfig);
|
||||
console.log(`[init] Registered novel "${def.slug}" as id=${id}`);
|
||||
} catch (err) {
|
||||
console.error(`[init] Failed to register novel "${def.slug}": ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
}
|
||||
|
||||
const app = new Hono();
|
||||
app.route('/qa/requests/novel', createNovelRouter(registry, cache, appConfig));
|
||||
|
||||
console.log(`[init] Listening on port ${appConfig.server.port}`);
|
||||
|
||||
export default {
|
||||
port: appConfig.server.port,
|
||||
fetch: app.fetch,
|
||||
};
|
||||
@@ -0,0 +1,62 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import path from 'path';
|
||||
import type { ChapterView } from '../types.ts';
|
||||
|
||||
const TEMPLATE_PATH = path.resolve(process.cwd(), 'templates', 'book.template.html');
|
||||
|
||||
let templateCache: string | null = null;
|
||||
|
||||
function getTemplate(): string {
|
||||
if (!templateCache) templateCache = readFileSync(TEMPLATE_PATH, 'utf-8');
|
||||
return templateCache;
|
||||
}
|
||||
|
||||
function escapeHtml(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"');
|
||||
}
|
||||
|
||||
function readChapterLines(filePath: string): string[] {
|
||||
try {
|
||||
const raw = readFileSync(filePath, 'utf-8');
|
||||
// Both plain-text and JSON chapters are read as plain text for HTML output,
|
||||
// matching the C# SomeBasicFileView.GetAsHTML() behaviour.
|
||||
try {
|
||||
const fragments = JSON.parse(raw) as Array<{ body: string }>;
|
||||
if (Array.isArray(fragments) && fragments.length > 0 && 'body' in (fragments[0] ?? {})) {
|
||||
return fragments.map(f => f.body).filter(Boolean);
|
||||
}
|
||||
} catch {
|
||||
// not JSON — fall through to plain text
|
||||
}
|
||||
return raw.split(/[\n\r]+/).map(l => l.trim()).filter(Boolean);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function renderHtml(view: ChapterView): string {
|
||||
const template = getTemplate();
|
||||
|
||||
const content = view.chapters
|
||||
.map((chapter, i) => {
|
||||
const chapNum = view.min + i;
|
||||
const lines = readChapterLines(chapter.path);
|
||||
if (lines.length === 0) return '';
|
||||
const paragraphs = lines
|
||||
.map((line, j) => ` <p id="para${chapNum}-${j}">${escapeHtml(line)}</p>`)
|
||||
.join('\n');
|
||||
return `<div id="chap${chapNum}">\n${paragraphs}\n</div>`;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
return template
|
||||
.replace('{id}', String(view.id))
|
||||
.replace('{min}', String(view.min))
|
||||
.replace('{max}', String(view.max))
|
||||
.replace('{content}', content);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import type { ChapterView, TextFragment } from '../types.ts';
|
||||
|
||||
interface ChapterRow {
|
||||
chapter: number;
|
||||
contents: string[] | TextFragment[];
|
||||
}
|
||||
|
||||
function readChapterContents(filePath: string, hasStyle: boolean): string[] | TextFragment[] {
|
||||
try {
|
||||
const raw = readFileSync(filePath, 'utf-8');
|
||||
if (hasStyle) {
|
||||
return JSON.parse(raw) as TextFragment[];
|
||||
}
|
||||
return raw.split(/[\n\r]+/).map(l => l.trim()).filter(Boolean);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function renderJson(view: ChapterView): string {
|
||||
const rows: ChapterRow[] = view.chapters.map((chapter, i) => ({
|
||||
chapter: view.min + i,
|
||||
contents: readChapterContents(chapter.path, chapter.hasStyle),
|
||||
}));
|
||||
return JSON.stringify(rows);
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import { Hono } from 'hono';
|
||||
import { createHash } from 'crypto';
|
||||
import { ResourceRegistry, ResourceConflictError, ResourceNotFoundError } from '../services/registry.ts';
|
||||
import { Cache } from '../services/cache.ts';
|
||||
import { fetchEpub } from '../services/epub.ts';
|
||||
import { fetchToc, fetchChapters } from '../services/scraper.ts';
|
||||
import { renderHtml } from '../renderers/html.ts';
|
||||
import { renderJson } from '../renderers/json.ts';
|
||||
import type { AppConfig } from '../config/types.ts';
|
||||
|
||||
export function createNovelRouter(registry: ResourceRegistry, cache: Cache, appConfig: AppConfig) {
|
||||
const app = new Hono();
|
||||
|
||||
// /available MUST be registered before /:novel — otherwise Hono matches "available" as an ID
|
||||
app.get('/available', c => {
|
||||
return c.json(registry.getAll());
|
||||
});
|
||||
|
||||
app.get('/:novel', async c => {
|
||||
const novelParam = c.req.param('novel');
|
||||
const novelId = parseInt(novelParam, 10);
|
||||
if (isNaN(novelId)) {
|
||||
return c.json({ error: 'Invalid novel ID', detail: 'Novel ID must be a non-negative integer' }, 400);
|
||||
}
|
||||
|
||||
const rangeString = c.req.query('r');
|
||||
if (!rangeString) {
|
||||
return c.json(
|
||||
{ error: 'Missing required query parameter', detail: '?r=min,max is required, e.g. ?r=0,10' },
|
||||
400,
|
||||
);
|
||||
}
|
||||
|
||||
const parts = rangeString.split(',');
|
||||
const min = parseInt(parts[0] ?? '', 10);
|
||||
const max = parseInt(parts[1] ?? '', 10);
|
||||
if (parts.length !== 2 || isNaN(min) || isNaN(max)) {
|
||||
return c.json(
|
||||
{
|
||||
error: 'Invalid range format',
|
||||
detail: `Expected ?r=min,max with integer values, got ?r=${rangeString}`,
|
||||
},
|
||||
400,
|
||||
);
|
||||
}
|
||||
if (min > max) {
|
||||
return c.json({ error: 'Invalid range', detail: 'min must be less than or equal to max' }, 400);
|
||||
}
|
||||
|
||||
const format = c.req.query('f') ?? 'html';
|
||||
|
||||
let resource;
|
||||
try {
|
||||
resource = registry.get(novelId);
|
||||
} catch (err) {
|
||||
if (err instanceof ResourceNotFoundError) {
|
||||
return c.json({ error: 'Unknown novel ID', detail: err.message }, 422);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
try {
|
||||
const view = await resource.fetch(min, max);
|
||||
if (format === 'json') {
|
||||
return c.body(renderJson(view), 200, { 'Content-Type': 'application/json' });
|
||||
}
|
||||
return c.html(renderHtml(view));
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
console.error(`[route] Fetch failed for novel ${novelId} range ${min}-${max}: ${detail}`);
|
||||
return c.json({ error: 'Failed to fetch or render content', detail }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/new', async c => {
|
||||
let body: { type?: unknown; slug?: unknown; urls?: unknown };
|
||||
try {
|
||||
body = await c.req.json<typeof body>();
|
||||
} catch {
|
||||
return c.json({ error: 'Invalid request body', detail: 'Body must be valid JSON' }, 400);
|
||||
}
|
||||
|
||||
if (typeof body.type !== 'string' || !body.type) {
|
||||
return c.json({ error: 'Missing field', detail: '"type" is required and must be a string' }, 400);
|
||||
}
|
||||
if (typeof body.slug !== 'string' || !body.slug) {
|
||||
return c.json({ error: 'Missing field', detail: '"slug" is required and must be a string' }, 400);
|
||||
}
|
||||
if (!Array.isArray(body.urls) || body.urls.length === 0 || typeof body.urls[0] !== 'string') {
|
||||
return c.json(
|
||||
{ error: 'Missing field', detail: '"urls" must be a non-empty array of strings' },
|
||||
400,
|
||||
);
|
||||
}
|
||||
|
||||
const { type, slug } = body;
|
||||
const urls = body.urls as string[];
|
||||
|
||||
if (type !== 'epub') {
|
||||
return c.json(
|
||||
{
|
||||
error: 'Unsupported type',
|
||||
detail: `Type "${type}" is not supported. Supported types: epub`,
|
||||
},
|
||||
400,
|
||||
);
|
||||
}
|
||||
|
||||
const hash = createHash('sha256').update(type + urls.join(';')).digest('hex');
|
||||
const url = urls[0]!;
|
||||
|
||||
let registeredId = -1;
|
||||
try {
|
||||
registeredId = registry.register({
|
||||
hash,
|
||||
slug,
|
||||
startsAt: 0,
|
||||
endsAt: -1,
|
||||
url,
|
||||
fetch: (from, to) =>
|
||||
fetchEpub(url, registeredId, from, to, cache, (start, end) =>
|
||||
registry.updateRange(registeredId, start, end),
|
||||
),
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof ResourceConflictError) {
|
||||
return c.json(
|
||||
{ error: 'Conflict', detail: 'A resource with those parameters is already registered' },
|
||||
409,
|
||||
);
|
||||
}
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
return c.json({ error: 'Registration failed', detail }, 500);
|
||||
}
|
||||
|
||||
return c.text(String(registeredId));
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
export function registerNovelFromConfig(
|
||||
def: import('../config/types.ts').NovelDefinition,
|
||||
registry: ResourceRegistry,
|
||||
cache: Cache,
|
||||
appConfig: AppConfig,
|
||||
): number {
|
||||
const sourceUrl = def.source.url ?? def.source.urls?.[0] ?? '';
|
||||
const hash = createHash('sha256').update(def.slug + sourceUrl).digest('hex');
|
||||
|
||||
let registeredId = -1;
|
||||
|
||||
if (def.type === 'epub') {
|
||||
registeredId = registry.register({
|
||||
hash,
|
||||
slug: def.slug,
|
||||
startsAt: 0,
|
||||
endsAt: -1,
|
||||
url: sourceUrl,
|
||||
fetch: (from, to) =>
|
||||
fetchEpub(sourceUrl, registeredId, from, to, cache, (start, end) =>
|
||||
registry.updateRange(registeredId, start, end),
|
||||
),
|
||||
});
|
||||
} else {
|
||||
registeredId = registry.register({
|
||||
hash,
|
||||
slug: def.slug,
|
||||
startsAt: 0,
|
||||
endsAt: -1,
|
||||
url: sourceUrl,
|
||||
fetch: async (from, to) => {
|
||||
const toc = await fetchToc(def, registeredId, cache, appConfig);
|
||||
return fetchChapters(def, registeredId, from, toc.slice(from, to + 1), cache);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return registeredId;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import type { ChapterFile, ChapterView, TextFragment } from '../types.ts';
|
||||
|
||||
const TOC_FILENAME = 'toc.links';
|
||||
|
||||
function looksLikeFragments(content: string): boolean {
|
||||
try {
|
||||
const parsed = JSON.parse(content);
|
||||
return (
|
||||
Array.isArray(parsed) &&
|
||||
parsed.length > 0 &&
|
||||
typeof parsed[0] === 'object' &&
|
||||
parsed[0] !== null &&
|
||||
'body' in (parsed[0] as object)
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export class Cache {
|
||||
private readonly base: string;
|
||||
|
||||
constructor(basePath: string) {
|
||||
this.base = basePath.replace(/^~/, os.homedir());
|
||||
}
|
||||
|
||||
private chapterPath(id: number, index: number): string {
|
||||
return path.join(this.base, String(id), String(index));
|
||||
}
|
||||
|
||||
private tocPath(id: number): string {
|
||||
return path.join(this.base, String(id), TOC_FILENAME);
|
||||
}
|
||||
|
||||
private ensureDir(id: number): void {
|
||||
const dir = path.join(this.base, String(id));
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
tryGetChapters(id: number, from: number, to: number): ChapterView | null {
|
||||
if (from > to) return null;
|
||||
|
||||
const chapters: ChapterFile[] = [];
|
||||
for (let i = from; i <= to; i++) {
|
||||
const p = this.chapterPath(id, i);
|
||||
if (!existsSync(p)) return null;
|
||||
const content = readFileSync(p, 'utf-8');
|
||||
if (content.length < 10) return null;
|
||||
chapters.push({ path: p, hasStyle: looksLikeFragments(content) });
|
||||
}
|
||||
return { id, min: from, max: to, chapters };
|
||||
}
|
||||
|
||||
tryGetToc(id: number): string[] | null {
|
||||
const p = this.tocPath(id);
|
||||
if (!existsSync(p)) return null;
|
||||
const lines = readFileSync(p, 'utf-8')
|
||||
.split('\n')
|
||||
.map(l => l.trim())
|
||||
.filter(Boolean);
|
||||
if (lines.length === 0) return null;
|
||||
return lines;
|
||||
}
|
||||
|
||||
saveChapters(id: number, chapters: { index: number; content: string }[]): ChapterFile[] {
|
||||
this.ensureDir(id);
|
||||
return chapters.map(({ index, content }) => {
|
||||
const p = this.chapterPath(id, index);
|
||||
writeFileSync(p, content, 'utf-8');
|
||||
return { path: p, hasStyle: looksLikeFragments(content) };
|
||||
});
|
||||
}
|
||||
|
||||
saveToc(id: number, urls: string[]): void {
|
||||
this.ensureDir(id);
|
||||
writeFileSync(this.tocPath(id), urls.join('\n'), 'utf-8');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { unzipSync } from 'fflate';
|
||||
import * as cheerio from 'cheerio';
|
||||
import type { Cache } from './cache.ts';
|
||||
import type { ChapterView } from '../types.ts';
|
||||
|
||||
const dec = new TextDecoder();
|
||||
|
||||
function parseContainerXml(xml: string): string {
|
||||
const $ = cheerio.load(xml, { xmlMode: true });
|
||||
const opfPath = $('rootfile').attr('full-path');
|
||||
if (!opfPath) throw new Error('EPUB container.xml is missing full-path attribute');
|
||||
return opfPath;
|
||||
}
|
||||
|
||||
function parseOpfSpine(xml: string): string[] {
|
||||
const $ = cheerio.load(xml, { xmlMode: true });
|
||||
|
||||
const manifest: Record<string, string> = {};
|
||||
$('manifest item').each((_, el) => {
|
||||
const id = $(el).attr('id');
|
||||
const href = $(el).attr('href');
|
||||
if (id && href) manifest[id] = href;
|
||||
});
|
||||
|
||||
const spine: string[] = [];
|
||||
$('spine itemref').each((_, el) => {
|
||||
const idref = $(el).attr('idref');
|
||||
if (idref) {
|
||||
const href = manifest[idref];
|
||||
if (href) spine.push(href);
|
||||
}
|
||||
});
|
||||
|
||||
return spine;
|
||||
}
|
||||
|
||||
function resolveInZip(opfPath: string, relative: string): string {
|
||||
const dir = opfPath.includes('/')
|
||||
? opfPath.slice(0, opfPath.lastIndexOf('/') + 1)
|
||||
: '';
|
||||
return dir + decodeURIComponent(relative);
|
||||
}
|
||||
|
||||
function extractDivText(xhtml: string): string {
|
||||
const $ = cheerio.load(xhtml);
|
||||
const lines: string[] = [];
|
||||
$('div').each((_, el) => {
|
||||
const text = $(el).text().trim();
|
||||
if (text) lines.push(text);
|
||||
});
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export async function fetchEpub(
|
||||
url: string,
|
||||
id: number,
|
||||
from: number,
|
||||
to: number,
|
||||
cache: Cache,
|
||||
onRangeKnown: (startsAt: number, endsAt: number) => void,
|
||||
): Promise<ChapterView> {
|
||||
const cached = cache.tryGetChapters(id, from, to);
|
||||
if (cached) return cached;
|
||||
|
||||
const response = await fetch(url, { redirect: 'follow' });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download EPUB from ${url}: HTTP ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const buffer = await response.arrayBuffer();
|
||||
const zip = unzipSync(new Uint8Array(buffer));
|
||||
|
||||
const containerEntry = zip['META-INF/container.xml'];
|
||||
if (!containerEntry) throw new Error('Invalid EPUB: missing META-INF/container.xml');
|
||||
|
||||
const opfPath = parseContainerXml(dec.decode(containerEntry));
|
||||
const opfEntry = zip[opfPath];
|
||||
if (!opfEntry) throw new Error(`Invalid EPUB: OPF not found at "${opfPath}"`);
|
||||
|
||||
const spineHrefs = parseOpfSpine(dec.decode(opfEntry));
|
||||
if (spineHrefs.length === 0) throw new Error('EPUB spine is empty — no readable chapters found');
|
||||
|
||||
const toSave: { index: number; content: string }[] = [];
|
||||
let chapterIndex = 0;
|
||||
|
||||
for (const href of spineHrefs) {
|
||||
const zipKey = resolveInZip(opfPath, href);
|
||||
const entry = zip[zipKey];
|
||||
if (!entry) continue;
|
||||
|
||||
const text = extractDivText(dec.decode(entry));
|
||||
if (!text.trim()) continue;
|
||||
|
||||
toSave.push({ index: chapterIndex, content: text });
|
||||
chapterIndex++;
|
||||
}
|
||||
|
||||
if (toSave.length === 0) throw new Error('EPUB parsed but all chapters were empty');
|
||||
|
||||
cache.saveChapters(id, toSave);
|
||||
onRangeKnown(0, chapterIndex - 1);
|
||||
|
||||
const effectiveTo = Math.min(to, chapterIndex - 1);
|
||||
const view = cache.tryGetChapters(id, from, effectiveTo);
|
||||
if (!view) throw new Error('Cache read failed immediately after EPUB save — this is a bug');
|
||||
return view;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { Resource, ResourceMeta } from '../types.ts';
|
||||
|
||||
export class ResourceConflictError extends Error {
|
||||
constructor() {
|
||||
super('A resource with that hash is already registered');
|
||||
}
|
||||
}
|
||||
|
||||
export class ResourceNotFoundError extends Error {
|
||||
constructor(id: number) {
|
||||
super(`No resource found with id ${id}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class ResourceRegistry {
|
||||
private readonly byId = new Map<number, Resource>();
|
||||
private readonly hashes = new Set<string>();
|
||||
|
||||
register(resource: Omit<Resource, 'id'>): number {
|
||||
if (this.hashes.has(resource.hash)) throw new ResourceConflictError();
|
||||
|
||||
let id = 0;
|
||||
while (this.byId.has(id)) id++;
|
||||
|
||||
const full: Resource = { ...resource, id };
|
||||
this.byId.set(id, full);
|
||||
this.hashes.add(resource.hash);
|
||||
return id;
|
||||
}
|
||||
|
||||
get(id: number): Resource {
|
||||
const r = this.byId.get(id);
|
||||
if (!r) throw new ResourceNotFoundError(id);
|
||||
return r;
|
||||
}
|
||||
|
||||
getAll(): Record<number, ResourceMeta> {
|
||||
const out: Record<number, ResourceMeta> = {};
|
||||
for (const [id, r] of this.byId) {
|
||||
const { fetch: _fetch, ...meta } = r;
|
||||
out[id] = meta;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
updateRange(id: number, startsAt: number, endsAt: number): void {
|
||||
const r = this.byId.get(id);
|
||||
if (!r) throw new ResourceNotFoundError(id);
|
||||
r.startsAt = startsAt;
|
||||
r.endsAt = endsAt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
import { Builder, type WebDriver } from 'selenium-webdriver';
|
||||
import { Options as FirefoxOptions } from 'selenium-webdriver/firefox.js';
|
||||
import * as cheerio from 'cheerio';
|
||||
import pLimit from 'p-limit';
|
||||
import pRetry from 'p-retry';
|
||||
import { readFileSync } from 'fs';
|
||||
import path from 'path';
|
||||
import type { AppConfig, NovelDefinition } from '../config/types.ts';
|
||||
import type { Cache } from './cache.ts';
|
||||
import type { ChapterView, TextFragment, Style } from '../types.ts';
|
||||
|
||||
export type DriverFactory = () => Promise<WebDriver>;
|
||||
|
||||
function buildDefaultDriverFactory(def: NovelDefinition, appConfig: AppConfig): DriverFactory {
|
||||
return async () => {
|
||||
const options = new FirefoxOptions();
|
||||
|
||||
for (const addonName of (def.stealth?.addons ?? [])) {
|
||||
const addonPath = path.resolve(appConfig.addons.directory, addonName);
|
||||
const addonData = readFileSync(addonPath);
|
||||
options.addExtensions(addonData);
|
||||
}
|
||||
|
||||
return new Builder()
|
||||
.usingServer(appConfig.selenium.remoteUrl)
|
||||
.forBrowser('firefox')
|
||||
.setFirefoxOptions(options)
|
||||
.build();
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchToc(
|
||||
def: NovelDefinition,
|
||||
id: number,
|
||||
cache: Cache,
|
||||
appConfig: AppConfig,
|
||||
driverFactory?: DriverFactory,
|
||||
): Promise<string[]> {
|
||||
const cached = cache.tryGetToc(id);
|
||||
if (cached) return cached;
|
||||
|
||||
const sourceUrl = def.source.url;
|
||||
if (!sourceUrl) throw new Error(`Novel "${def.slug}" is missing source.url for TOC fetch`);
|
||||
|
||||
const tocCfg = def.selectors?.toc;
|
||||
if (!tocCfg) throw new Error(`Novel "${def.slug}" is missing selectors.toc`);
|
||||
|
||||
if (!def.stealth?.enabled) {
|
||||
throw new Error(
|
||||
`Novel "${def.slug}" requires stealth: true — the TOC page cannot be fetched without a real browser and uBlock`,
|
||||
);
|
||||
}
|
||||
|
||||
const createDriver = driverFactory ?? buildDefaultDriverFactory(def, appConfig);
|
||||
const driver = await createDriver();
|
||||
|
||||
try {
|
||||
const urls = await pRetry(
|
||||
async () => {
|
||||
await driver.get(sourceUrl);
|
||||
await driver.sleep(def.stealth!.waitMs);
|
||||
|
||||
// Get the live rendered DOM (post-JS, post-uBlock)
|
||||
const html = (await driver.executeScript(
|
||||
'return document.documentElement.outerHTML',
|
||||
)) as string;
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
const container = $(tocCfg.scope);
|
||||
|
||||
if (!container.length) {
|
||||
throw new Error(
|
||||
`TOC container "${tocCfg.scope}" not found on page — page may not have loaded correctly`,
|
||||
);
|
||||
}
|
||||
|
||||
const pattern = new RegExp(tocCfg.linkPattern);
|
||||
const found: string[] = [];
|
||||
container.find('a').each((_, el) => {
|
||||
const href = $(el).attr('href');
|
||||
if (href && pattern.test(href)) found.push(href);
|
||||
});
|
||||
|
||||
if (found.length === 0) {
|
||||
throw new Error(
|
||||
`No TOC links matching "${tocCfg.linkPattern}" found within "${tocCfg.scope}"`,
|
||||
);
|
||||
}
|
||||
|
||||
return found;
|
||||
},
|
||||
{
|
||||
retries: 3,
|
||||
onFailedAttempt: err =>
|
||||
console.warn(
|
||||
`[scraper] TOC fetch attempt ${err.attemptNumber}/${err.attemptNumber + err.retriesLeft} failed: ${err.message}`,
|
||||
),
|
||||
},
|
||||
);
|
||||
|
||||
cache.saveToc(id, urls);
|
||||
return urls;
|
||||
} finally {
|
||||
await driver.quit();
|
||||
}
|
||||
}
|
||||
|
||||
const errorStyle: Style = {
|
||||
display: 'inline',
|
||||
fontFamily: 'sans-serif',
|
||||
fontSize: '12pt',
|
||||
fontStyle: 'regular',
|
||||
fontWeight: 'bold',
|
||||
color: 'red',
|
||||
};
|
||||
|
||||
export async function fetchChapters(
|
||||
def: NovelDefinition,
|
||||
id: number,
|
||||
from: number,
|
||||
tocUrls: string[],
|
||||
cache: Cache,
|
||||
): Promise<ChapterView> {
|
||||
const to = from + tocUrls.length - 1;
|
||||
|
||||
const cached = cache.tryGetChapters(id, from, to);
|
||||
if (cached) return cached;
|
||||
|
||||
const contentScope = def.selectors?.content?.scope ?? '#chr-content';
|
||||
const minBytes = def.download?.minContentBytes ?? 10;
|
||||
const parallelism = def.download?.parallelism ?? 4;
|
||||
|
||||
const limit = pLimit(parallelism);
|
||||
|
||||
const chapterResults = await Promise.all(
|
||||
tocUrls.map((url, i) =>
|
||||
limit(async (): Promise<{ index: number; content: string }> => {
|
||||
const index = from + i;
|
||||
try {
|
||||
return await pRetry(
|
||||
async () => {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
||||
|
||||
const html = await response.text();
|
||||
const $ = cheerio.load(html);
|
||||
const rawText = $(contentScope).text().trim();
|
||||
|
||||
if (rawText.length < minBytes) {
|
||||
throw new Error(
|
||||
`Content at "${url}" is too short (${rawText.length} bytes, minimum ${minBytes})`,
|
||||
);
|
||||
}
|
||||
|
||||
const fragments: TextFragment[] = rawText
|
||||
.split(/[\n\r]+/)
|
||||
.map(line => line.trim())
|
||||
.filter(Boolean)
|
||||
.map(body => ({ body }));
|
||||
|
||||
return { index, content: JSON.stringify(fragments) };
|
||||
},
|
||||
{
|
||||
retries: 2,
|
||||
onFailedAttempt: err =>
|
||||
console.warn(
|
||||
`[scraper] Chapter ${index} attempt ${err.attemptNumber} failed: ${err.message}`,
|
||||
),
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(`[scraper] Chapter ${index} permanently failed: ${message}`);
|
||||
const errorFragments: TextFragment[] = [
|
||||
{ body: `Chapter ${index} failed to download`, style: errorStyle },
|
||||
{ body: message, style: { ...errorStyle, fontWeight: 'light' } },
|
||||
];
|
||||
return { index, content: JSON.stringify(errorFragments) };
|
||||
}
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const files = cache.saveChapters(id, chapterResults);
|
||||
return { id, min: from, max: to, chapters: files };
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
export interface Style {
|
||||
display: string;
|
||||
fontFamily: string;
|
||||
fontSize: string;
|
||||
fontStyle: string;
|
||||
fontWeight: string;
|
||||
color: string;
|
||||
}
|
||||
|
||||
export const defaultStyle: Style = {
|
||||
display: 'inline-block',
|
||||
fontFamily: 'sans-serif',
|
||||
fontSize: '12pt',
|
||||
fontStyle: 'regular',
|
||||
fontWeight: 'light',
|
||||
color: 'white',
|
||||
};
|
||||
|
||||
export interface TextFragment {
|
||||
body: string;
|
||||
style?: Style;
|
||||
}
|
||||
|
||||
export interface ChapterFile {
|
||||
path: string;
|
||||
hasStyle: boolean;
|
||||
}
|
||||
|
||||
export interface ChapterView {
|
||||
id: number;
|
||||
min: number;
|
||||
max: number;
|
||||
chapters: ChapterFile[];
|
||||
}
|
||||
|
||||
export interface Resource {
|
||||
id: number;
|
||||
hash: string;
|
||||
slug: string;
|
||||
startsAt: number;
|
||||
endsAt: number;
|
||||
url?: string;
|
||||
fetch: (from: number, to: number) => Promise<ChapterView>;
|
||||
}
|
||||
|
||||
export type ResourceMeta = Omit<Resource, 'fetch'>;
|
||||
@@ -0,0 +1,46 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Novel #{id} — Chapters {min}–{max}</title>
|
||||
<style>
|
||||
*, *::before, *::after { box-sizing: border-box; }
|
||||
|
||||
body {
|
||||
background: #111;
|
||||
color: #ddd;
|
||||
font-family: sans-serif;
|
||||
font-size: 14pt;
|
||||
line-height: 1.8;
|
||||
max-width: 820px;
|
||||
margin: 0 auto;
|
||||
padding: 2rem 1.5rem 4rem;
|
||||
}
|
||||
|
||||
header {
|
||||
border-bottom: 1px solid #333;
|
||||
padding-bottom: 0.75rem;
|
||||
margin-bottom: 2rem;
|
||||
color: #888;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
div[id^="chap"] {
|
||||
margin-bottom: 3.5rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid #222;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0.4rem 0;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>id: {id} | chapters {min}–{max}</header>
|
||||
<main>
|
||||
{content}
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { Cache } from '../src/services/cache.ts';
|
||||
import type { TextFragment } from '../src/types.ts';
|
||||
|
||||
let tmpDir: string;
|
||||
let cache: Cache;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), 'qa-cache-test-'));
|
||||
cache = new Cache(tmpDir);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('cache — chapters', () => {
|
||||
test('returns null on a cold cache', () => {
|
||||
expect(cache.tryGetChapters(1, 0, 5)).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null when a chapter in range is missing', () => {
|
||||
cache.saveChapters(1, [
|
||||
{ index: 0, content: 'Chapter zero content here' },
|
||||
// index 1 is intentionally missing
|
||||
{ index: 2, content: 'Chapter two content here' },
|
||||
]);
|
||||
expect(cache.tryGetChapters(1, 0, 2)).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null for invalid range (min > max)', () => {
|
||||
expect(cache.tryGetChapters(1, 5, 2)).toBeNull();
|
||||
});
|
||||
|
||||
test('saves and retrieves plain-text chapters', () => {
|
||||
cache.saveChapters(1, [
|
||||
{ index: 0, content: 'Line one\nLine two' },
|
||||
{ index: 1, content: 'Chapter one content here' },
|
||||
]);
|
||||
const view = cache.tryGetChapters(1, 0, 1);
|
||||
expect(view).not.toBeNull();
|
||||
expect(view!.id).toBe(1);
|
||||
expect(view!.min).toBe(0);
|
||||
expect(view!.max).toBe(1);
|
||||
expect(view!.chapters).toHaveLength(2);
|
||||
expect(view!.chapters[0]!.hasStyle).toBe(false);
|
||||
});
|
||||
|
||||
test('detects styled (JSON fragment) chapters', () => {
|
||||
const fragments: TextFragment[] = [{ body: 'Hello world', style: undefined }];
|
||||
cache.saveChapters(1, [{ index: 0, content: JSON.stringify(fragments) }]);
|
||||
const view = cache.tryGetChapters(1, 0, 0);
|
||||
expect(view).not.toBeNull();
|
||||
expect(view!.chapters[0]!.hasStyle).toBe(true);
|
||||
});
|
||||
|
||||
test('returns null when a chapter file is too short', () => {
|
||||
cache.saveChapters(1, [{ index: 0, content: 'hi' }]); // < 10 bytes
|
||||
expect(cache.tryGetChapters(1, 0, 0)).toBeNull();
|
||||
});
|
||||
|
||||
test('handles single-chapter range', () => {
|
||||
cache.saveChapters(42, [{ index: 7, content: 'Single chapter content ok' }]);
|
||||
const view = cache.tryGetChapters(42, 7, 7);
|
||||
expect(view).not.toBeNull();
|
||||
expect(view!.chapters).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cache — TOC', () => {
|
||||
test('returns null on a cold cache', () => {
|
||||
expect(cache.tryGetToc(1)).toBeNull();
|
||||
});
|
||||
|
||||
test('round-trips a TOC', () => {
|
||||
const urls = [
|
||||
'https://example.com/chapter-1',
|
||||
'https://example.com/chapter-2',
|
||||
'https://example.com/chapter-3',
|
||||
];
|
||||
cache.saveToc(1, urls);
|
||||
expect(cache.tryGetToc(1)).toEqual(urls);
|
||||
});
|
||||
|
||||
test('returns null for an empty TOC file', () => {
|
||||
cache.saveToc(1, []);
|
||||
expect(cache.tryGetToc(1)).toBeNull();
|
||||
});
|
||||
|
||||
test('overwrites an existing TOC on re-save', () => {
|
||||
cache.saveToc(1, ['https://example.com/old']);
|
||||
cache.saveToc(1, ['https://example.com/new-1', 'https://example.com/new-2']);
|
||||
expect(cache.tryGetToc(1)).toEqual([
|
||||
'https://example.com/new-1',
|
||||
'https://example.com/new-2',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
import { describe, test, expect, beforeEach, afterEach, spyOn } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { zipSync } from 'fflate';
|
||||
import { Cache } from '../src/services/cache.ts';
|
||||
import { fetchEpub } from '../src/services/epub.ts';
|
||||
|
||||
const enc = new TextEncoder();
|
||||
|
||||
function buildEpub(chapters: { title: string; content: string }[]): Uint8Array {
|
||||
const files: Record<string, Uint8Array> = {};
|
||||
|
||||
files['META-INF/container.xml'] = enc.encode(
|
||||
`<?xml version="1.0"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>`,
|
||||
);
|
||||
|
||||
const manifestItems = chapters
|
||||
.map((_, i) => `<item id="ch${i}" href="ch${i}.xhtml" media-type="application/xhtml+xml"/>`)
|
||||
.join('\n ');
|
||||
const spineItems = chapters.map((_, i) => `<itemref idref="ch${i}"/>`).join('\n ');
|
||||
|
||||
files['content.opf'] = enc.encode(
|
||||
`<?xml version="1.0"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0">
|
||||
<manifest>
|
||||
${manifestItems}
|
||||
</manifest>
|
||||
<spine>
|
||||
${spineItems}
|
||||
</spine>
|
||||
</package>`,
|
||||
);
|
||||
|
||||
for (let i = 0; i < chapters.length; i++) {
|
||||
const ch = chapters[i]!;
|
||||
files[`ch${i}.xhtml`] = enc.encode(
|
||||
`<html><body><div>${ch.content}</div></body></html>`,
|
||||
);
|
||||
}
|
||||
|
||||
return zipSync(files);
|
||||
}
|
||||
|
||||
let tmpDir: string;
|
||||
let cache: Cache;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), 'qa-epub-test-'));
|
||||
cache = new Cache(tmpDir);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function mockFetch(data: Uint8Array) {
|
||||
return spyOn(globalThis, 'fetch').mockResolvedValue(
|
||||
new Response(data, { status: 200 }),
|
||||
);
|
||||
}
|
||||
|
||||
describe('epub — parsing', () => {
|
||||
test('extracts the correct number of chapters', async () => {
|
||||
const epub = buildEpub([
|
||||
{ title: 'Ch 1', content: 'First chapter content here.' },
|
||||
{ title: 'Ch 2', content: 'Second chapter content here.' },
|
||||
{ title: 'Ch 3', content: 'Third chapter content here.' },
|
||||
]);
|
||||
const spy = mockFetch(epub);
|
||||
|
||||
let rangeStart = -1, rangeEnd = -1;
|
||||
const view = await fetchEpub('http://fake/book.epub', 1, 0, 2, cache, (s, e) => {
|
||||
rangeStart = s; rangeEnd = e;
|
||||
});
|
||||
|
||||
expect(view.chapters).toHaveLength(3);
|
||||
expect(rangeStart).toBe(0);
|
||||
expect(rangeEnd).toBe(2);
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
test('serves from cache on second call without re-fetching', async () => {
|
||||
const epub = buildEpub([
|
||||
{ title: 'Ch 1', content: 'Chapter one long enough content.' },
|
||||
]);
|
||||
const spy = mockFetch(epub);
|
||||
|
||||
await fetchEpub('http://fake/book.epub', 2, 0, 0, cache, () => {});
|
||||
expect(spy).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Second call — should hit cache
|
||||
await fetchEpub('http://fake/book.epub', 2, 0, 0, cache, () => {});
|
||||
expect(spy).toHaveBeenCalledTimes(1);
|
||||
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
test('skips blank chapters', async () => {
|
||||
const epub = buildEpub([
|
||||
{ title: 'Cover', content: ' ' }, // blank — should be skipped
|
||||
{ title: 'Ch 1', content: 'Real content here for this chapter.' },
|
||||
{ title: 'Ch 2', content: 'More real content for the next chapter.' },
|
||||
]);
|
||||
const spy = mockFetch(epub);
|
||||
|
||||
let rangeEnd = -1;
|
||||
await fetchEpub('http://fake/book.epub', 3, 0, 10, cache, (_, e) => { rangeEnd = e; });
|
||||
expect(rangeEnd).toBe(1); // only 2 non-blank chapters (indices 0 and 1)
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
test('throws on HTTP error', async () => {
|
||||
const spy = spyOn(globalThis, 'fetch').mockResolvedValue(
|
||||
new Response('Not Found', { status: 404 }),
|
||||
);
|
||||
await expect(fetchEpub('http://fake/missing.epub', 4, 0, 0, cache, () => {})).rejects.toThrow(
|
||||
'HTTP 404',
|
||||
);
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
test('slices to the requested range', async () => {
|
||||
const epub = buildEpub([
|
||||
{ title: 'Ch 0', content: 'Chapter zero has enough content here.' },
|
||||
{ title: 'Ch 1', content: 'Chapter one has enough content here.' },
|
||||
{ title: 'Ch 2', content: 'Chapter two has enough content here.' },
|
||||
{ title: 'Ch 3', content: 'Chapter three has enough content here.' },
|
||||
]);
|
||||
const spy = mockFetch(epub);
|
||||
|
||||
const view = await fetchEpub('http://fake/book.epub', 5, 1, 2, cache, () => {});
|
||||
expect(view.min).toBe(1);
|
||||
expect(view.max).toBe(2);
|
||||
expect(view.chapters).toHaveLength(2);
|
||||
spy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
import { describe, test, expect, beforeEach } from 'bun:test';
|
||||
import { Hono } from 'hono';
|
||||
import { ResourceRegistry } from '../src/services/registry.ts';
|
||||
import { createNovelRouter } from '../src/routes/novel.ts';
|
||||
import type { ChapterView } from '../src/types.ts';
|
||||
import type { AppConfig } from '../src/config/types.ts';
|
||||
|
||||
// Cache is not exercised by route tests — pass a stub that satisfies the type
|
||||
const stubCache = {} as import('../src/services/cache.ts').Cache;
|
||||
|
||||
const stubAppConfig: AppConfig = {
|
||||
server: { port: 8080 },
|
||||
selenium: { remoteUrl: 'http://selenium:4444/wd/hub', browser: 'firefox' },
|
||||
cache: { basePath: '/tmp' },
|
||||
addons: { directory: './addons' },
|
||||
};
|
||||
|
||||
function makeView(id: number, min: number, max: number): ChapterView {
|
||||
return { id, min, max, chapters: [] };
|
||||
}
|
||||
|
||||
let registry: ResourceRegistry;
|
||||
let app: Hono;
|
||||
|
||||
beforeEach(() => {
|
||||
registry = new ResourceRegistry();
|
||||
const router = createNovelRouter(registry, stubCache, stubAppConfig);
|
||||
app = new Hono();
|
||||
app.route('/qa/requests/novel', router);
|
||||
});
|
||||
|
||||
describe('GET /qa/requests/novel/available', () => {
|
||||
test('returns empty object when nothing registered', async () => {
|
||||
const res = await app.request('/qa/requests/novel/available');
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.json();
|
||||
expect(body).toEqual({});
|
||||
});
|
||||
|
||||
test('returns registered resources (without fetch fn)', async () => {
|
||||
registry.register({
|
||||
hash: 'abc123',
|
||||
slug: 'my-novel',
|
||||
startsAt: 0,
|
||||
endsAt: 100,
|
||||
fetch: async () => makeView(0, 0, 0),
|
||||
});
|
||||
const res = await app.request('/qa/requests/novel/available');
|
||||
const body = await res.json() as Record<string, unknown>;
|
||||
expect(Object.keys(body)).toHaveLength(1);
|
||||
expect((body['0'] as Record<string, unknown>)['slug']).toBe('my-novel');
|
||||
expect((body['0'] as Record<string, unknown>)['fetch']).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /qa/requests/novel/:novel', () => {
|
||||
beforeEach(() => {
|
||||
registry.register({
|
||||
hash: 'deadbeef',
|
||||
slug: 'test-novel',
|
||||
startsAt: 0,
|
||||
endsAt: 9,
|
||||
fetch: async (from, to) => makeView(0, from, to),
|
||||
});
|
||||
});
|
||||
|
||||
test('returns 400 when r param is missing', async () => {
|
||||
const res = await app.request('/qa/requests/novel/0');
|
||||
expect(res.status).toBe(400);
|
||||
const body = await res.json() as { error: string };
|
||||
expect(body.error).toContain('Missing');
|
||||
});
|
||||
|
||||
test('returns 400 for malformed r param', async () => {
|
||||
const res = await app.request('/qa/requests/novel/0?r=notanumber');
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
test('returns 400 when min > max', async () => {
|
||||
const res = await app.request('/qa/requests/novel/0?r=10,5');
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
test('returns 422 for unknown novel id', async () => {
|
||||
const res = await app.request('/qa/requests/novel/999?r=0,5');
|
||||
expect(res.status).toBe(422);
|
||||
const body = await res.json() as { error: string };
|
||||
expect(body.error).toContain('Unknown');
|
||||
});
|
||||
|
||||
test('returns 400 for non-integer novel id', async () => {
|
||||
const res = await app.request('/qa/requests/novel/abc?r=0,5');
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
test('returns 200 html by default', async () => {
|
||||
const res = await app.request('/qa/requests/novel/0?r=0,0');
|
||||
// fetch returns empty chapters so HTML template would be rendered (template read from disk)
|
||||
// We just verify it doesn't 5xx from the route layer itself
|
||||
expect([200, 500]).toContain(res.status);
|
||||
});
|
||||
|
||||
test('returns 200 json when f=json', async () => {
|
||||
const res = await app.request('/qa/requests/novel/0?r=0,0&f=json');
|
||||
expect([200, 500]).toContain(res.status);
|
||||
if (res.status === 200) {
|
||||
expect(res.headers.get('content-type')).toContain('application/json');
|
||||
}
|
||||
});
|
||||
|
||||
test('returns 500 when fetch throws', async () => {
|
||||
registry.register({
|
||||
hash: 'badfetch',
|
||||
slug: 'broken-novel',
|
||||
startsAt: 0,
|
||||
endsAt: 0,
|
||||
fetch: async () => { throw new Error('something went wrong internally'); },
|
||||
});
|
||||
const res = await app.request('/qa/requests/novel/1?r=0,0');
|
||||
expect(res.status).toBe(500);
|
||||
const body = await res.json() as { error: string; detail: string };
|
||||
expect(body.error).toBeDefined();
|
||||
expect(body.detail).toBe('something went wrong internally');
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /qa/requests/novel/new', () => {
|
||||
test('returns 400 for non-JSON body', async () => {
|
||||
const res = await app.request('/qa/requests/novel/new', {
|
||||
method: 'POST',
|
||||
body: 'not json',
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
test('returns 400 when type is missing', async () => {
|
||||
const res = await app.request('/qa/requests/novel/new', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ slug: 'test', urls: ['http://x.com/book.epub'] }),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
test('returns 400 for unsupported type', async () => {
|
||||
const res = await app.request('/qa/requests/novel/new', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ type: 'novel', slug: 'test', urls: ['http://x.com'] }),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
const body = await res.json() as { error: string };
|
||||
expect(body.error).toContain('Unsupported');
|
||||
});
|
||||
|
||||
test('registers an epub and returns numeric id', async () => {
|
||||
const res = await app.request('/qa/requests/novel/new', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ type: 'epub', slug: 'my-book', urls: ['http://x.com/book.epub'] }),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const text = await res.text();
|
||||
expect(parseInt(text, 10)).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test('returns 409 on duplicate registration', async () => {
|
||||
const body = JSON.stringify({ type: 'epub', slug: 'dup', urls: ['http://x.com/dup.epub'] });
|
||||
const headers = { 'Content-Type': 'application/json' };
|
||||
await app.request('/qa/requests/novel/new', { method: 'POST', body, headers });
|
||||
const res = await app.request('/qa/requests/novel/new', { method: 'POST', body, headers });
|
||||
expect(res.status).toBe(409);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import { describe, test, expect, beforeEach, afterEach, mock, spyOn } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { Cache } from '../src/services/cache.ts';
|
||||
import { fetchToc, fetchChapters, type DriverFactory } from '../src/services/scraper.ts';
|
||||
import type { NovelDefinition, AppConfig } from '../src/config/types.ts';
|
||||
|
||||
const DEF: NovelDefinition = {
|
||||
slug: 'test-novel',
|
||||
type: 'web-scraped',
|
||||
source: { url: 'https://example.com/novel' },
|
||||
selectors: {
|
||||
toc: { scope: '#chapters', linkPattern: 'https://example\\.com/chapter/.*' },
|
||||
content: { scope: '#content' },
|
||||
},
|
||||
stealth: { enabled: true, waitMs: 0, addons: [] },
|
||||
download: { parallelism: 2, minContentBytes: 5 },
|
||||
};
|
||||
|
||||
const APP_CONFIG: AppConfig = {
|
||||
server: { port: 8080 },
|
||||
selenium: { remoteUrl: 'http://selenium:4444/wd/hub', browser: 'firefox' },
|
||||
cache: { basePath: '/tmp' },
|
||||
addons: { directory: './addons' },
|
||||
};
|
||||
|
||||
let tmpDir: string;
|
||||
let cache: Cache;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), 'qa-scraper-test-'));
|
||||
cache = new Cache(tmpDir);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function buildMockPage(links: string[]): string {
|
||||
const anchors = links.map(href => `<a href="${href}">link</a>`).join('');
|
||||
return `<html><body><div id="chapters">${anchors}</div></body></html>`;
|
||||
}
|
||||
|
||||
function makeMockDriver(html: string): { driver: ReturnType<DriverFactory> extends Promise<infer T> ? T : never; driverFactory: DriverFactory; quitSpy: ReturnType<typeof mock> } {
|
||||
const quitSpy = mock(() => Promise.resolve());
|
||||
const driver = {
|
||||
get: mock((_url: string) => Promise.resolve()),
|
||||
sleep: mock((_ms: number) => Promise.resolve()),
|
||||
executeScript: mock((_script: string) => Promise.resolve(html)),
|
||||
quit: quitSpy,
|
||||
} as unknown as Awaited<ReturnType<DriverFactory>>;
|
||||
const driverFactory: DriverFactory = () => Promise.resolve(driver);
|
||||
return { driver, driverFactory, quitSpy };
|
||||
}
|
||||
|
||||
describe('fetchToc', () => {
|
||||
test('returns cached TOC without opening a browser', async () => {
|
||||
const urls = ['https://example.com/chapter/1', 'https://example.com/chapter/2'];
|
||||
cache.saveToc(0, urls);
|
||||
|
||||
const driverFactory = mock(() => Promise.reject(new Error('should not be called')));
|
||||
const result = await fetchToc(DEF, 0, cache, APP_CONFIG, driverFactory as unknown as DriverFactory);
|
||||
expect(result).toEqual(urls);
|
||||
expect(driverFactory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('fetches TOC via browser when cache is cold', async () => {
|
||||
const links = ['https://example.com/chapter/1', 'https://example.com/chapter/2'];
|
||||
const { driverFactory, quitSpy } = makeMockDriver(buildMockPage(links));
|
||||
|
||||
const result = await fetchToc(DEF, 1, cache, APP_CONFIG, driverFactory);
|
||||
expect(result).toEqual(links);
|
||||
expect(quitSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('filters links by linkPattern', async () => {
|
||||
const html = `<html><body><div id="chapters">
|
||||
<a href="https://example.com/chapter/1">ch1</a>
|
||||
<a href="https://other.com/chapter/2">other</a>
|
||||
<a href="https://example.com/chapter/3">ch3</a>
|
||||
</div></body></html>`;
|
||||
const { driverFactory } = makeMockDriver(html);
|
||||
|
||||
const result = await fetchToc(DEF, 2, cache, APP_CONFIG, driverFactory);
|
||||
expect(result).toEqual(['https://example.com/chapter/1', 'https://example.com/chapter/3']);
|
||||
});
|
||||
|
||||
// Extended timeout: pRetry's default exponential backoff across 4 attempts can take ~7s
|
||||
test('quits driver even on failure', async () => {
|
||||
const { driverFactory, quitSpy } = makeMockDriver('<html><body></body></html>');
|
||||
await expect(fetchToc(DEF, 3, cache, APP_CONFIG, driverFactory)).rejects.toThrow();
|
||||
expect(quitSpy).toHaveBeenCalledTimes(1);
|
||||
}, 15000);
|
||||
|
||||
test('finds anchors nested at arbitrary depth within the scope element', async () => {
|
||||
// Simulates a TOC where links are buried inside ul > li > span > a
|
||||
// rather than being direct children of the scope container.
|
||||
const html = `<html><body><div id="chapters">
|
||||
<ul>
|
||||
<li><span><a href="https://example.com/chapter/1">deep link 1</a></span></li>
|
||||
<li>
|
||||
<div class="wrapper">
|
||||
<div class="inner">
|
||||
<a href="https://example.com/chapter/2">deep link 2</a>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</div></body></html>`;
|
||||
const { driverFactory } = makeMockDriver(html);
|
||||
const result = await fetchToc(DEF, 5, cache, APP_CONFIG, driverFactory);
|
||||
expect(result).toEqual([
|
||||
'https://example.com/chapter/1',
|
||||
'https://example.com/chapter/2',
|
||||
]);
|
||||
});
|
||||
|
||||
test('saves fetched TOC to cache', async () => {
|
||||
const links = ['https://example.com/chapter/1'];
|
||||
const { driverFactory } = makeMockDriver(buildMockPage(links));
|
||||
await fetchToc(DEF, 4, cache, APP_CONFIG, driverFactory);
|
||||
expect(cache.tryGetToc(4)).toEqual(links);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchChapters', () => {
|
||||
test('returns cached view without fetching', async () => {
|
||||
const content = JSON.stringify([{ body: 'Chapter content here.' }]);
|
||||
cache.saveChapters(10, [{ index: 0, content }]);
|
||||
|
||||
const fetchSpy = spyOn(globalThis, 'fetch');
|
||||
const view = await fetchChapters(DEF, 10, 0, ['https://example.com/chapter/1'], cache);
|
||||
expect(view.chapters).toHaveLength(1);
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
fetchSpy.mockRestore();
|
||||
});
|
||||
|
||||
test('downloads chapters and saves them', async () => {
|
||||
// mockImplementation (not mockResolvedValue) so each call gets a fresh Response —
|
||||
// reusing the same instance causes "Body already used" on retry reads.
|
||||
const spy = spyOn(globalThis, 'fetch').mockImplementation(() =>
|
||||
Promise.resolve(new Response('<html><body><div id="content">Chapter body text here.</div></body></html>', { status: 200 })),
|
||||
);
|
||||
|
||||
const view = await fetchChapters(
|
||||
DEF,
|
||||
11,
|
||||
0,
|
||||
['https://example.com/chapter/1', 'https://example.com/chapter/2'],
|
||||
cache,
|
||||
);
|
||||
|
||||
expect(view.min).toBe(0);
|
||||
expect(view.max).toBe(1);
|
||||
expect(view.chapters).toHaveLength(2);
|
||||
expect(spy).toHaveBeenCalledTimes(2);
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
test('stores error fragment when chapter content is too short', async () => {
|
||||
const spy = spyOn(globalThis, 'fetch').mockImplementation(() =>
|
||||
Promise.resolve(new Response('<html><body><div id="content"></div></body></html>', { status: 200 })),
|
||||
);
|
||||
|
||||
const view = await fetchChapters(DEF, 12, 0, ['https://example.com/chapter/1'], cache);
|
||||
expect(view.chapters).toHaveLength(1);
|
||||
|
||||
// The saved content should be the error fragment JSON
|
||||
const saved = cache.tryGetChapters(12, 0, 0);
|
||||
expect(saved).not.toBeNull();
|
||||
spy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
// Environment setup & latest features
|
||||
"lib": ["ESNext"],
|
||||
"target": "ESNext",
|
||||
"module": "Preserve",
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"allowJs": true,
|
||||
|
||||
// Bundler mode
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"noEmit": true,
|
||||
|
||||
// Best practices
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
|
||||
// Some stricter flags (disabled by default)
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"noPropertyAccessFromIndexSignature": false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user