From d61207b6994e28e650f11e862b129404359af8c1 Mon Sep 17 00:00:00 2001 From: Laimuslime <208247554+Laimuslime@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:13:31 +0000 Subject: [PATCH] feat: replace resident workers with serverless jobs Stop keeping node-cron and systemd processes alive. Crawl and attribution now run as short slices with cursors in Postgres, triggered by Vercel Cron or GitHub Actions. Site config and crawler logs live in the database so a read-only filesystem still works. --- .env.example | 15 +- .github/workflows/serverless-jobs.yml | 126 ++++++ README.md | 53 ++- auto-attribution.js | 167 ++------ auto-crawler.js | 533 ++---------------------- deploy/wikitdb-attribution.service | 14 +- jobs/attribution.js | 183 ++++++++ jobs/crawler.js | 576 ++++++++++++++++++++++++++ lib/prisma.js | 40 +- middleware.js | 79 ++-- next.config.js | 3 + package-lock.json | 10 - package.json | 4 +- pages/admin.js | 14 +- pages/api/admin/crawler-status.js | 7 +- pages/api/admin/file-logs.js | 73 ++-- pages/api/admin/sites.js | 25 +- pages/api/cron/attribution.js | 21 + pages/api/cron/crawl.js | 29 ++ pages/api/tools/delete-bot.js | 6 + prisma/schema.prisma | 3 +- tests/cronAuth.test.js | 42 ++ utils/cache.js | 50 +-- utils/cronAuth.js | 38 ++ utils/jobLock.js | 53 +++ utils/jobLog.js | 52 +++ utils/siteConfig.js | 75 +++- vercel.json | 12 + 28 files changed, 1482 insertions(+), 821 deletions(-) create mode 100644 .github/workflows/serverless-jobs.yml create mode 100644 jobs/attribution.js create mode 100644 jobs/crawler.js create mode 100644 pages/api/cron/attribution.js create mode 100644 pages/api/cron/crawl.js create mode 100644 tests/cronAuth.test.js create mode 100644 utils/cronAuth.js create mode 100644 utils/jobLock.js create mode 100644 utils/jobLog.js create mode 100644 vercel.json diff --git a/.env.example b/.env.example index 0fe4cbf..bce11bf 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,17 @@ -DATABASE_URL=postgresql://user:password@localhost:5432/wikitdb +# 定时任务密钥(至少 16 字符)。Vercel Cron 和 GitHub Actions 用它调用 /api/cron/*。 +# 生产环境必须设置,否则调度接口一律 401。 +CRON_SECRET= + +# 可选:Upstash Redis。配上之后全局限速在所有无服务器实例之间共享。 +# 不配则只在单个实例内计数。 +# UPSTASH_REDIS_REST_URL= +# UPSTASH_REDIS_REST_TOKEN= + +# 无服务器函数不要用「直连 + 大连接池」。 +# Neon / Supabase 请用带 pgbouncer 的 pooler 连接串,并带上 connection_limit=1。 +# 在 Vercel 上如果漏了 connection_limit,运行时会自动补上。 +DATABASE_URL=postgresql://user:password@localhost:5432/wikitdb?connection_limit=1 + JWT_SECRET=replace-with-at-least-32-random-bytes SITE_ORIGIN=https://example.com TRUST_PROXY=false diff --git a/.github/workflows/serverless-jobs.yml b/.github/workflows/serverless-jobs.yml new file mode 100644 index 0000000..9a0d8b8 --- /dev/null +++ b/.github/workflows/serverless-jobs.yml @@ -0,0 +1,126 @@ +name: Serverless jobs + +# 替代原来的常驻 worker / systemd。 +# 爬虫和归属同步优先在 Actions 里跑(单次最长约 8 分钟,游标写在数据库里)。 +# 没配 DATABASE_URL 时,改为请求已部署站点的 /api/cron/*(需要 SITE_ORIGIN 与 CRON_SECRET)。 +# 每天 12:00(Asia/Shanghai,04:00 UTC)用 wikit CLI 打备份包并上传 artifact。 + +on: + schedule: + - cron: '*/10 * * * *' + - cron: '0 4 * * *' + workflow_dispatch: + inputs: + job: + description: Which job to run + required: true + default: crawl + type: choice + options: + - crawl + - attribution + - backup + +permissions: + contents: read + +jobs: + crawl: + if: >- + (github.event_name == 'workflow_dispatch' && inputs.job == 'crawl') || + (github.event_name == 'schedule' && github.event.schedule == '*/10 * * * *') + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + WIKIDOT_BOT_USER: ${{ secrets.WIKIDOT_BOT_USER }} + WIKIDOT_BOT_PASS: ${{ secrets.WIKIDOT_BOT_PASS }} + SITE_ORIGIN: ${{ vars.SITE_ORIGIN }} + CRON_SECRET: ${{ secrets.CRON_SECRET }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + - name: Crawl with database + if: env.DATABASE_URL != '' + run: | + npm ci + npx prisma generate + node auto-crawler.js --for 480 + - name: Crawl via deployed cron endpoint + if: env.DATABASE_URL == '' && env.SITE_ORIGIN != '' && env.CRON_SECRET != '' + run: | + curl -fsS -H "Authorization: Bearer $CRON_SECRET" "$SITE_ORIGIN/api/cron/crawl" + - name: Skip notice + if: env.DATABASE_URL == '' && (env.SITE_ORIGIN == '' || env.CRON_SECRET == '') + run: echo "未配置 DATABASE_URL,也没有 SITE_ORIGIN + CRON_SECRET,跳过爬虫。在仓库 Secrets / Variables 里补上即可。" + + attribution: + if: >- + (github.event_name == 'workflow_dispatch' && inputs.job == 'attribution') || + (github.event_name == 'schedule' && github.event.schedule == '*/10 * * * *') + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + SITE_ORIGIN: ${{ vars.SITE_ORIGIN }} + CRON_SECRET: ${{ secrets.CRON_SECRET }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + - name: Attribution with database + if: env.DATABASE_URL != '' + run: | + npm ci + npx prisma generate + node auto-attribution.js --for 480 + - name: Attribution via deployed cron endpoint + if: env.DATABASE_URL == '' && env.SITE_ORIGIN != '' && env.CRON_SECRET != '' + run: | + curl -fsS -H "Authorization: Bearer $CRON_SECRET" "$SITE_ORIGIN/api/cron/attribution" + - name: Skip notice + if: env.DATABASE_URL == '' && (env.SITE_ORIGIN == '' || env.CRON_SECRET == '') + run: echo "未配置归属同步所需的 DATABASE_URL 或 SITE_ORIGIN + CRON_SECRET,跳过。" + + backup: + if: >- + (github.event_name == 'workflow_dispatch' && inputs.job == 'backup') || + (github.event_name == 'schedule' && github.event.schedule == '0 4 * * *') + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + - name: Skip notice + if: env.DATABASE_URL == '' + run: echo "备份需要 Secrets.DATABASE_URL 来读取站点列表。未配置,跳过。" + - name: Install wikit CLI + if: env.DATABASE_URL != '' + run: | + curl -fsSL -o /tmp/wikit "https://github.com/WikitTeam/wikit/releases/download/v0.2.3/wikit-linux-amd64" + chmod +x /tmp/wikit + sudo mv /tmp/wikit /usr/local/bin/wikit + - name: Run backup + if: env.DATABASE_URL != '' + run: | + npm ci + npx prisma generate + node auto-crawler.js --backup + - uses: actions/upload-artifact@v4 + if: env.DATABASE_URL != '' + with: + name: wikit-backup + path: wikit_data + if-no-files-found: ignore + retention-days: 14 + diff --git a/README.md b/README.md index 328a93f..b98d045 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ WikitDB 是一个面向 Wikidot 社区生态的**非营利性同人项目**。 | 论坛渲染 | FTML (Wikidot 文本标记语言) WASM 内核 | | HTML 消毒 | sanitize-html | | 鉴权 | JWT (HS256) + bcrypt + httpOnly Cookie | -| 定时任务 | node-cron | +| 定时任务 | GitHub Actions(`.github/workflows/serverless-jobs.yml`)或 Vercel Cron | | 邮件 | nodemailer | | 缓存 | Upstash Redis (可选) | | 语言 | JavaScript (逐步迁移至 TypeScript) | @@ -86,39 +86,52 @@ npm run dev 默认监听 `http://localhost:3000`。 -## 生产部署 +## 生产部署(无服务器) -### 构建 +站点本身是 Next.js,部署到 Vercel 后每个页面和 API 都是按需启动的函数,**不要再跑 `npm start` + PM2,也不要再跑常驻 worker**。 -```bash -npm run build -npm start -``` +状态都在 Postgres 里:爬虫游标(`crawler:cursor`)、运行状态(`crawler:status`)、任务日志(`logs:crawler`)、管理后台改过的站点列表(`site:config`)。函数结束不影响进度。 -### 备份 Worker(可选) +### 1. 部署 Web -如需启用 Wikidot 站点定时备份,需将 wikit CLI 安装到服务器 PATH 中,并以常驻进程方式运行 worker: +把仓库接到 Vercel(或任何支持 Next.js 的无服务器平台),配上: -```bash -npm run worker -``` +- `DATABASE_URL` — 用连接池地址(Neon / Supabase pooler),带 `connection_limit=1` +- `JWT_SECRET`、`SITE_ORIGIN` +- `CRON_SECRET` — 至少 16 字符,调度接口靠它鉴权 +- `UPSTASH_REDIS_REST_URL` / `UPSTASH_REDIS_REST_TOKEN` — 可选,配了之后限速跨实例共享 + +`vercel.json` 里有两条每天一次的 Cron(Hobby 计划不允许更密)。真正的推进频率靠下面的 GitHub Actions。 + +### 2. 调度爬虫和归属同步 -Worker 会在每天 12:00(Asia/Shanghai 时区)对所有 `wikitdb.config.js` 中声明的站点执行备份,归档写入工作目录下的 `wikit_data/`。任务限定为单进程串行执行,且服务端始终传递 `--keep-removed`,确保从 sitemap 消失的页面仍保留在归档中。 +`.github/workflows/serverless-jobs.yml` 每 10 分钟跑一片(最长约 8 分钟),游标没跑完下次接着跑。爬虫一轮结束后冷却 3 小时,归属同步冷却 30 分钟。 -### 进程管理 +在仓库 Secrets 里设置 `DATABASE_URL`(以及可选的 `WIKIDOT_BOT_USER` / `WIKIDOT_BOT_PASS`)。没有数据库密钥时,可以改成调用线上接口:仓库 Variables 设 `SITE_ORIGIN`,Secrets 设 `CRON_SECRET`。 -推荐使用 PM2 守护 `npm start` 与 `npm run worker` 两个进程: +本地手动跑一整轮或一片: ```bash -pm2 start "npm start" --name wikitdb -pm2 start "npm run worker" --name wikitdb-worker -pm2 save +node auto-crawler.js # 把当前一轮爬完后退出 +node auto-crawler.js --for 480 # 最多 480 秒 +node auto-attribution.js --slice ``` +`npm run worker` 仍指向爬虫脚本,但**跑完就退出**,不要再用 PM2 守护它。 + +### 3. 全站备份 + +`wikit` CLI 要二进制和磁盘,不能塞进函数。每天 12:00(北京时间)由同一条 Actions 工作流下载 `wikit`、执行 `node auto-crawler.js --backup`,把 `wikit_data/` 收成 artifact。 + +### 4. 仍然想自建虚拟机 + +可以 `npm run build && npm start`,但调度请用系统 cron 调用上面的一次性命令,不要再装 `deploy/wikitdb-attribution.service` 那种常驻进程。 + ## 项目结构 ``` WikitDB-Server/ +├── jobs/ # 无服务器任务切片(爬虫 / 归属 / 备份) ├── pages/ # Next.js Pages Router │ ├── api/ # API 路由 │ ├── tools/ # 工具页面 @@ -148,7 +161,9 @@ WikitDB-Server/ | `npm run typecheck` | TypeScript 类型检查 (`tsc --noEmit`) | | `npm test` | 运行单元测试 (`node --test tests/*.test.js`) | | `npm run test:ftml` | FTML 运行时测试 | -| `npm run worker` | 启动备份定时任务 worker | +| `npm run worker` / `npm run job:crawl` | 跑一轮爬虫后退出(不再常驻) | +| `npm run job:attribution` | 跑一轮归属同步后退出 | +| `npm run job:backup` | 调用本机 wikit CLI 做备份 | | `npm run prisma:generate` | 生成 Prisma Client | | `npm run prisma:push` | 同步 schema 到数据库 | | `npm run prisma:studio` | 打开 Prisma Studio 可视化管理数据库 | diff --git a/auto-attribution.js b/auto-attribution.js index b48cf8b..61083d2 100644 --- a/auto-attribution.js +++ b/auto-attribution.js @@ -1,148 +1,39 @@ /** - * WikitDB 归属资料独立定时任务 - * 与主爬虫(auto-crawler.js 逐页爬取)完全解耦: - * 每 30 分钟单独抓取各站点的「归属资料页」+ listpages 页面评分, - * 同步 author_attributions 表并聚合作者分数(author_score:{site})。 + * 一次性归属同步。跑完退出。 + * + * node auto-attribution.js + * node auto-attribution.js --for 300 + * node auto-attribution.js --slice + * node auto-attribution.js --force */ -const cron = require('node-cron'); -const http = require('http'); -const https = require('https'); -const fs = require('fs'); -const path = require('path'); -const prisma = require('./lib/prisma'); -const config = require('./wikitdb.config.js'); -const { fetchAttributionPage, aggregateAuthorScores, normalizePageKey } = require('./utils/attribution'); +const { runAttributionSlice } = require('./jobs/attribution'); -const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); -const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 }); -const request = require('axios').create({ - httpAgent, - httpsAgent, - timeout: 30000, - validateStatus: (s) => s >= 200 && s < 400 -}); - -const LOG_FILE = path.join(process.cwd(), 'crawler.log'); -const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); - -function logLine(...args) { - const line = args.map(String).join(' '); - console.log(line); - try { - fs.appendFileSync(LOG_FILE, `[${new Date().toLocaleString()}] ${line}\n`, 'utf8'); - } catch (e) { /* 忽略 */ } -} - -/** 获取站点全部页面及评分(listpages 分页) */ -async function fetchAllPages(siteConfig) { - const actualWikiName = siteConfig.URL.replace(/^https?:\/\//i, '').replace(/^www\./i, '').split('.')[0]; - const baseUrl = siteConfig.URL.replace(/\/$/, ''); - const pages = []; - let pageNum = 1; - let totalPages = 1; - - while (true) { - const res = await request.get(`https://wikit.unitreaty.org/listpages?wiki=${actualWikiName}&p=${pageNum}`); - const lines = String(res.data).split('\n').map(l => l.trim()).filter(Boolean); - - const totalMatch = lines.find(l => l.startsWith('Total Pages:'))?.match(/(\d+)/); - if (totalMatch) totalPages = parseInt(totalMatch[1], 10) || 1; - - let countThisPage = 0; - for (const line of lines) { - if (!line.startsWith('http')) continue; - const parts = line.split('|').map(s => s.trim()); - if (parts.length < 6) continue; - const pageSlug = parts[0].split('/').pop(); - pages.push({ - page: pageSlug, - title: parts[1] || '', - rating: parseInt(parts[3], 10) || 0, - upvotes: parseInt(parts[4], 10) || 0, - downvotes: parseInt(parts[5], 10) || 0 - }); - countThisPage++; - } - - if (pageNum >= totalPages || countThisPage === 0) break; - pageNum++; - await sleep(400); - } - return pages; -} - -/** 处理单个站点:抓归属页 + 页面评分 + 聚合作者分数 */ -async function processSite(siteConfig) { - const wikiParam = siteConfig.PARAM; - const attrRecords = await fetchAttributionPage(siteConfig, request); - if (attrRecords.length === 0) { - logLine(`[归属] ${wikiParam} 归属资料页无有效数据,跳过`); - return; - } - - // 1. 全量同步归属记录 - await prisma.$transaction([ - prisma.authorAttribution.deleteMany({ where: { siteParam: wikiParam } }), - prisma.authorAttribution.createMany({ - data: attrRecords.map(a => ({ siteParam: wikiParam, page: a.page, username: a.username, type: a.type, date: a.date })), - skipDuplicates: true - }) - ]); - - // 2. 获取页面评分(listpages) - const allPages = await fetchAllPages(siteConfig); - - // 3. 保存页面评分 - const pageScoreRows = allPages.map(p => ({ - page: p.page, title: p.title, rating: p.rating, - upvotes: p.upvotes, downvotes: p.downvotes - })); - await prisma.setting.upsert({ - where: { key: `page_scores:${wikiParam}` }, - update: { value: JSON.stringify(pageScoreRows) }, - create: { key: `page_scores:${wikiParam}`, value: JSON.stringify(pageScoreRows) } - }); - - // 4. 按归属聚合作者分数(每个归属用户获得页面全分) - const pageRatings = new Map(); - for (const pg of allPages) pageRatings.set(normalizePageKey(pg.page), { rating: pg.rating }); - const authorScores = aggregateAuthorScores(attrRecords, pageRatings); - await prisma.setting.upsert({ - where: { key: `author_score:${wikiParam}` }, - update: { value: JSON.stringify(authorScores) }, - create: { key: `author_score:${wikiParam}`, value: JSON.stringify(authorScores) } - }); - - logLine(`[归属] ${wikiParam} 归属 ${attrRecords.length} 条 | 页面评分 ${allPages.length} | 作者 ${Object.keys(authorScores).length} 位`); +function argNumber(flag) { + const index = process.argv.indexOf(flag); + if (index === -1 || process.argv[index + 1] == null) return null; + const value = Number(process.argv[index + 1]); + return Number.isFinite(value) ? value : null; } -let isRunning = false; - -async function runAttribution() { - if (isRunning) { - logLine('[归属] 上一轮尚未结束,跳过本次触发'); +async function main() { + const force = process.argv.includes('--force'); + if (process.argv.includes('--slice')) { + console.log(JSON.stringify(await runAttributionSlice({ force }))); return; } - isRunning = true; - const start = Date.now(); - try { - const sites = config.SUPPORT_WIKI.filter(w => w.ATTRIBUTION_PAGE); - logLine(`[归属] 开始执行(${sites.length} 个站点配置了归属资料页)...`); - for (const siteConfig of sites) { - try { - await processSite(siteConfig); - } catch (e) { - logLine(`[归属] ${siteConfig.PARAM} 处理失败: ${e.message}`); - } - } - logLine(`[归属] 执行完成,耗时 ${Math.round((Date.now() - start) / 1000)}s`); - } catch (e) { - logLine(`[归属] 发生异常: ${e.message}`); - } finally { - isRunning = false; + const forSec = argNumber('--for'); + const deadline = forSec == null ? Infinity : Date.now() + forSec * 1000; + let last = null; + while (Date.now() < deadline) { + last = await runAttributionSlice({ force }); + if (!last || last.skipped || last.done || last.error) break; } + console.log(JSON.stringify(last)); } -// 每 30 分钟执行一次;启动时立即执行一轮 -cron.schedule('*/30 * * * *', () => runAttribution()); -runAttribution(); +main() + .then(() => process.exit(process.exitCode || 0)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/auto-crawler.js b/auto-crawler.js index 096ae16..fac2710 100644 --- a/auto-crawler.js +++ b/auto-crawler.js @@ -1,506 +1,49 @@ -const cron = require('node-cron'); -const axios = require('axios'); -const cheerio = require('cheerio'); -const http = require('http'); -const https = require('https'); -const { spawn } = require('child_process'); -const prisma = require('./lib/prisma'); -const config = require('./wikitdb.config.js'); -const { buildBackupArgs, getWikiName } = require('./utils/wikitBackup'); -const fs = require('fs'); -const path = require('path'); - -const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); -const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 }); -const request = axios.create({ - httpAgent, - httpsAgent, - timeout: 15000 -}); - -const sleep = ms => new Promise(r => setTimeout(r, ms)); - -const CRAWLER_LOG_FILE = path.join(process.cwd(), 'crawler.log'); - -/** 同时输出到控制台和 crawler.log,方便管理后台查看 */ -function logLine(...args) { - const line = args.map(String).join(' '); - console.log(line); - try { - fs.appendFileSync(CRAWLER_LOG_FILE, `[${new Date().toLocaleString()}] ${line}\n`, 'utf8'); - } catch (e) { /* 日志文件写入失败不影响主流程 */ } -} - -const CRAWLER_STATUS_KEY = 'crawler:status'; - -let crawlStatus = { - running: false, - startedAt: null, - finishedAt: null, - currentSite: null, - currentStage: '', - overall: { totalSites: 0, doneSites: 0 }, - sites: [], - lastRun: null -}; - -/** 将当前爬取状态持久化到数据库(管理后台 /api/admin/crawler-status 读取) */ -async function persistCrawlStatus() { - try { - await prisma.setting.upsert({ - where: { key: CRAWLER_STATUS_KEY }, - update: { value: JSON.stringify(crawlStatus) }, - create: { key: CRAWLER_STATUS_KEY, value: JSON.stringify(crawlStatus) } - }); - } catch (e) { - console.error(`[crawler-status] 状态持久化失败: ${e.message}`); - } -} - -function buildSiteStatus(siteConfig) { - return { - param: siteConfig.PARAM, - name: siteConfig.NAME, - status: 'pending', - pagesFound: 0, - pagesProcessed: 0, - votes: 0, - discussions: 0, - errors: 0, - startedAt: null, - finishedAt: null, - lastRun: crawlStatus.lastRun, - error: null - }; -} - -let botCookieCache = null; - -async function getBotCookie() { - if (botCookieCache) return botCookieCache; - const user = process.env.WIKIDOT_BOT_USER; - const pass = process.env.WIKIDOT_BOT_PASS; - if (!user || !pass) { - console.log("未配置机器人账号,将以访客身份进行抓取..."); - return null; - } - - try { - const payload = new URLSearchParams({ login: user, password: pass, action: 'Login2Action', event: 'login' }); - const res = await axios.post('https://www.wikidot.com/default--flow/login__LoginPopupScreen', payload.toString(), { - headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'WikitDB-Bot/1.0' }, - maxRedirects: 0, - validateStatus: status => status >= 200 && status < 400 - }); - let sessionId = ''; - const cookies = res.headers['set-cookie'] || []; - for (const c of cookies) { - if (c.includes('WIKIDOT_SESSION_ID=')) { - sessionId = c.split('WIKIDOT_SESSION_ID=')[1].split(';')[0]; - } - } - if (sessionId) { - botCookieCache = `WIKIDOT_SESSION_ID=${sessionId}; wikidot_token7=123456;`; - console.log("机器人账号登录成功,已获取受限站点抓取权限。"); - } - } catch (e) { - console.error('获取 Bot Cookie 失败:', e.message); - } - return botCookieCache; +/** + * 一次性爬虫 / 备份入口。进程跑完就退出,不再常驻,也不再内置 node-cron。 + * + * node auto-crawler.js 把当前这一轮爬完(切片续跑)后退出 + * node auto-crawler.js --for 480 最多跑 480 秒,游标留下次继续 + * node auto-crawler.js --slice 只跑一个默认时间片(约 45 秒) + * node auto-crawler.js --backup 调 wikit CLI 做全站备份(不能在 Vercel 函数里跑) + * node auto-crawler.js --force 忽略 3 小时冷却,立刻开新一轮 + */ +const { runCrawlSlice, runBackup } = require('./jobs/crawler'); + +function argNumber(flag) { + const index = process.argv.indexOf(flag); + if (index === -1 || process.argv[index + 1] == null) return null; + const value = Number(process.argv[index + 1]); + return Number.isFinite(value) ? value : null; } -let isRunning = false; - -async function runCrawler() { - if (isRunning) { - logLine(`[${new Date().toLocaleString()}] 警告:上一轮爬虫尚未结束,跳过本次触发。`); +async function main() { + const force = process.argv.includes('--force'); + if (process.argv.includes('--backup')) { + const result = await runBackup(); + console.log(JSON.stringify(result)); + if (result && result.exitCode && result.exitCode !== 0) process.exitCode = result.exitCode; return; } - isRunning = true; - - try { - const botCookie = await getBotCookie(); - const baseHeaders = { 'User-Agent': 'Mozilla/5.0' }; - if (botCookie) baseHeaders['Cookie'] = botCookie; - - logLine(`\n[${new Date().toLocaleString()}] 开始执行全站数据(评分+讨论区)爬取...`); - crawlStatus = { - running: true, - startedAt: Date.now(), - finishedAt: null, - currentSite: null, - currentStage: '', - overall: { totalSites: config.SUPPORT_WIKI.length, doneSites: 0 }, - sites: config.SUPPORT_WIKI.map(buildSiteStatus), - lastRun: crawlStatus.lastRun - }; - await persistCrawlStatus(); - for (const siteConfig of config.SUPPORT_WIKI) { - const wikiParam = siteConfig.PARAM; - const actualWikiName = siteConfig.URL.replace(/^https?:\/\//i, '').split('.')[0]; - const baseUrl = siteConfig.URL.replace(/\/$/, ''); - - let siteVotes = 0, siteDiscussions = 0, siteErrors = 0; - let siteStatus = crawlStatus.sites.find(s => s.param === wikiParam); - if (siteStatus) { - siteStatus.status = 'running'; - siteStatus.startedAt = Date.now(); - siteStatus.error = null; - crawlStatus.currentSite = wikiParam; - crawlStatus.currentStage = 'list'; - await persistCrawlStatus(); - } - - let allPages = []; - let pageNum = 1; - let totalPages = 1; - let hasMore = true; - - while (hasMore) { - try { - process.stdout.write(`获取 [${wikiParam}] 清单 第 ${pageNum} 页... `); - const res = await request.get(`https://wikit.unitreaty.org/listpages?wiki=${actualWikiName}&p=${pageNum}`); - const lines = res.data.split('\n').map(l => l.trim()).filter(Boolean); - let countThisPage = 0; - - lines.forEach(line => { - if (line.startsWith('Total Pages:')) { - totalPages = parseInt(line.replace('Total Pages:', '').trim(), 10) || 1; - } else if (line.includes('http') && line.includes('|')) { - const parts = line.split('|').map(item => item.trim()); - if (parts.length >= 7) { - const url = parts[0]; - const pageSlug = url.split('/').pop(); - let author = parts[6] || '未知'; - const match = author.match(/^(.*?)\s*\(\d+\)$/); - if (match) author = match[1].trim(); - - allPages.push({ page: pageSlug, title: parts[1], author: author, wiki: wikiParam, rating: parseInt(parts[3], 10) || 0, upvotes: parseInt(parts[4], 10) || 0, downvotes: parseInt(parts[5], 10) || 0 }); - countThisPage++; - } - } - }); - logLine(`成功 ${countThisPage} 篇`); - if (pageNum >= totalPages) hasMore = false; - else pageNum++; - } catch (e) { - await sleep(3000); - } - await sleep(1000); - } - - if (siteStatus) { - siteStatus.pagesFound = allPages.length; - crawlStatus.currentStage = 'crawl'; - } - await persistCrawlStatus(); - - let userVotesMap = {}; - let count = 0; - const CONCURRENCY = 3; - - for (let i = 0; i < allPages.length; i += CONCURRENCY) { - const batch = allPages.slice(i, i + CONCURRENCY); - await Promise.all(batch.map(async (pageNode) => { - const secureUrl = `${baseUrl}/${pageNode.page}`; - let success = false, attempt = 0; - - while (!success && attempt < 3) { - attempt++; - try { - const { data: html } = await request.get(secureUrl, { headers: baseHeaders }); - - const $page = cheerio.load(html); - let threadId = null; - - let href = $page('#discuss-button').attr('href'); - if (!href) href = $page('#page-info a').filter((_, el) => ($page(el).attr('href') || '').includes('/forum/t-')).attr('href'); - if (!href) href = $page('#page-content').parent().find('a').filter((_, el) => { - const text = $page(el).text().toLowerCase(); - return (text.includes('discuss') || text.includes('讨论') || text.includes('评论')) && ($page(el).attr('href') || '').includes('/forum/t-'); - }).attr('href'); - - if (href) { - const match = href.match(/\/forum\/t-(\d+)/); - if (match) threadId = match[1]; - } - - const cacheKey = `forum_v7:${wikiParam}:${pageNode.page}`; - - if (threadId) { - siteDiscussions++; - try { - const forumUrl = `${baseUrl}/forum/t-${threadId}`; - const { data: forumHtml } = await request.get(forumUrl, { headers: baseHeaders }); - const $forum = cheerio.load(forumHtml); - const posts = []; - const userIdCache = {}; - - const postElements = $forum('.post').toArray(); - for (const el of postElements) { - const $el = $forum(el); - const postId = ($el.attr('id') || '').replace('post-', ''); - if (!postId) continue; - - let parentId = null; - const $parentContainer = $el.parent('.post-container').parent('.post-container'); - if ($parentContainer.length) { - const $parentPost = $parentContainer.children('.post').first(); - parentId = ($parentPost.attr('id') || '').replace('post-', ''); - } - - let author = '未知用户'; - const $printUser = $el.find('.head .printuser').length ? $el.find('.head .printuser').first() : $el.find('.info .printuser').first(); - - if ($printUser.length) { - const $links = $printUser.find('a'); - if ($links.length) author = $links.last().text().trim(); - else author = $printUser.text().trim(); - } else { - author = $el.find('.head .author, .info .author').first().text().trim() || '未知用户'; - } - author = author.replace(/[\r\n\t]+/g, '').trim(); - - let userid = null; - const headHtml = $el.find('.head').html() || $el.find('.info').html() || $el.html() || ''; - const srcMatch = headHtml.match(/avatar\.php\?userid=(\d+)/i); - const clickMatch = headHtml.match(/userInfo\(\s*(\d+)\s*\)/i); - const karmaMatch = headHtml.match(/userkarma\.php\?u=(\d+)/i); - - if (srcMatch) userid = srcMatch[1]; - else if (clickMatch) userid = clickMatch[1]; - else if (karmaMatch) userid = karmaMatch[1]; - - if (!userid && author !== '未知用户') { - if (userIdCache[author]) { - userid = userIdCache[author]; - } else { - try { - const lookupRes = await axios.get(`https://www.wikidot.com/quickmodule.php?module=UserLookupQModule&q=${encodeURIComponent(author)}`, { timeout: 5000 }); - if (lookupRes.data && lookupRes.data.users && lookupRes.data.users.length > 0) { - userid = lookupRes.data.users[0].user_id; - userIdCache[author] = userid; - } - } catch (lookupErr) { - // 忽略查询报错 - } - } - } - - let avatarUrl = ''; - if (userid) { - const currentTs = Math.floor(Date.now() / 1000); - avatarUrl = `https://www.wikidot.com/avatar.php?userid=${userid}×tamp=${currentTs}`; - } else { - avatarUrl = `https://www.wikidot.com/avatar.php?account=default`; - } - - const contentHtml = $el.find('.content').html() || ''; - const odate = $el.find('.odate').first(); - const odateClass = odate.attr('class') || ''; - const timeMatch = odateClass.match(/time_(\d+)/); - - let timestamp = odate.text().trim(); - if (timeMatch) { - const dateObj = new Date(parseInt(timeMatch[1]) * 1000); - timestamp = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')} ${String(dateObj.getHours()).padStart(2, '0')}:${String(dateObj.getMinutes()).padStart(2, '0')}`; - } else if (odate.attr('title')) timestamp = odate.attr('title'); - - posts.push({ postId, parentId, author, avatarUrl, timestamp, contentHtml, children: [] }); - } - - const postMap = {}; - const rootPosts = []; - posts.forEach(p => { p.children = []; postMap[p.postId] = p; }); - posts.forEach(p => { - if (p.parentId && postMap[p.parentId]) postMap[p.parentId].children.push(p); - else rootPosts.push(p); - }); - - const forumData = { threadId, url: forumUrl, total: posts.length, threads: rootPosts }; - - await prisma.setting.upsert({ - where: { key: cacheKey }, - update: { value: JSON.stringify(forumData) }, - create: { key: cacheKey, value: JSON.stringify(forumData) } - }); - console.log(`[成功] 讨论区入库: ${pageNode.page} (${posts.length}条)`); - } catch (forumErr) { - console.error(`[失败] ${pageNode.page} 讨论区抓取报错: ${forumErr.message}`); - } - } else { - const emptyData = { threadId: null, url: '', total: 0, threads: [] }; - await prisma.setting.upsert({ - where: { key: cacheKey }, - update: { value: JSON.stringify(emptyData) }, - create: { key: cacheKey, value: JSON.stringify(emptyData) } - }); - } - - let pageId = null; - const idMatch = html.match(/pageId\s*[:=]\s*['"]?(\d+)['"]?/i) || html.match(/page_id\s*[:=]\s*['"]?(\d+)['"]?/i); - if (idMatch) pageId = idMatch[1]; - if (!pageId) { success = true; return; } - - const origin = new URL(secureUrl).origin; - const ajaxUrl = `${origin}/ajax-module-connector.php`; - - const ajaxHeaders = { - 'User-Agent': 'Mozilla/5.0', - 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'Cookie': botCookie ? botCookie : 'wikidot_token7=123456;' - }; - const { data: rateData } = await request.post(ajaxUrl, `pageId=${pageId}&page_id=${pageId}&moduleName=pagerate/WhoRatedPageModule&wikidot_token7=123456`, { - headers: ajaxHeaders - }); - - if (rateData.status === 'ok' && rateData.body) { - const $rate = cheerio.load(rateData.body); - $rate('.printuser').each((_, el) => { - const user = $rate(el).text().trim(); - let vote = '+1', textAfter = '', curr = el.next; - while (curr) { - if (curr.type === 'tag' && (curr.tagName === 'br' || (curr.attribs && curr.attribs.class && curr.attribs.class.includes('printuser')))) break; - if (curr.type === 'text') textAfter += curr.data; - else if (curr.type === 'tag') textAfter += $rate(curr).text(); - curr = curr.next; - } - if (textAfter.includes('-')) vote = '-1'; - if (!userVotesMap[user]) userVotesMap[user] = []; - userVotesMap[user].push({ wiki: wikiParam, page: pageNode.page, title: pageNode.title, vote: vote, author: pageNode.author, date: Date.now() }); - siteVotes++; - }); - } - success = true; - } catch (err) { - siteErrors++; - if (attempt < 3) await sleep(2000); - } - } - })); - - count += batch.length; - logLine(`--- 当前进度: [${count}/${allPages.length}] ---`); - - if (count % 100 === 0 || count >= allPages.length) { - if (siteStatus) { - siteStatus.pagesProcessed = count; - siteStatus.votes = siteVotes; - siteStatus.discussions = siteDiscussions; - siteStatus.errors = siteErrors; - await persistCrawlStatus(); - } - for (const [user, newVotes] of Object.entries(userVotesMap)) { - const key = `user_votes_${user.toLowerCase().replace(/_/g, '-').replace(/ /g, '-')}`; - const record = await prisma.setting.findUnique({ where: { key } }); - let existingMap = new Map(); - if (record) { - // lib/prisma.js 的 setting 扩展已自动解析 value 为对象,避免二次 JSON.parse 崩溃 - const parsed = typeof record.value === 'string' ? JSON.parse(record.value) : record.value; - if (Array.isArray(parsed)) { - parsed.forEach(v => existingMap.set(`${v.wiki}:${v.page}`, v)); - } - } - - newVotes.forEach(nv => { - const id = `${nv.wiki}:${nv.page}`; - if (!existingMap.has(id)) existingMap.set(id, nv); - else if (existingMap.get(id).vote !== nv.vote) { - existingMap.get(id).vote = nv.vote; - existingMap.get(id).date = Date.now(); - } - }); - - const truncatedVotes = Array.from(existingMap.values()).sort((a, b) => b.date - a.date).slice(0, 800); - await prisma.setting.upsert({ - where: { key }, - update: { value: JSON.stringify(truncatedVotes) }, - create: { key, value: JSON.stringify(truncatedVotes) } - }); - } - userVotesMap = {}; - } - await sleep(2500); - } - - if (siteStatus) { - siteStatus.status = 'done'; - siteStatus.finishedAt = Date.now(); - siteStatus.lastRun = Date.now(); - siteStatus.pagesProcessed = count; - siteStatus.votes = siteVotes; - siteStatus.discussions = siteDiscussions; - siteStatus.errors = siteErrors; - crawlStatus.overall.doneSites = crawlStatus.sites.filter(s => s.status === 'done').length; - crawlStatus.currentSite = null; - crawlStatus.currentStage = 'done'; - await persistCrawlStatus(); - } - } - } catch (e) { - logLine(`发生异常: ${e.message}`); - crawlStatus.sites.forEach(s => { - if (s.status === 'running') { - s.status = 'error'; - s.error = e.message; - } - }); - await persistCrawlStatus(); - } finally { - isRunning = false; - crawlStatus.running = false; - crawlStatus.finishedAt = Date.now(); - crawlStatus.lastRun = Date.now(); - crawlStatus.currentSite = null; - crawlStatus.currentStage = ''; - crawlStatus.overall.doneSites = crawlStatus.sites.filter(s => s.status === 'done').length; - await persistCrawlStatus(); - } -} - -cron.schedule('0 */3 * * *', () => runCrawler()); -runCrawler(); -let isBackupRunning = false; - -async function runWikitBackup() { - if (isBackupRunning) { - logLine(`[${new Date().toLocaleString()}] Wikit backup is already running; skipping.`); + if (process.argv.includes('--slice')) { + const result = await runCrawlSlice({ force }); + console.log(JSON.stringify(result)); return; } - const wikiNames = config.SUPPORT_WIKI.map(getWikiName).filter(Boolean); - if (wikiNames.length === 0) { - logLine(`[${new Date().toLocaleString()}] No valid wikis configured for backup.`); - return; + const forSec = argNumber('--for'); + const deadline = forSec == null ? Infinity : Date.now() + forSec * 1000; + let last = null; + while (Date.now() < deadline) { + last = await runCrawlSlice({ force }); + if (!last || last.skipped || last.done || last.error) break; } - - isBackupRunning = true; - const args = buildBackupArgs({ wikiNames, keepRemoved: true }); - logLine(`[${new Date().toLocaleString()}] Starting wikit backup for ${wikiNames.length} wikis.`); - - await new Promise((resolve) => { - const child = spawn('wikit', args, { - cwd: process.cwd(), - env: process.env, - shell: false, - windowsHide: true, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - child.stdout.on('data', (chunk) => process.stdout.write(`[wikit] ${chunk}`)); - child.stderr.on('data', (chunk) => process.stderr.write(`[wikit] ${chunk}`)); - child.on('error', (error) => { - logLine(`[${new Date().toLocaleString()}] Failed to start wikit: ${error.message}`); - }); - child.on('close', (code) => { - logLine(`[${new Date().toLocaleString()}] Wikit backup finished with exit code ${code}.`); - resolve(); - }); - }); - - isBackupRunning = false; + console.log(JSON.stringify(last)); } -// Back up every configured wiki daily at 12:00 China Standard Time. -cron.schedule('0 12 * * *', () => runWikitBackup(), { timezone: 'Asia/Shanghai' }); +main() + .then(() => process.exit(process.exitCode || 0)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/deploy/wikitdb-attribution.service b/deploy/wikitdb-attribution.service index ecf67f3..5783990 100644 --- a/deploy/wikitdb-attribution.service +++ b/deploy/wikitdb-attribution.service @@ -1,19 +1,23 @@ +# DEPRECATED +# 无服务器部署不要启用这个单元。归属同步由 /api/cron/attribution +# 或 .github/workflows/serverless-jobs.yml 触发,进程跑完即退出。 +# 若仍在虚拟机上临时使用:不要 Restart=always,否则冷却期也会被立刻拉起。 + [Unit] -Description=WikitDB Attribution Sync (fetch attribution pages + compute author scores every 30min) +Description=WikitDB Attribution Sync (one-shot; scheduler should invoke this) After=network.target postgresql.service Wants=postgresql.service [Service] -Type=simple +Type=oneshot User=root WorkingDirectory=/opt/WikitDB-Server EnvironmentFile=/opt/WikitDB-Server/.env Environment=NODE_ENV=production Environment=NODE_OPTIONS=--max-old-space-size=1024 Environment=PATH=/root/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ExecStart=/usr/bin/node /opt/WikitDB-Server/auto-attribution.js -Restart=always -RestartSec=10 +ExecStart=/usr/bin/node /opt/WikitDB-Server/auto-attribution.js --for 480 +Restart=no [Install] WantedBy=multi-user.target diff --git a/jobs/attribution.js b/jobs/attribution.js new file mode 100644 index 0000000..238ab1e --- /dev/null +++ b/jobs/attribution.js @@ -0,0 +1,183 @@ +/** + * 归属资料同步。按站点、按清单页切片,避免一次函数调用爬完整站。 + * 调度间隔 30 分钟:一轮跑完后游标进入 idle,到期再开下一轮。 + */ +const http = require('http'); +const https = require('https'); +const prisma = require('../lib/prisma'); +const { loadEffectiveConfig } = require('../utils/siteConfig'); +const { fetchAttributionPage, aggregateAuthorScores, normalizePageKey } = require('../utils/attribution'); +const { appendJobLogs } = require('../utils/jobLog'); +const { withJobLock } = require('../utils/jobLock'); + +const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); +const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 }); +const request = require('axios').create({ + httpAgent, + httpsAgent, + timeout: 30000, + validateStatus: (s) => s >= 200 && s < 400, +}); + +const INTERVAL_MS = 30 * 60 * 1000; +const DEFAULT_BUDGET_MS = 45 * 1000; +const CURSOR_KEY = 'attribution:cursor'; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const logBuffer = []; + +function logLine(...args) { + const line = args.map(String).join(' '); + console.log(line); + logBuffer.push(`[${new Date().toLocaleString()}] ${line}`); +} + +async function flushLogs() { + if (logBuffer.length === 0) return; + const lines = logBuffer.splice(0, logBuffer.length); + await appendJobLogs('crawler', lines); +} + +function normalize(value) { + if (!value) return null; + if (typeof value === 'string') { + try { return JSON.parse(value); } catch (e) { return null; } + } + return value; +} + +function wikiHost(siteConfig) { + return siteConfig.URL.replace(/^https?:\/\//i, '').replace(/^www\./i, '').split('.')[0]; +} + +async function fetchListPage(cursor, siteConfig) { + const res = await request.get(`https://wikit.unitreaty.org/listpages?wiki=${wikiHost(siteConfig)}&p=${cursor.listPage}`); + const lines = String(res.data).split('\n').map((l) => l.trim()).filter(Boolean); + const totalMatch = lines.find((l) => l.startsWith('Total Pages:'))?.match(/(\d+)/); + if (totalMatch) cursor.totalPages = parseInt(totalMatch[1], 10) || 1; + let countThisPage = 0; + for (const line of lines) { + if (!line.startsWith('http')) continue; + const parts = line.split('|').map((s) => s.trim()); + if (parts.length < 6) continue; + cursor.pages.push({ + page: parts[0].split('/').pop(), + title: parts[1] || '', + rating: parseInt(parts[3], 10) || 0, + upvotes: parseInt(parts[4], 10) || 0, + downvotes: parseInt(parts[5], 10) || 0, + }); + countThisPage++; + } + if (cursor.listPage >= cursor.totalPages || countThisPage === 0) cursor.phase = 'commit'; + else cursor.listPage += 1; +} + +async function commitSite(cursor, siteConfig) { + const wikiParam = siteConfig.PARAM; + const attrRecords = cursor.records || []; + await prisma.$transaction([ + prisma.authorAttribution.deleteMany({ where: { siteParam: wikiParam } }), + prisma.authorAttribution.createMany({ + data: attrRecords.map((a) => ({ + siteParam: wikiParam, + page: a.page, + username: a.username, + type: a.type, + date: a.date, + })), + skipDuplicates: true, + }), + ]); + const pageScoreRows = (cursor.pages || []).map((p) => ({ + page: p.page, title: p.title, rating: p.rating, upvotes: p.upvotes, downvotes: p.downvotes, + })); + await prisma.setting.upsert({ + where: { key: `page_scores:${wikiParam}` }, + update: { value: pageScoreRows }, + create: { key: `page_scores:${wikiParam}`, value: pageScoreRows }, + }); + const pageRatings = new Map(); + for (const pg of cursor.pages || []) pageRatings.set(normalizePageKey(pg.page), { rating: pg.rating }); + const authorScores = aggregateAuthorScores(attrRecords, pageRatings); + await prisma.setting.upsert({ + where: { key: `author_score:${wikiParam}` }, + update: { value: authorScores }, + create: { key: `author_score:${wikiParam}`, value: authorScores }, + }); + logLine(`[归属] ${wikiParam} 归属 ${attrRecords.length} 条 | 页面评分 ${pageScoreRows.length} | 作者 ${Object.keys(authorScores).length} 位`); +} + +async function runAttributionSlice(opts = {}) { + const budgetMs = Number(opts.budgetMs) > 0 ? Number(opts.budgetMs) : DEFAULT_BUDGET_MS; + const started = Date.now(); + const over = () => Date.now() - started >= budgetMs; + + return withJobLock('attribution', budgetMs + 30 * 1000, async () => { + const sites = (((await loadEffectiveConfig()).SUPPORT_WIKI) || []).filter((w) => w.ATTRIBUTION_PAGE); + if (sites.length === 0) return { skipped: true, reason: 'no-sites' }; + + let cursor = normalize((await prisma.setting.findUnique({ where: { key: CURSOR_KEY } }))?.value); + if (!cursor || cursor.phase === 'idle') { + if (cursor && cursor.nextCycleAt && Date.now() < cursor.nextCycleAt && !opts.force) { + return { skipped: true, reason: 'cooldown', nextCycleAt: cursor.nextCycleAt }; + } + logLine(`[归属] 开始执行(${sites.length} 个站点配置了归属资料页)...`); + cursor = { phase: 'fetch', siteIndex: 0, listPage: 1, totalPages: 1, pages: [], records: [], nextCycleAt: 0 }; + } + + try { + while (!over()) { + if (cursor.siteIndex >= sites.length) { + cursor = { phase: 'idle', siteIndex: 0, pages: [], records: [], nextCycleAt: Date.now() + INTERVAL_MS }; + await prisma.setting.upsert({ + where: { key: CURSOR_KEY }, + update: { value: cursor }, + create: { key: CURSOR_KEY, value: cursor }, + }); + logLine('[归属] 本轮执行完成。'); + await flushLogs(); + return { done: true, phase: 'idle' }; + } + + const siteConfig = sites[cursor.siteIndex]; + if (cursor.phase === 'fetch') { + cursor.records = await fetchAttributionPage(siteConfig, request); + cursor.pages = []; + cursor.listPage = 1; + cursor.totalPages = 1; + if (!cursor.records || cursor.records.length === 0) { + logLine(`[归属] ${siteConfig.PARAM} 归属资料页无有效数据,跳过`); + cursor.siteIndex += 1; + cursor.phase = 'fetch'; + } else { + cursor.phase = 'list'; + } + } else if (cursor.phase === 'list') { + await fetchListPage(cursor, siteConfig); + await sleep(200); + } else if (cursor.phase === 'commit') { + await commitSite(cursor, siteConfig); + cursor.siteIndex += 1; + cursor.phase = 'fetch'; + cursor.pages = []; + cursor.records = []; + } + + await prisma.setting.upsert({ + where: { key: CURSOR_KEY }, + update: { value: cursor }, + create: { key: CURSOR_KEY, value: cursor }, + }); + } + } catch (e) { + logLine(`[归属] ${sites[cursor.siteIndex] ? sites[cursor.siteIndex].PARAM : '?'} 处理失败: ${e.message}`); + await flushLogs(); + return { done: false, error: e.message, phase: cursor.phase, siteIndex: cursor.siteIndex }; + } + + await flushLogs(); + return { done: false, phase: cursor.phase, siteIndex: cursor.siteIndex, listPage: cursor.listPage }; + }); +} + +module.exports = { runAttributionSlice, INTERVAL_MS }; diff --git a/jobs/crawler.js b/jobs/crawler.js new file mode 100644 index 0000000..4d802b7 --- /dev/null +++ b/jobs/crawler.js @@ -0,0 +1,576 @@ +/** + * 评分 / 讨论区爬虫,按时间片推进。 + * 每一片都把游标写进 settings,调用结束即可销毁进程。 + * 整轮间隔仍是 3 小时,由外部调度反复调用,而不是进程内 node-cron。 + */ +const axios = require('axios'); +const cheerio = require('cheerio'); +const http = require('http'); +const https = require('https'); +const { spawn } = require('child_process'); +const prisma = require('../lib/prisma'); +const { loadEffectiveConfig } = require('../utils/siteConfig'); +const { buildBackupArgs, getWikiName } = require('../utils/wikitBackup'); +const { appendJobLogs } = require('../utils/jobLog'); +const { withJobLock } = require('../utils/jobLock'); + +const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); +const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 }); +const request = axios.create({ httpAgent, httpsAgent, timeout: 15000 }); + +const CRAWL_INTERVAL_MS = 3 * 60 * 60 * 1000; +const DEFAULT_BUDGET_MS = 45 * 1000; +const CONCURRENCY = 3; +const CURSOR_KEY = 'crawler:cursor'; +const STATUS_KEY = 'crawler:status'; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const logBuffer = []; + +function logLine(...args) { + const line = args.map(String).join(' '); + console.log(line); + logBuffer.push(`[${new Date().toLocaleString()}] ${line}`); +} + +async function flushLogs() { + if (logBuffer.length === 0) return; + const lines = logBuffer.splice(0, logBuffer.length); + await appendJobLogs('crawler', lines); +} + +function normalize(value) { + if (!value) return null; + if (typeof value === 'string') { + try { return JSON.parse(value); } catch (e) { return null; } + } + return value; +} + +async function readJsonSetting(key) { + const row = await prisma.setting.findUnique({ where: { key } }); + return normalize(row && row.value); +} + +async function writeJsonSetting(key, value) { + await prisma.setting.upsert({ + where: { key }, + update: { value }, + create: { key, value }, + }); +} + +function buildSiteStatus(siteConfig) { + return { + param: siteConfig.PARAM, + name: siteConfig.NAME, + status: 'pending', + pagesFound: 0, + pagesProcessed: 0, + votes: 0, + discussions: 0, + errors: 0, + startedAt: null, + finishedAt: null, + lastRun: null, + error: null, + }; +} + +function freshCursor(sites, previous) { + return { + phase: 'list', + siteIndex: 0, + listPage: 1, + totalPages: 1, + listErrors: 0, + pages: [], + pageOffset: 0, + siteVotes: 0, + siteDiscussions: 0, + siteErrors: 0, + userVotesMap: {}, + nextCycleAt: 0, + status: { + running: true, + startedAt: Date.now(), + finishedAt: null, + currentSite: null, + currentStage: 'list', + overall: { totalSites: sites.length, doneSites: 0 }, + sites: sites.map(buildSiteStatus), + lastRun: previous && previous.status ? previous.status.lastRun : null, + }, + }; +} + +function summary(cursor, extra = {}) { + return { + phase: cursor ? cursor.phase : 'idle', + siteIndex: cursor ? cursor.siteIndex : 0, + pageOffset: cursor ? cursor.pageOffset : 0, + pages: cursor && cursor.pages ? cursor.pages.length : 0, + nextCycleAt: cursor ? cursor.nextCycleAt : null, + ...extra, + }; +} + +let botCookieCache = null; + +async function getBotCookie() { + if (botCookieCache) return botCookieCache; + const user = process.env.WIKIDOT_BOT_USER; + const pass = process.env.WIKIDOT_BOT_PASS; + if (!user || !pass) { + logLine('未配置机器人账号,将以访客身份进行抓取...'); + return null; + } + try { + const payload = new URLSearchParams({ login: user, password: pass, action: 'Login2Action', event: 'login' }); + const res = await axios.post('https://www.wikidot.com/default--flow/login__LoginPopupScreen', payload.toString(), { + headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'WikitDB-Bot/1.0' }, + maxRedirects: 0, + validateStatus: (status) => status >= 200 && status < 400, + }); + let sessionId = ''; + const cookies = res.headers['set-cookie'] || []; + for (const c of cookies) { + if (c.includes('WIKIDOT_SESSION_ID=')) sessionId = c.split('WIKIDOT_SESSION_ID=')[1].split(';')[0]; + } + if (sessionId) { + botCookieCache = `WIKIDOT_SESSION_ID=${sessionId}; wikidot_token7=123456;`; + logLine('机器人账号登录成功,已获取受限站点抓取权限。'); + } + } catch (e) { + console.error('获取 Bot Cookie 失败:', e.message); + } + return botCookieCache; +} + +function wikiHost(siteConfig) { + return siteConfig.URL.replace(/^https?:\/\//i, '').replace(/^www\./i, '').split('.')[0]; +} + +async function fetchListPage(cursor, siteConfig) { + const wikiParam = siteConfig.PARAM; + const actualWikiName = wikiHost(siteConfig); + try { + logLine(`获取 [${wikiParam}] 清单 第 ${cursor.listPage} 页...`); + const res = await request.get(`https://wikit.unitreaty.org/listpages?wiki=${actualWikiName}&p=${cursor.listPage}`); + const lines = String(res.data).split('\n').map((l) => l.trim()).filter(Boolean); + let countThisPage = 0; + lines.forEach((line) => { + if (line.startsWith('Total Pages:')) { + cursor.totalPages = parseInt(line.replace('Total Pages:', '').trim(), 10) || 1; + } else if (line.includes('http') && line.includes('|')) { + const parts = line.split('|').map((item) => item.trim()); + if (parts.length >= 7) { + const url = parts[0]; + const pageSlug = url.split('/').pop(); + let author = parts[6] || '未知'; + const match = author.match(/^(.*?)\s*\(\d+\)$/); + if (match) author = match[1].trim(); + cursor.pages.push({ + page: pageSlug, + title: parts[1], + author, + wiki: wikiParam, + rating: parseInt(parts[3], 10) || 0, + upvotes: parseInt(parts[4], 10) || 0, + downvotes: parseInt(parts[5], 10) || 0, + }); + countThisPage++; + } + } + }); + logLine(`成功 ${countThisPage} 篇`); + cursor.listErrors = 0; + await sleep(400); + if (cursor.listPage >= cursor.totalPages) { + cursor.phase = 'pages'; + cursor.pageOffset = 0; + const siteStatus = cursor.status.sites.find((s) => s.param === wikiParam); + if (siteStatus) { + siteStatus.pagesFound = cursor.pages.length; + cursor.status.currentStage = 'crawl'; + } + } else { + cursor.listPage += 1; + } + } catch (e) { + cursor.listErrors = (cursor.listErrors || 0) + 1; + logLine(`[${wikiParam}] 清单第 ${cursor.listPage} 页失败 (${cursor.listErrors}): ${e.message}`); + if (cursor.listErrors >= 3) { + cursor.listPage += 1; + cursor.listErrors = 0; + if (cursor.listPage > cursor.totalPages) { + cursor.phase = 'pages'; + cursor.pageOffset = 0; + } + } else { + await sleep(1500); + } + } +} + +async function flushVotes(userVotesMap) { + for (const [user, newVotes] of Object.entries(userVotesMap)) { + const key = `user_votes_${user.toLowerCase().replace(/_/g, '-').replace(/ /g, '-')}`; + const record = await prisma.setting.findUnique({ where: { key } }); + const existingMap = new Map(); + if (record) { + const parsed = typeof record.value === 'string' ? JSON.parse(record.value) : record.value; + if (Array.isArray(parsed)) parsed.forEach((v) => existingMap.set(`${v.wiki}:${v.page}`, v)); + } + newVotes.forEach((nv) => { + const id = `${nv.wiki}:${nv.page}`; + if (!existingMap.has(id)) existingMap.set(id, nv); + else if (existingMap.get(id).vote !== nv.vote) { + existingMap.get(id).vote = nv.vote; + existingMap.get(id).date = Date.now(); + } + }); + const truncatedVotes = Array.from(existingMap.values()).sort((a, b) => b.date - a.date).slice(0, 800); + await prisma.setting.upsert({ + where: { key }, + update: { value: truncatedVotes }, + create: { key, value: truncatedVotes }, + }); + } +} + +async function scrapeOne(pageNode, siteConfig, baseHeaders, cursor) { + const baseUrl = siteConfig.URL.replace(/\/$/, ''); + const secureUrl = `${baseUrl}/${pageNode.page}`; + const wikiParam = siteConfig.PARAM; + let success = false; + let attempt = 0; + while (!success && attempt < 3) { + attempt++; + try { + const { data: html } = await request.get(secureUrl, { headers: baseHeaders }); + const $page = cheerio.load(html); + let threadId = null; + let href = $page('#discuss-button').attr('href'); + if (!href) href = $page('#page-info a').filter((_, el) => ($page(el).attr('href') || '').includes('/forum/t-')).attr('href'); + if (!href) { + href = $page('#page-content').parent().find('a').filter((_, el) => { + const text = $page(el).text().toLowerCase(); + return (text.includes('discuss') || text.includes('讨论') || text.includes('评论')) && ($page(el).attr('href') || '').includes('/forum/t-'); + }).attr('href'); + } + if (href) { + const match = href.match(/\/forum\/t-(\d+)/); + if (match) threadId = match[1]; + } + const cacheKey = `forum_v7:${wikiParam}:${pageNode.page}`; + if (threadId) { + cursor.siteDiscussions++; + try { + const forumUrl = `${baseUrl}/forum/t-${threadId}`; + const { data: forumHtml } = await request.get(forumUrl, { headers: baseHeaders }); + const $forum = cheerio.load(forumHtml); + const posts = []; + const userIdCache = {}; + const postElements = $forum('.post').toArray(); + for (const el of postElements) { + const $el = $forum(el); + const postId = ($el.attr('id') || '').replace('post-', ''); + if (!postId) continue; + let parentId = null; + const $parentContainer = $el.parent('.post-container').parent('.post-container'); + if ($parentContainer.length) { + const $parentPost = $parentContainer.children('.post').first(); + parentId = ($parentPost.attr('id') || '').replace('post-', ''); + } + let author = '未知用户'; + const $printUser = $el.find('.head .printuser').length ? $el.find('.head .printuser').first() : $el.find('.info .printuser').first(); + if ($printUser.length) { + const $links = $printUser.find('a'); + author = $links.length ? $links.last().text().trim() : $printUser.text().trim(); + } else { + author = $el.find('.head .author, .info .author').first().text().trim() || '未知用户'; + } + author = author.replace(/[\r\n\t]+/g, '').trim(); + let userid = null; + const headHtml = $el.find('.head').html() || $el.find('.info').html() || $el.html() || ''; + const srcMatch = headHtml.match(/avatar\.php\?userid=(\d+)/i); + const clickMatch = headHtml.match(/userInfo\(\s*(\d+)\s*\)/i); + const karmaMatch = headHtml.match(/userkarma\.php\?u=(\d+)/i); + if (srcMatch) userid = srcMatch[1]; + else if (clickMatch) userid = clickMatch[1]; + else if (karmaMatch) userid = karmaMatch[1]; + if (!userid && author !== '未知用户') { + if (userIdCache[author]) userid = userIdCache[author]; + else { + try { + const lookupRes = await axios.get(`https://www.wikidot.com/quickmodule.php?module=UserLookupQModule&q=${encodeURIComponent(author)}`, { timeout: 5000 }); + if (lookupRes.data && lookupRes.data.users && lookupRes.data.users.length > 0) { + userid = lookupRes.data.users[0].user_id; + userIdCache[author] = userid; + } + } catch (lookupErr) { /* 忽略 */ } + } + } + const currentTs = Math.floor(Date.now() / 1000); + const avatarUrl = userid + ? `https://www.wikidot.com/avatar.php?userid=${userid}×tamp=${currentTs}` + : 'https://www.wikidot.com/avatar.php?account=default'; + const contentHtml = $el.find('.content').html() || ''; + const odate = $el.find('.odate').first(); + const odateClass = odate.attr('class') || ''; + const timeMatch = odateClass.match(/time_(\d+)/); + let timestamp = odate.text().trim(); + if (timeMatch) { + const dateObj = new Date(parseInt(timeMatch[1], 10) * 1000); + timestamp = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')} ${String(dateObj.getHours()).padStart(2, '0')}:${String(dateObj.getMinutes()).padStart(2, '0')}`; + } else if (odate.attr('title')) timestamp = odate.attr('title'); + posts.push({ postId, parentId, author, avatarUrl, timestamp, contentHtml, children: [] }); + } + const postMap = {}; + const rootPosts = []; + posts.forEach((p) => { p.children = []; postMap[p.postId] = p; }); + posts.forEach((p) => { + if (p.parentId && postMap[p.parentId]) postMap[p.parentId].children.push(p); + else rootPosts.push(p); + }); + await prisma.setting.upsert({ + where: { key: cacheKey }, + update: { value: { threadId, url: forumUrl, total: posts.length, threads: rootPosts } }, + create: { key: cacheKey, value: { threadId, url: forumUrl, total: posts.length, threads: rootPosts } }, + }); + logLine(`[成功] 讨论区入库: ${pageNode.page} (${posts.length}条)`); + } catch (forumErr) { + console.error(`[失败] ${pageNode.page} 讨论区抓取报错: ${forumErr.message}`); + } + } else { + await prisma.setting.upsert({ + where: { key: cacheKey }, + update: { value: { threadId: null, url: '', total: 0, threads: [] } }, + create: { key: cacheKey, value: { threadId: null, url: '', total: 0, threads: [] } }, + }); + } + + let pageId = null; + const idMatch = html.match(/pageId\s*[:=]\s*['"]?(\d+)['"]?/i) || html.match(/page_id\s*[:=]\s*['"]?(\d+)['"]?/i); + if (idMatch) pageId = idMatch[1]; + if (!pageId) { success = true; return; } + + const origin = new URL(secureUrl).origin; + const ajaxUrl = `${origin}/ajax-module-connector.php`; + const botCookie = baseHeaders.Cookie; + const { data: rateData } = await request.post( + ajaxUrl, + `pageId=${pageId}&page_id=${pageId}&moduleName=pagerate/WhoRatedPageModule&wikidot_token7=123456`, + { headers: { 'User-Agent': 'Mozilla/5.0', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', Cookie: botCookie || 'wikidot_token7=123456;' } }, + ); + if (rateData.status === 'ok' && rateData.body) { + const $rate = cheerio.load(rateData.body); + $rate('.printuser').each((_, el) => { + const user = $rate(el).text().trim(); + let vote = '+1'; + let textAfter = ''; + let curr = el.next; + while (curr) { + if (curr.type === 'tag' && (curr.tagName === 'br' || (curr.attribs && curr.attribs.class && curr.attribs.class.includes('printuser')))) break; + if (curr.type === 'text') textAfter += curr.data; + else if (curr.type === 'tag') textAfter += $rate(curr).text(); + curr = curr.next; + } + if (textAfter.includes('-')) vote = '-1'; + if (!cursor.userVotesMap[user]) cursor.userVotesMap[user] = []; + cursor.userVotesMap[user].push({ + wiki: wikiParam, page: pageNode.page, title: pageNode.title, vote, author: pageNode.author, date: Date.now(), + }); + cursor.siteVotes++; + }); + } + success = true; + } catch (err) { + cursor.siteErrors++; + if (attempt < 3) await sleep(1000); + } + } +} + +function touchSite(cursor, siteConfig, patch) { + const siteStatus = cursor.status.sites.find((s) => s.param === siteConfig.PARAM); + if (!siteStatus) return; + Object.assign(siteStatus, patch); +} + +async function finishSite(cursor, siteConfig) { + await flushVotes(cursor.userVotesMap || {}); + cursor.userVotesMap = {}; + touchSite(cursor, siteConfig, { + status: 'done', + finishedAt: Date.now(), + lastRun: Date.now(), + pagesProcessed: cursor.pageOffset, + pagesFound: cursor.pages.length, + votes: cursor.siteVotes, + discussions: cursor.siteDiscussions, + errors: cursor.siteErrors, + }); + cursor.status.overall.doneSites = cursor.status.sites.filter((s) => s.status === 'done').length; + cursor.status.currentSite = null; + cursor.status.currentStage = 'done'; + cursor.siteIndex += 1; + cursor.phase = 'list'; + cursor.listPage = 1; + cursor.totalPages = 1; + cursor.listErrors = 0; + cursor.pages = []; + cursor.pageOffset = 0; + cursor.siteVotes = 0; + cursor.siteDiscussions = 0; + cursor.siteErrors = 0; +} + +function finishCycle(cursor) { + cursor.phase = 'idle'; + cursor.pages = []; + cursor.userVotesMap = {}; + cursor.nextCycleAt = Date.now() + CRAWL_INTERVAL_MS; + cursor.status.running = false; + cursor.status.finishedAt = Date.now(); + cursor.status.lastRun = Date.now(); + cursor.status.currentSite = null; + cursor.status.currentStage = ''; + cursor.status.overall.doneSites = cursor.status.sites.filter((s) => s.status === 'done').length; + return cursor; +} + +async function runCrawlSlice(opts = {}) { + const budgetMs = Number(opts.budgetMs) > 0 ? Number(opts.budgetMs) : DEFAULT_BUDGET_MS; + const started = Date.now(); + const over = () => Date.now() - started >= budgetMs; + + return withJobLock('crawl', budgetMs + 30 * 1000, async () => { + const sites = ((await loadEffectiveConfig()).SUPPORT_WIKI) || []; + let cursor = await readJsonSetting(CURSOR_KEY); + if (!cursor || cursor.phase === 'idle' || !cursor.status) { + if (cursor && cursor.nextCycleAt && Date.now() < cursor.nextCycleAt && !opts.force) { + return summary(cursor, { skipped: true, reason: 'cooldown' }); + } + logLine(`\n[${new Date().toLocaleString()}] 开始执行全站数据(评分+讨论区)爬取...`); + cursor = freshCursor(sites, cursor); + } + + const botCookie = await getBotCookie(); + const baseHeaders = { 'User-Agent': 'Mozilla/5.0' }; + if (botCookie) baseHeaders.Cookie = botCookie; + + try { + while (!over()) { + if (cursor.siteIndex >= sites.length) { + cursor = finishCycle(cursor); + await writeJsonSetting(CURSOR_KEY, cursor); + await writeJsonSetting(STATUS_KEY, cursor.status); + logLine('本轮爬取结束。'); + await flushLogs(); + return summary(cursor, { done: true }); + } + + const siteConfig = sites[cursor.siteIndex]; + cursor.status.currentSite = siteConfig.PARAM; + const siteStatus = cursor.status.sites.find((s) => s.param === siteConfig.PARAM); + if (siteStatus && siteStatus.status === 'pending') { + siteStatus.status = 'running'; + siteStatus.startedAt = Date.now(); + siteStatus.error = null; + } + + if (cursor.phase === 'list') { + cursor.status.currentStage = 'list'; + await fetchListPage(cursor, siteConfig); + } else if (cursor.pageOffset >= cursor.pages.length) { + await finishSite(cursor, siteConfig); + } else { + cursor.status.currentStage = 'crawl'; + const batch = cursor.pages.slice(cursor.pageOffset, cursor.pageOffset + CONCURRENCY); + await Promise.all(batch.map((pageNode) => scrapeOne(pageNode, siteConfig, baseHeaders, cursor))); + cursor.pageOffset += batch.length; + touchSite(cursor, siteConfig, { + pagesProcessed: cursor.pageOffset, + pagesFound: cursor.pages.length, + votes: cursor.siteVotes, + discussions: cursor.siteDiscussions, + errors: cursor.siteErrors, + }); + if (cursor.pageOffset % 100 < CONCURRENCY || cursor.pageOffset >= cursor.pages.length) { + await flushVotes(cursor.userVotesMap || {}); + cursor.userVotesMap = {}; + } + logLine(`--- 当前进度: [${cursor.pageOffset}/${cursor.pages.length}] ${siteConfig.PARAM} ---`); + await sleep(2500); + } + + await writeJsonSetting(CURSOR_KEY, cursor); + await writeJsonSetting(STATUS_KEY, cursor.status); + } + } catch (e) { + logLine(`发生异常: ${e.message}`); + cursor.status.sites.forEach((s) => { + if (s.status === 'running') { + s.status = 'error'; + s.error = e.message; + } + }); + cursor.status.running = false; + await writeJsonSetting(CURSOR_KEY, cursor); + await writeJsonSetting(STATUS_KEY, cursor.status); + await flushLogs(); + return summary(cursor, { done: false, error: e.message }); + } + + await flushLogs(); + return summary(cursor, { done: false }); + }); +} + +function runBackup() { + if (process.env.VERCEL) { + return Promise.resolve({ + skipped: true, + reason: 'wikit CLI 需要本地二进制和磁盘,不能在无服务器函数里跑。请用 GitHub Actions 的备份工作流。', + }); + } + return withJobLock('backup', 6 * 60 * 60 * 1000, async () => { + const config = await loadEffectiveConfig(); + const wikiNames = (config.SUPPORT_WIKI || []).map(getWikiName).filter(Boolean); + if (wikiNames.length === 0) { + logLine(`[${new Date().toLocaleString()}] 没有可备份的站点。`); + await flushLogs(); + return { skipped: true, reason: 'no-wikis' }; + } + const args = buildBackupArgs({ wikiNames, keepRemoved: true, baseDir: process.env.WIKIT_BASE_DIR || undefined }); + logLine(`[${new Date().toLocaleString()}] 开始 wikit 备份,共 ${wikiNames.length} 个站点。`); + const code = await new Promise((resolve) => { + const child = spawn('wikit', args, { + cwd: process.cwd(), + env: process.env, + shell: false, + windowsHide: true, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout.on('data', (chunk) => logLine(`[wikit] ${chunk}`)); + child.stderr.on('data', (chunk) => logLine(`[wikit] ${chunk}`)); + child.on('error', (error) => { + logLine(`无法启动 wikit: ${error.message}`); + resolve(127); + }); + child.on('close', (exitCode) => resolve(exitCode)); + }); + logLine(`Wikit 备份结束,退出码 ${code}。`); + await flushLogs(); + return { done: true, exitCode: code }; + }); +} + +module.exports = { + runCrawlSlice, + runBackup, + CRAWL_INTERVAL_MS, + DEFAULT_BUDGET_MS, +}; diff --git a/lib/prisma.js b/lib/prisma.js index adcaac4..c248ee3 100644 --- a/lib/prisma.js +++ b/lib/prisma.js @@ -1,30 +1,42 @@ const { PrismaClient } = require('@prisma/client'); -let prismaBase; +/** + * 无服务器温启动会复用同一个实例。 + * 客户端必须挂在 globalThis 上,否则每次冷启动路径重复 new 会打满数据库连接。 + * 在 Vercel 上如果连接串没写 connection_limit,默认补 1,避免一个函数占一整池连接。 + */ +function serverlessDatabaseUrl(url) { + if (!url || !process.env.VERCEL) return url; + if (/[?&]connection_limit=/.test(url)) return url; + const join = url.includes('?') ? '&' : '?'; + return `${url}${join}connection_limit=1`; +} -if (process.env.NODE_ENV === 'production') { - prismaBase = new PrismaClient(); -} else { - if (!global.__prisma) { - global.__prisma = new PrismaClient(); - } - prismaBase = global.__prisma; +function createClient() { + const url = serverlessDatabaseUrl(process.env.DATABASE_URL); + const options = url && url !== process.env.DATABASE_URL + ? { datasources: { db: { url } } } + : undefined; + return new PrismaClient(options); } +const prismaBase = global.__prisma || createClient(); +global.__prisma = prismaBase; + const prisma = prismaBase.$extends({ query: { setting: { async findUnique({ args, query }) { const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, async findFirst({ args, query }) { const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, @@ -32,7 +44,7 @@ const prisma = prismaBase.$extends({ const results = await query(args); results.forEach(r => { if (r && r.value) { - try { r.value = JSON.parse(r.value); } catch {} + try { r.value = JSON.parse(r.value); } catch { /* 保持原字符串 */ } } }); return results; @@ -43,7 +55,7 @@ const prisma = prismaBase.$extends({ } const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, @@ -53,7 +65,7 @@ const prisma = prismaBase.$extends({ } const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, @@ -66,7 +78,7 @@ const prisma = prismaBase.$extends({ } const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, diff --git a/middleware.js b/middleware.js index 83c43a1..2b6a779 100644 --- a/middleware.js +++ b/middleware.js @@ -1,48 +1,69 @@ import { NextResponse } from 'next/server'; +import { Redis } from '@upstash/redis'; -const ipRequestCounts = new Map(); const WINDOW_MS = 60 * 1000; const MAX_REQUESTS_PER_MINUTE = 120; -function cleanupStore() { +// 没配 Upstash 时只在当前边缘实例里计数,跨实例不共享。 +const ipRequestCounts = new Map(); + +const redis = process.env.UPSTASH_REDIS_REST_URL && process.env.UPSTASH_REDIS_REST_TOKEN + ? new Redis({ + url: process.env.UPSTASH_REDIS_REST_URL, + token: process.env.UPSTASH_REDIS_REST_TOKEN, + }) + : null; + +function clientIp(request) { + const trustProxy = process.env.TRUST_PROXY === 'true'; + return (trustProxy && request.headers.get('x-forwarded-for')?.split(',')[0]?.trim()) + || (trustProxy && request.headers.get('x-real-ip')) + || request.ip + || 'unknown'; +} + +function tooMany() { + return NextResponse.json( + { error: '请求过于频繁,请稍后再试' }, + { status: 429 } + ); +} + +function memoryLimit(ip) { const now = Date.now(); + const record = ipRequestCounts.get(ip); + if (!record || now - record.windowStart > WINDOW_MS) { + ipRequestCounts.set(ip, { count: 1, windowStart: now }); + return false; + } + record.count += 1; if (ipRequestCounts.size > 5000) { for (const [key, data] of ipRequestCounts) { - if (now - data.windowStart > WINDOW_MS) { - ipRequestCounts.delete(key); - } + if (now - data.windowStart > WINDOW_MS) ipRequestCounts.delete(key); } } + return record.count > MAX_REQUESTS_PER_MINUTE; } -export function middleware(request) { - const trustProxy = process.env.TRUST_PROXY === 'true'; - // IP 解析优先级:反代头(需 TRUST_PROXY)→ Next.js request.ip → 兜底 'unknown' - // 修复:直连场景下避免所有请求共享 'unknown' 桶导致单点 DoS - const ip = (trustProxy && request.headers.get('x-forwarded-for')?.split(',')[0]?.trim()) - || (trustProxy && request.headers.get('x-real-ip')) - || request.ip - || 'unknown'; +export async function middleware(request) { + if (!request.nextUrl.pathname.startsWith('/api/')) return NextResponse.next(); - if (request.nextUrl.pathname.startsWith('/api/')) { - const now = Date.now(); - const record = ipRequestCounts.get(ip); - - if (!record || now - record.windowStart > WINDOW_MS) { - ipRequestCounts.set(ip, { count: 1, windowStart: now }); - } else { - record.count++; - if (record.count > MAX_REQUESTS_PER_MINUTE) { - return NextResponse.json( - { error: '请求过于频繁,请稍后再试' }, - { status: 429 } - ); - } - } + const ip = clientIp(request); - cleanupStore(); + if (redis) { + try { + const bucket = Math.floor(Date.now() / WINDOW_MS); + const key = `rl:${ip}:${bucket}`; + const count = await redis.incr(key); + if (count === 1) await redis.expire(key, 70); + if (count > MAX_REQUESTS_PER_MINUTE) return tooMany(); + return NextResponse.next(); + } catch (e) { + // Redis 故障时退回本实例计数,不把整个 API 打挂。 + } } + if (memoryLimit(ip)) return tooMany(); return NextResponse.next(); } diff --git a/next.config.js b/next.config.js index 3f3336d..ba87564 100644 --- a/next.config.js +++ b/next.config.js @@ -9,6 +9,9 @@ const ALLOWED_FRAME_DOMAINS = ['https://timer.backroomswiki.cn']; const isDev = process.env.NODE_ENV !== 'production'; const nextConfig = { + outputFileTracingIncludes: { + '/*': ['./wikitdb.config.js', './content/**/*'], + }, async rewrites() { return [ // Honeypot traps for common attack paths diff --git a/package-lock.json b/package-lock.json index 2c24375..3186b88 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,6 @@ "jsonwebtoken": "^9.0.2", "lightweight-charts": "^4.1.3", "next": "15.5.21", - "node-cron": "^4.2.1", "nodemailer": "^9.0.3", "nvm": "^0.0.4", "postcss": "^8.5.23", @@ -5640,15 +5639,6 @@ } } }, - "node_modules/node-cron": { - "version": "4.2.1", - "resolved": "https://registry.npmmirror.com/node-cron/-/node-cron-4.2.1.tgz", - "integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==", - "license": "ISC", - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/node-exports-info": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.2.tgz", diff --git a/package.json b/package.json index e271e10..6b576bf 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,9 @@ "test:ftml": "node scripts/test-ftml-runtime.mjs", "start": "next start", "worker": "node auto-crawler.js", + "job:crawl": "node auto-crawler.js", + "job:attribution": "node auto-attribution.js", + "job:backup": "node auto-crawler.js --backup", "lint": "next lint --quiet", "test": "node --test tests/*.test.js", "prisma:generate": "prisma generate", @@ -29,7 +32,6 @@ "jsonwebtoken": "^9.0.2", "lightweight-charts": "^4.1.3", "next": "15.5.21", - "node-cron": "^4.2.1", "nodemailer": "^9.0.3", "nvm": "^0.0.4", "postcss": "^8.5.23", diff --git a/pages/admin.js b/pages/admin.js index a43f778..765d9ea 100644 --- a/pages/admin.js +++ b/pages/admin.js @@ -201,7 +201,7 @@ export default function AdminDashboard() { body: JSON.stringify(isEdit ? { param: editingSiteParam, ...siteForm } : siteForm) }); setSites(d.sites || []); - setSiteMsg({ type: 'ok', text: isEdit ? (d.message || '站点已更新') : '站点已添加,配置已写入 wikitdb.config.js' }); + setSiteMsg({ type: 'ok', text: isEdit ? (d.message || '站点已更新') : '站点已写入数据库(磁盘可写时会同时刷新配置文件)' }); setEditingSiteParam(null); setSiteForm({ NAME: '', URL: '', PARAM: '', WIKIT_ID: '', ImgURL: '', GQL_API: '', AUTHOR_TAG: '', ATTRIBUTION_PAGE: '', FORUM_SYNC: false }); } catch (e) { @@ -235,7 +235,7 @@ export default function AdminDashboard() { }; const handleDeleteSite = async (param) => { - if (!confirm(`确定删除站点 "${param}"?此操作会立即修改 wikitdb.config.js。`)) return; + if (!confirm(`确定删除站点 "${param}"?此操作会写入数据库里的站点配置。`)) return; try { const d = await api('/api/admin/sites', { method: 'DELETE', body: JSON.stringify({ param }) }); setSites(d.sites || []); @@ -440,7 +440,7 @@ export default function AdminDashboard() { {activeTab === 'sites' && (
-
收录站点管理(写入 wikitdb.config.js)
+
收录站点管理(写入数据库)
@@ -514,7 +514,7 @@ export default function AdminDashboard() { {editingSiteParam && }
-
提示:站点变更会立即写入 wikitdb.config.js。已构建的静态页面与正在运行的爬虫进程(npm run worker)需重启后才会使用新的站点列表。
+
提示:站点变更写入数据库,爬虫下一片就会读到。已经打包进页面的导航列表要重新部署后才更新。无服务器环境写不了 wikitdb.config.js,这是预期行为。
)} @@ -523,7 +523,7 @@ export default function AdminDashboard() { {activeTab === 'crawler' && (
-
爬虫运行状态(每 3 小时自动执行)
+
爬虫运行状态(无服务器切片,每轮间隔 3 小时)
{crawlerLoading && 加载中...} @@ -806,11 +806,11 @@ export default function AdminDashboard() { {activeTab === 'logs' && (
-
服务器 / 爬虫文件日志
+
爬虫日志(数据库)
diff --git a/pages/api/admin/crawler-status.js b/pages/api/admin/crawler-status.js index ba4c40a..87f0e62 100644 --- a/pages/api/admin/crawler-status.js +++ b/pages/api/admin/crawler-status.js @@ -1,12 +1,10 @@ import prisma from '../../../lib/prisma'; import { withAdmin } from '../../../utils/withAdmin'; -import { loadSiteConfig } from '../../../utils/siteConfig'; +import { loadEffectiveConfig } from '../../../utils/siteConfig'; /** * 爬虫状态 API * GET /api/admin/crawler-status - * 返回:爬虫持久化的结构化状态(crawler:status), - * 并与当前配置中的站点列表合并,保证新增/删除站点后状态表始终完整。 */ async function handler(req, res) { if (req.method !== 'GET') return res.status(405).json({ error: 'Method not allowed' }); @@ -20,7 +18,7 @@ async function handler(req, res) { let configuredSites = []; try { - configuredSites = (loadSiteConfig().SUPPORT_WIKI) || []; + configuredSites = ((await loadEffectiveConfig()).SUPPORT_WIKI) || []; } catch (e) { configuredSites = []; } @@ -46,7 +44,6 @@ async function handler(req, res) { }; }); - // 用合并后的站点列表修正总进度统计,保证新增站点也能反映到进度里 if (status && typeof status === 'object' && !status.running) { status = { ...status, diff --git a/pages/api/admin/file-logs.js b/pages/api/admin/file-logs.js index 997c80f..dfe7c9c 100644 --- a/pages/api/admin/file-logs.js +++ b/pages/api/admin/file-logs.js @@ -1,47 +1,68 @@ import fs from 'fs'; import path from 'path'; import { withAdmin } from '../../../utils/withAdmin'; +import { readJobLogs } from '../../../utils/jobLog'; /** - * 服务器 / 爬虫文件日志查看 API - * GET /api/admin/file-logs?file=crawler|server|serverErr&lines=500 - * 返回指定日志文件末尾若干行。 + * 日志查看。 + * 爬虫日志在无服务器部署里来自数据库(logs:crawler)。 + * server.log / server-err.log 只在自建虚拟机上存在,读不到就返回说明。 */ -const LOG_FILES = { - crawler: 'crawler.log', +const FILE_LOGS = { server: 'server.log', serverErr: 'server-err.log', }; +async function readDisk(fileName, linesLimit) { + const filePath = path.join(process.cwd(), fileName); + if (!fs.existsSync(filePath)) return null; + const content = fs.readFileSync(filePath, 'utf8'); + const stat = fs.statSync(filePath); + const allLines = content.split(/\r?\n/).map(l => l.replace(/\r$/, '')).filter(l => l.trim().length > 0); + return { + lines: allLines.slice(-linesLimit), + totalLines: allLines.length, + size: stat.size, + mtime: stat.mtime, + }; +} + async function handler(req, res) { if (req.method !== 'GET') return res.status(405).json({ error: 'Method not allowed' }); const fileKey = String(req.query.file || 'crawler'); - const linesLimit = Math.min(parseInt(req.query.lines) || 500, 5000); - const fileName = LOG_FILES[fileKey]; - if (!fileName) return res.status(400).json({ error: '未知日志文件' }); - - const filePath = path.join(process.cwd(), fileName); + const linesLimit = Math.min(parseInt(req.query.lines, 10) || 500, 5000); try { - let content = ''; - let stat = null; - if (fs.existsSync(filePath)) { - content = fs.readFileSync(filePath, 'utf8'); - stat = fs.statSync(filePath); + if (fileKey === 'crawler') { + const stored = await readJobLogs('crawler', linesLimit); + return res.status(200).json({ + file: fileKey, + name: 'logs:crawler', + source: 'database', + lines: stored.lines, + totalLines: stored.totalLines, + size: stored.lines.join('\n').length, + mtime: stored.lines.length ? new Date().toISOString() : null, + }); } - const allLines = content.split(/\r?\n/); - const lines = allLines.map(l => l.replace(/\r$/, '')).filter(l => l.trim().length > 0).slice(-linesLimit); - - return res.status(200).json({ - file: fileKey, - name: fileName, - lines, - totalLines: allLines.length, - size: stat ? stat.size : 0, - mtime: stat ? stat.mtime : null, - }); + const fileName = FILE_LOGS[fileKey]; + if (!fileName) return res.status(400).json({ error: '未知日志文件' }); + const disk = await readDisk(fileName, linesLimit); + if (!disk) { + return res.status(200).json({ + file: fileKey, + name: fileName, + source: 'missing', + lines: [], + totalLines: 0, + size: 0, + mtime: null, + note: '无服务器部署没有常驻进程日志文件。爬虫输出在「crawler」里,来自数据库。', + }); + } + return res.status(200).json({ file: fileKey, name: fileName, source: 'file', ...disk }); } catch (error) { console.error('[file-logs] 读取失败:', error); return res.status(500).json({ error: `读取日志失败: ${error.message}` }); diff --git a/pages/api/admin/sites.js b/pages/api/admin/sites.js index cfa1815..f82250d 100644 --- a/pages/api/admin/sites.js +++ b/pages/api/admin/sites.js @@ -1,25 +1,27 @@ import { withAdmin } from '../../../utils/withAdmin'; -import { loadSiteConfig, saveSiteConfig, validateSite } from '../../../utils/siteConfig'; +import { loadEffectiveConfig, saveSiteConfig, validateSite } from '../../../utils/siteConfig'; /** * 站点管理 API - * GET /api/admin/sites -> 获取站点列表(读磁盘最新配置) + * GET /api/admin/sites -> 有效站点列表(数据库覆盖优先) * POST /api/admin/sites -> 添加站点 * PUT /api/admin/sites -> 编辑站点(body 传 { param: 原PARAM, ...站点字段 }) * DELETE /api/admin/sites -> 删除站点(body 或 query 传 { param }) + * + * 写入 settings.site:config。磁盘可写时同步刷新 wikitdb.config.js。 */ async function handler(req, res) { try { if (req.method === 'GET') { - const config = loadSiteConfig(); - return res.status(200).json({ sites: config.SUPPORT_WIKI }); + const config = await loadEffectiveConfig(); + return res.status(200).json({ sites: config.SUPPORT_WIKI, source: 'db-or-file' }); } if (req.method === 'POST') { const result = validateSite(req.body); if (result.error) return res.status(400).json({ error: result.error }); - const config = loadSiteConfig(); + const config = await loadEffectiveConfig(); if (config.SUPPORT_WIKI.some(s => s.PARAM === result.site.PARAM)) { return res.status(400).json({ error: `PARAM "${result.site.PARAM}" 已被使用` }); @@ -29,7 +31,7 @@ async function handler(req, res) { } config.SUPPORT_WIKI.push(result.site); - saveSiteConfig(config); + await saveSiteConfig(config); return res.status(200).json({ success: true, sites: config.SUPPORT_WIKI }); } @@ -40,11 +42,10 @@ async function handler(req, res) { const result = validateSite(req.body); if (result.error) return res.status(400).json({ error: result.error }); - const config = loadSiteConfig(); + const config = await loadEffectiveConfig(); const idx = config.SUPPORT_WIKI.findIndex(s => s.PARAM === param); if (idx === -1) return res.status(404).json({ error: '未找到该站点' }); - // 唯一性检查(排除自身) if (config.SUPPORT_WIKI.some((s, i) => i !== idx && s.PARAM === result.site.PARAM)) { return res.status(400).json({ error: `PARAM "${result.site.PARAM}" 已被其他站点使用` }); } @@ -53,14 +54,14 @@ async function handler(req, res) { } config.SUPPORT_WIKI[idx] = result.site; - saveSiteConfig(config); + await saveSiteConfig(config); return res.status(200).json({ success: true, message: `站点 "${result.site.NAME}" 已更新`, sites: config.SUPPORT_WIKI }); } if (req.method === 'DELETE') { const param = String((req.body && req.body.param) || req.query.param || '').trim(); - const config = loadSiteConfig(); + const config = await loadEffectiveConfig(); if (config.SUPPORT_WIKI.length <= 1) { return res.status(400).json({ error: '至少需要保留一个站点,不能删除全部站点' }); } @@ -69,14 +70,14 @@ async function handler(req, res) { if (idx === -1) return res.status(404).json({ error: '未找到该站点' }); config.SUPPORT_WIKI.splice(idx, 1); - saveSiteConfig(config); + await saveSiteConfig(config); return res.status(200).json({ success: true, sites: config.SUPPORT_WIKI }); } return res.status(405).json({ error: 'Method not allowed' }); } catch (error) { console.error('[sites] 处理失败:', error); - return res.status(500).json({ error: '站点管理失败,请检查 wikitdb.config.js 是否可写' }); + return res.status(500).json({ error: '站点管理失败。无服务器部署会把站点写进数据库,请确认 DATABASE_URL 可用。' }); } } diff --git a/pages/api/cron/attribution.js b/pages/api/cron/attribution.js new file mode 100644 index 0000000..8efb0d7 --- /dev/null +++ b/pages/api/cron/attribution.js @@ -0,0 +1,21 @@ +import { authorizeCron } from '../../../utils/cronAuth'; + +export const config = { maxDuration: 60 }; + +/** GET/POST /api/cron/attribution */ +export default async function handler(req, res) { + if (req.method !== 'GET' && req.method !== 'POST') { + return res.status(405).json({ error: 'Method not allowed' }); + } + if (!authorizeCron(req)) return res.status(401).json({ error: '未授权的调度请求' }); + + const force = req.query.force === '1' || req.query.force === 'true'; + try { + const { runAttributionSlice } = require('../../../jobs/attribution'); + const result = await runAttributionSlice({ force }); + return res.status(200).json({ ok: true, result }); + } catch (error) { + console.error('[cron/attribution]', error); + return res.status(500).json({ error: error.message || '归属同步失败' }); + } +} diff --git a/pages/api/cron/crawl.js b/pages/api/cron/crawl.js new file mode 100644 index 0000000..b472cf4 --- /dev/null +++ b/pages/api/cron/crawl.js @@ -0,0 +1,29 @@ +import { authorizeCron } from '../../../utils/cronAuth'; + +export const config = { maxDuration: 60 }; + +/** + * GET/POST /api/cron/crawl + * 无服务器调度入口。Vercel Cron 或 GitHub Actions 带 CRON_SECRET 调用。 + * 每次只推进一个时间片,游标在数据库里。 + */ +export default async function handler(req, res) { + if (req.method !== 'GET' && req.method !== 'POST') { + return res.status(405).json({ error: 'Method not allowed' }); + } + if (!authorizeCron(req)) return res.status(401).json({ error: '未授权的调度请求' }); + + const budget = Number(req.query.budget); + const force = req.query.force === '1' || req.query.force === 'true'; + try { + const { runCrawlSlice } = require('../../../jobs/crawler'); + const result = await runCrawlSlice({ + force, + budgetMs: Number.isFinite(budget) && budget > 0 ? Math.min(budget * 1000, 50 * 1000) : undefined, + }); + return res.status(200).json({ ok: true, result }); + } catch (error) { + console.error('[cron/crawl]', error); + return res.status(500).json({ error: error.message || '爬虫切片失败' }); + } +} diff --git a/pages/api/tools/delete-bot.js b/pages/api/tools/delete-bot.js index 521ceb1..2c8983d 100644 --- a/pages/api/tools/delete-bot.js +++ b/pages/api/tools/delete-bot.js @@ -70,6 +70,12 @@ function readLatestLog(lines = 40) { } async function handler(req, res) { + if (process.env.VERCEL) { + return res.status(501).json({ + success: false, + error: '宿主机删帖机器人依赖 systemd 和本地磁盘,无服务器部署请改用职员面板里的 BotAccount 自动扫描。', + }); + } // GET:状态 + 配置 + 日志 if (req.method === 'GET') { const active = run(`systemctl is-active ${SERVICE}`).toLowerCase() === 'active'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 049eeb0..cc8981c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,5 +1,6 @@ generator client { - provider = "prisma-client-js" + provider = "prisma-client-js" + binaryTargets = ["native", "rhel-openssl-3.0.x"] } datasource db { diff --git a/tests/cronAuth.test.js b/tests/cronAuth.test.js new file mode 100644 index 0000000..809a1a0 --- /dev/null +++ b/tests/cronAuth.test.js @@ -0,0 +1,42 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { authorizeCron } = require('../utils/cronAuth'); +const { normalizeLines } = require('../utils/jobLog'); + +function withEnv(patch, fn) { + const previous = {}; + for (const key of Object.keys(patch)) { + previous[key] = process.env[key]; + if (patch[key] == null) delete process.env[key]; + else process.env[key] = patch[key]; + } + try { + return fn(); + } finally { + for (const key of Object.keys(patch)) { + if (previous[key] == null) delete process.env[key]; + else process.env[key] = previous[key]; + } + } +} + +test('cron auth fails closed without a secret', () => { + withEnv({ CRON_SECRET: null, ALLOW_INSECURE_CRON: null, NODE_ENV: 'production', VERCEL: '1' }, () => { + assert.equal(authorizeCron({ headers: { authorization: 'Bearer whatever-token' } }), false); + }); +}); + +test('cron auth accepts the bearer secret and rejects a mismatch', () => { + withEnv({ CRON_SECRET: 'cron-secret-value-32chars!!', ALLOW_INSECURE_CRON: null, NODE_ENV: 'production', VERCEL: '1' }, () => { + assert.equal(authorizeCron({ headers: { authorization: 'Bearer cron-secret-value-32chars!!' } }), true); + assert.equal(authorizeCron({ headers: { 'x-cron-secret': 'cron-secret-value-32chars!!' } }), true); + assert.equal(authorizeCron({ headers: { authorization: 'Bearer nope' } }), false); + assert.equal(authorizeCron({ headers: {} }), false); + }); +}); + +test('job log normalizes stored lines', () => { + assert.deepEqual(normalizeLines({ lines: ['a', 'b'] }), ['a', 'b']); + assert.deepEqual(normalizeLines('["a"]'), ['a']); + assert.deepEqual(normalizeLines(null), []); +}); diff --git a/utils/cache.js b/utils/cache.js index c8a3e08..541ecf5 100644 --- a/utils/cache.js +++ b/utils/cache.js @@ -1,34 +1,14 @@ /** - * TTL 内存缓存 - * 支持不同过期时间,自动清理过期条目 + * 单实例 TTL 缓存。 + * 无服务器函数不能靠 setInterval 打扫(会拖住实例),读取时顺手丢掉过期项。 + * 命中范围只在当前温实例内;跨实例共享的数据放数据库或 Upstash。 */ const store = new Map(); -const DEFAULT_TTL = 5 * 60 * 1000; // 5 分钟 +const DEFAULT_TTL = 5 * 60 * 1000; const MAX_ENTRIES = 500; -let cleanupTimer = null; -function startCleanup() { - if (cleanupTimer) return; - cleanupTimer = setInterval(() => { - const now = Date.now(); - for (const [key, entry] of store) { - if (now > entry.expiresAt) store.delete(key); - } - if (store.size === 0) { - clearInterval(cleanupTimer); - cleanupTimer = null; - } - }, 60 * 1000); - if (cleanupTimer.unref) cleanupTimer.unref(); -} - -/** - * 获取缓存值 - * @param {string} key - * @returns {*} 缓存值或 undefined - */ function get(key) { const entry = store.get(key); if (!entry) return undefined; @@ -40,15 +20,10 @@ function get(key) { return entry.value; } -/** - * 设置缓存 - * @param {string} key - * @param {*} value - * @param {number} ttl 毫秒,默认 5 分钟 - */ function set(key, value, ttl = DEFAULT_TTL) { if (store.size >= MAX_ENTRIES) { - let oldest = null, oldestKey = null; + let oldest = null; + let oldestKey = null; for (const [k, v] of store) { if (!oldest || v.lastAccess < oldest.lastAccess) { oldest = v; @@ -58,29 +33,16 @@ function set(key, value, ttl = DEFAULT_TTL) { if (oldestKey) store.delete(oldestKey); } store.set(key, { value, expiresAt: Date.now() + ttl, lastAccess: Date.now() }); - startCleanup(); } -/** - * 删除缓存 - */ function del(key) { store.delete(key); } -/** - * 清空所有缓存 - */ function flush() { store.clear(); } -/** - * 带缓存的异步函数包装器 - * @param {string} key 缓存键 - * @param {Function} fn 异步函数 - * @param {number} ttl 缓存时间(毫秒) - */ async function cached(key, fn, ttl = DEFAULT_TTL) { const hit = get(key); if (hit !== undefined) return hit; diff --git a/utils/cronAuth.js b/utils/cronAuth.js new file mode 100644 index 0000000..bc01234 --- /dev/null +++ b/utils/cronAuth.js @@ -0,0 +1,38 @@ +/** + * 定时任务入口鉴权。 + * Vercel Cron 会带 Authorization: Bearer $CRON_SECRET。 + * 未配置 CRON_SECRET 时拒绝(避免把爬虫端点暴露到公网)。 + * 本地手动调试可设 ALLOW_INSECURE_CRON=1。 + */ +const crypto = require('crypto'); + +function header(req, name) { + const headers = (req && req.headers) || {}; + const direct = headers[name] ?? headers[name.toLowerCase()]; + if (Array.isArray(direct)) return String(direct[0] || ''); + return direct ? String(direct) : ''; +} + +function tokenFromRequest(req) { + const auth = header(req, 'authorization'); + if (/^Bearer\s+/i.test(auth)) return auth.replace(/^Bearer\s+/i, '').trim(); + return header(req, 'x-cron-secret').trim(); +} + +function safeEqual(a, b) { + const left = Buffer.from(String(a)); + const right = Buffer.from(String(b)); + if (left.length !== right.length) return false; + return crypto.timingSafeEqual(left, right); +} + +function authorizeCron(req) { + if (process.env.ALLOW_INSECURE_CRON === '1' && process.env.NODE_ENV !== 'production' && !process.env.VERCEL) { + return true; + } + const secret = process.env.CRON_SECRET || ''; + if (!secret || secret.length < 16) return false; + return safeEqual(tokenFromRequest(req), secret); +} + +module.exports = { authorizeCron, tokenFromRequest }; diff --git a/utils/jobLock.js b/utils/jobLock.js new file mode 100644 index 0000000..b52e643 --- /dev/null +++ b/utils/jobLock.js @@ -0,0 +1,53 @@ +/** + * 用 settings 行做互斥锁。 + * 无服务器平台会并发触发同一个 cron,锁保证同一任务只有一个切片在跑。 + */ +function normalize(value) { + if (!value) return null; + if (typeof value === 'string') { + try { return JSON.parse(value); } catch (e) { return null; } + } + return value; +} + +async function withJobLock(name, ttlMs, fn) { + const prisma = require('../lib/prisma'); + const key = `lock:${name}`; + const owner = `${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`; + const until = Date.now() + ttlMs; + + let state; + try { + state = await prisma.$transaction(async (tx) => { + const row = await tx.setting.findUnique({ where: { key } }); + const cur = normalize(row && row.value); + if (cur && Number(cur.until) > Date.now()) return 'locked'; + const value = { owner, until }; + await tx.setting.upsert({ + where: { key }, + update: { value }, + create: { key, value }, + }); + return 'ok'; + }, { isolationLevel: 'Serializable' }); + } catch (e) { + if (e.code === 'P2034') return { skipped: true, reason: 'locked' }; + throw e; + } + + if (state !== 'ok') return { skipped: true, reason: 'locked' }; + + try { + return await fn(); + } finally { + try { + const row = await prisma.setting.findUnique({ where: { key } }); + const cur = normalize(row && row.value); + if (cur && cur.owner === owner) { + await prisma.setting.delete({ where: { key } }); + } + } catch (e) { /* 锁过期由下一轮覆盖 */ } + } +} + +module.exports = { withJobLock }; diff --git a/utils/jobLog.js b/utils/jobLog.js new file mode 100644 index 0000000..a5339ff --- /dev/null +++ b/utils/jobLog.js @@ -0,0 +1,52 @@ +/** + * 任务日志写入 settings 表(logs:),替代 crawler.log 这类本地文件。 + * 无服务器实例的磁盘在调用结束后就没了,管理后台仍从数据库读最近若干行。 + */ +const MAX_LINES = 800; + +function normalizeLines(value) { + if (!value) return []; + const parsed = typeof value === 'string' ? safeJson(value) : value; + if (Array.isArray(parsed)) return parsed.map(String); + if (parsed && Array.isArray(parsed.lines)) return parsed.lines.map(String); + return []; +} + +function safeJson(text) { + try { return JSON.parse(text); } catch (e) { return null; } +} + +async function appendJobLogs(source, lines) { + const incoming = (Array.isArray(lines) ? lines : [lines]).map(String).filter(Boolean); + if (incoming.length === 0) return; + let prisma; + try { + prisma = require('../lib/prisma'); + } catch (e) { + return; + } + const key = `logs:${source}`; + try { + const row = await prisma.setting.findUnique({ where: { key } }); + const prev = normalizeLines(row && row.value); + const next = prev.concat(incoming).slice(-MAX_LINES); + await prisma.setting.upsert({ + where: { key }, + update: { value: { lines: next } }, + create: { key, value: { lines: next } }, + }); + } catch (e) { + console.error(`[job-log] ${source} 写入失败: ${e.message}`); + } +} + +async function readJobLogs(source, limit = 500) { + const prisma = require('../lib/prisma'); + const key = `logs:${source}`; + const row = await prisma.setting.findUnique({ where: { key } }); + const lines = normalizeLines(row && row.value); + const sliced = lines.slice(-Math.max(1, limit)); + return { lines: sliced, totalLines: lines.length }; +} + +module.exports = { appendJobLogs, readJobLogs, normalizeLines, MAX_LINES }; diff --git a/utils/siteConfig.js b/utils/siteConfig.js index 9baa4ac..2b0589b 100644 --- a/utils/siteConfig.js +++ b/utils/siteConfig.js @@ -1,7 +1,8 @@ /** - * wikitdb.config.js 运行时读写工具 - * 使用 vm 沙箱解析配置文件,避免 Next.js webpack 打包导致的 require 缓存问题, - * 确保读取到的始终是磁盘上的最新配置。 + * 站点配置。 + * 仓库里的 wikitdb.config.js 是默认值(会打进部署包)。 + * 管理后台的改动写入 settings.site:config,无服务器实例没有可写磁盘时也能生效。 + * 磁盘可写时仍会尝试刷新文件,方便自建虚拟机沿用原来的工作流。 */ const fs = require('fs'); @@ -9,13 +10,13 @@ const path = require('path'); const vm = require('vm'); const CONFIG_FILENAME = 'wikitdb.config.js'; +const CONFIG_KEY = 'site:config'; function getConfigPath() { return path.join(process.cwd(), CONFIG_FILENAME); } -/** 从磁盘加载配置(vm 沙箱执行,不污染全局) */ -function loadSiteConfig() { +function loadSiteConfigFromFile() { const filePath = getConfigPath(); const source = fs.readFileSync(filePath, 'utf8'); const sandbox = { module: { exports: {} }, exports: {} }; @@ -25,7 +26,37 @@ function loadSiteConfig() { return cfg; } -/** 使 Node 侧的 require 缓存失效(对 auto-crawler.js 等独立进程生效) */ +/** 同步读取磁盘配置。已打包的页面仍直接 require 该文件。 */ +function loadSiteConfig() { + return loadSiteConfigFromFile(); +} + +function normalizeStored(value) { + if (!value) return null; + if (typeof value === 'string') { + try { return JSON.parse(value); } catch (e) { return null; } + } + return value; +} + +/** 数据库覆盖优先,没有记录或数据库不可用时退回文件。 */ +async function loadEffectiveConfig() { + const fileCfg = loadSiteConfigFromFile(); + try { + const prisma = require('../lib/prisma'); + const row = await prisma.setting.findUnique({ where: { key: CONFIG_KEY } }); + const stored = normalizeStored(row && row.value); + if (stored && Array.isArray(stored.SUPPORT_WIKI)) { + return { + ...fileCfg, + ...stored, + SUPPORT_WIKI: stored.SUPPORT_WIKI, + }; + } + } catch (e) { /* 构建期或未连上数据库时用文件 */ } + return fileCfg; +} + function invalidateConfigCache() { try { const resolved = require.resolve(getConfigPath()); @@ -33,7 +64,6 @@ function invalidateConfigCache() { } catch (e) { /* 打包环境中可能无法解析,忽略 */ } } -/** 生成配置文件内容(保留站点模板注释,便于后续手工编辑) */ function buildConfigFileContent(config) { const lines = []; lines.push('module.exports = {'); @@ -61,10 +91,27 @@ function buildConfigFileContent(config) { return lines.join('\n') + '\n'; } -/** 写回配置文件并刷新缓存 */ -function saveSiteConfig(config) { - fs.writeFileSync(getConfigPath(), buildConfigFileContent(config), 'utf8'); - invalidateConfigCache(); +async function saveSiteConfig(config) { + const payload = { + SITE_NAME: config.SITE_NAME || 'WikitDB', + SITE_URL: config.SITE_URL || '', + SITE_SINCE: config.SITE_SINCE || '2026', + SITE_AUTHOR: config.SITE_AUTHOR || 'WikitDB Team', + SUPPORT_WIKI: config.SUPPORT_WIKI || [], + }; + const prisma = require('../lib/prisma'); + await prisma.setting.upsert({ + where: { key: CONFIG_KEY }, + update: { value: payload }, + create: { key: CONFIG_KEY, value: payload }, + }); + try { + fs.writeFileSync(getConfigPath(), buildConfigFileContent(payload), 'utf8'); + invalidateConfigCache(); + } catch (e) { + if (e.code !== 'EROFS' && e.code !== 'EACCES' && e.code !== 'EPERM' && e.code !== 'ENOENT') throw e; + } + return payload; } function isHttpUrl(value) { @@ -77,10 +124,8 @@ function isHttpUrl(value) { } } -/** 校验并规范化一个站点对象,返回 { site } 或 { error } */ function validateSite(input) { input = input || {}; - const site = {}; site.NAME = String(input.NAME || '').trim(); @@ -100,7 +145,6 @@ function validateSite(input) { return { error: 'WIKIT_ID 只能包含字母、数字、点、下划线和连字符(1-80 位)' }; } - // 可选字段 if (input.ImgURL) { const imgUrl = String(input.ImgURL).trim(); if (!isHttpUrl(imgUrl)) return { error: 'Logo 链接 (ImgURL) 必须是有效的 http(s) 链接' }; @@ -127,8 +171,11 @@ function validateSite(input) { module.exports = { getConfigPath, loadSiteConfig, + loadSiteConfigFromFile, + loadEffectiveConfig, saveSiteConfig, validateSite, buildConfigFileContent, invalidateConfigCache, + CONFIG_KEY, }; diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..3dff4a7 --- /dev/null +++ b/vercel.json @@ -0,0 +1,12 @@ +{ + "crons": [ + { + "path": "/api/cron/crawl", + "schedule": "0 18 * * *" + }, + { + "path": "/api/cron/attribution", + "schedule": "30 18 * * *" + } + ] +}