diff --git a/.env.example b/.env.example index 0fe4cbf..bce11bf 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,17 @@ -DATABASE_URL=postgresql://user:password@localhost:5432/wikitdb +# 定时任务密钥(至少 16 字符)。Vercel Cron 和 GitHub Actions 用它调用 /api/cron/*。 +# 生产环境必须设置,否则调度接口一律 401。 +CRON_SECRET= + +# 可选:Upstash Redis。配上之后全局限速在所有无服务器实例之间共享。 +# 不配则只在单个实例内计数。 +# UPSTASH_REDIS_REST_URL= +# UPSTASH_REDIS_REST_TOKEN= + +# 无服务器函数不要用「直连 + 大连接池」。 +# Neon / Supabase 请用带 pgbouncer 的 pooler 连接串,并带上 connection_limit=1。 +# 在 Vercel 上如果漏了 connection_limit,运行时会自动补上。 +DATABASE_URL=postgresql://user:password@localhost:5432/wikitdb?connection_limit=1 + JWT_SECRET=replace-with-at-least-32-random-bytes SITE_ORIGIN=https://example.com TRUST_PROXY=false diff --git a/.github/workflows/serverless-jobs.yml b/.github/workflows/serverless-jobs.yml new file mode 100644 index 0000000..9a0d8b8 --- /dev/null +++ b/.github/workflows/serverless-jobs.yml @@ -0,0 +1,126 @@ +name: Serverless jobs + +# 替代原来的常驻 worker / systemd。 +# 爬虫和归属同步优先在 Actions 里跑(单次最长约 8 分钟,游标写在数据库里)。 +# 没配 DATABASE_URL 时,改为请求已部署站点的 /api/cron/*(需要 SITE_ORIGIN 与 CRON_SECRET)。 +# 每天 12:00(Asia/Shanghai,04:00 UTC)用 wikit CLI 打备份包并上传 artifact。 + +on: + schedule: + - cron: '*/10 * * * *' + - cron: '0 4 * * *' + workflow_dispatch: + inputs: + job: + description: Which job to run + required: true + default: crawl + type: choice + options: + - crawl + - attribution + - backup + +permissions: + contents: read + +jobs: + crawl: + if: >- + (github.event_name == 'workflow_dispatch' && inputs.job == 'crawl') || + (github.event_name == 'schedule' && github.event.schedule == '*/10 * * * *') + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + WIKIDOT_BOT_USER: ${{ secrets.WIKIDOT_BOT_USER }} + WIKIDOT_BOT_PASS: ${{ secrets.WIKIDOT_BOT_PASS }} + SITE_ORIGIN: ${{ vars.SITE_ORIGIN }} + CRON_SECRET: ${{ secrets.CRON_SECRET }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + - name: Crawl with database + if: env.DATABASE_URL != '' + run: | + npm ci + npx prisma generate + node auto-crawler.js --for 480 + - name: Crawl via deployed cron endpoint + if: env.DATABASE_URL == '' && env.SITE_ORIGIN != '' && env.CRON_SECRET != '' + run: | + curl -fsS -H "Authorization: Bearer $CRON_SECRET" "$SITE_ORIGIN/api/cron/crawl" + - name: Skip notice + if: env.DATABASE_URL == '' && (env.SITE_ORIGIN == '' || env.CRON_SECRET == '') + run: echo "未配置 DATABASE_URL,也没有 SITE_ORIGIN + CRON_SECRET,跳过爬虫。在仓库 Secrets / Variables 里补上即可。" + + attribution: + if: >- + (github.event_name == 'workflow_dispatch' && inputs.job == 'attribution') || + (github.event_name == 'schedule' && github.event.schedule == '*/10 * * * *') + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + SITE_ORIGIN: ${{ vars.SITE_ORIGIN }} + CRON_SECRET: ${{ secrets.CRON_SECRET }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + - name: Attribution with database + if: env.DATABASE_URL != '' + run: | + npm ci + npx prisma generate + node auto-attribution.js --for 480 + - name: Attribution via deployed cron endpoint + if: env.DATABASE_URL == '' && env.SITE_ORIGIN != '' && env.CRON_SECRET != '' + run: | + curl -fsS -H "Authorization: Bearer $CRON_SECRET" "$SITE_ORIGIN/api/cron/attribution" + - name: Skip notice + if: env.DATABASE_URL == '' && (env.SITE_ORIGIN == '' || env.CRON_SECRET == '') + run: echo "未配置归属同步所需的 DATABASE_URL 或 SITE_ORIGIN + CRON_SECRET,跳过。" + + backup: + if: >- + (github.event_name == 'workflow_dispatch' && inputs.job == 'backup') || + (github.event_name == 'schedule' && github.event.schedule == '0 4 * * *') + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + - name: Skip notice + if: env.DATABASE_URL == '' + run: echo "备份需要 Secrets.DATABASE_URL 来读取站点列表。未配置,跳过。" + - name: Install wikit CLI + if: env.DATABASE_URL != '' + run: | + curl -fsSL -o /tmp/wikit "https://github.com/WikitTeam/wikit/releases/download/v0.2.3/wikit-linux-amd64" + chmod +x /tmp/wikit + sudo mv /tmp/wikit /usr/local/bin/wikit + - name: Run backup + if: env.DATABASE_URL != '' + run: | + npm ci + npx prisma generate + node auto-crawler.js --backup + - uses: actions/upload-artifact@v4 + if: env.DATABASE_URL != '' + with: + name: wikit-backup + path: wikit_data + if-no-files-found: ignore + retention-days: 14 + diff --git a/README.md b/README.md index 328a93f..b98d045 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ WikitDB 是一个面向 Wikidot 社区生态的**非营利性同人项目**。 | 论坛渲染 | FTML (Wikidot 文本标记语言) WASM 内核 | | HTML 消毒 | sanitize-html | | 鉴权 | JWT (HS256) + bcrypt + httpOnly Cookie | -| 定时任务 | node-cron | +| 定时任务 | GitHub Actions(`.github/workflows/serverless-jobs.yml`)或 Vercel Cron | | 邮件 | nodemailer | | 缓存 | Upstash Redis (可选) | | 语言 | JavaScript (逐步迁移至 TypeScript) | @@ -86,39 +86,52 @@ npm run dev 默认监听 `http://localhost:3000`。 -## 生产部署 +## 生产部署(无服务器) -### 构建 +站点本身是 Next.js,部署到 Vercel 后每个页面和 API 都是按需启动的函数,**不要再跑 `npm start` + PM2,也不要再跑常驻 worker**。 -```bash -npm run build -npm start -``` +状态都在 Postgres 里:爬虫游标(`crawler:cursor`)、运行状态(`crawler:status`)、任务日志(`logs:crawler`)、管理后台改过的站点列表(`site:config`)。函数结束不影响进度。 -### 备份 Worker(可选) +### 1. 部署 Web -如需启用 Wikidot 站点定时备份,需将 wikit CLI 安装到服务器 PATH 中,并以常驻进程方式运行 worker: +把仓库接到 Vercel(或任何支持 Next.js 的无服务器平台),配上: -```bash -npm run worker -``` +- `DATABASE_URL` — 用连接池地址(Neon / Supabase pooler),带 `connection_limit=1` +- `JWT_SECRET`、`SITE_ORIGIN` +- `CRON_SECRET` — 至少 16 字符,调度接口靠它鉴权 +- `UPSTASH_REDIS_REST_URL` / `UPSTASH_REDIS_REST_TOKEN` — 可选,配了之后限速跨实例共享 + +`vercel.json` 里有两条每天一次的 Cron(Hobby 计划不允许更密)。真正的推进频率靠下面的 GitHub Actions。 + +### 2. 调度爬虫和归属同步 -Worker 会在每天 12:00(Asia/Shanghai 时区)对所有 `wikitdb.config.js` 中声明的站点执行备份,归档写入工作目录下的 `wikit_data/`。任务限定为单进程串行执行,且服务端始终传递 `--keep-removed`,确保从 sitemap 消失的页面仍保留在归档中。 +`.github/workflows/serverless-jobs.yml` 每 10 分钟跑一片(最长约 8 分钟),游标没跑完下次接着跑。爬虫一轮结束后冷却 3 小时,归属同步冷却 30 分钟。 -### 进程管理 +在仓库 Secrets 里设置 `DATABASE_URL`(以及可选的 `WIKIDOT_BOT_USER` / `WIKIDOT_BOT_PASS`)。没有数据库密钥时,可以改成调用线上接口:仓库 Variables 设 `SITE_ORIGIN`,Secrets 设 `CRON_SECRET`。 -推荐使用 PM2 守护 `npm start` 与 `npm run worker` 两个进程: +本地手动跑一整轮或一片: ```bash -pm2 start "npm start" --name wikitdb -pm2 start "npm run worker" --name wikitdb-worker -pm2 save +node auto-crawler.js # 把当前一轮爬完后退出 +node auto-crawler.js --for 480 # 最多 480 秒 +node auto-attribution.js --slice ``` +`npm run worker` 仍指向爬虫脚本,但**跑完就退出**,不要再用 PM2 守护它。 + +### 3. 全站备份 + +`wikit` CLI 要二进制和磁盘,不能塞进函数。每天 12:00(北京时间)由同一条 Actions 工作流下载 `wikit`、执行 `node auto-crawler.js --backup`,把 `wikit_data/` 收成 artifact。 + +### 4. 仍然想自建虚拟机 + +可以 `npm run build && npm start`,但调度请用系统 cron 调用上面的一次性命令,不要再装 `deploy/wikitdb-attribution.service` 那种常驻进程。 + ## 项目结构 ``` WikitDB-Server/ +├── jobs/ # 无服务器任务切片(爬虫 / 归属 / 备份) ├── pages/ # Next.js Pages Router │ ├── api/ # API 路由 │ ├── tools/ # 工具页面 @@ -148,7 +161,9 @@ WikitDB-Server/ | `npm run typecheck` | TypeScript 类型检查 (`tsc --noEmit`) | | `npm test` | 运行单元测试 (`node --test tests/*.test.js`) | | `npm run test:ftml` | FTML 运行时测试 | -| `npm run worker` | 启动备份定时任务 worker | +| `npm run worker` / `npm run job:crawl` | 跑一轮爬虫后退出(不再常驻) | +| `npm run job:attribution` | 跑一轮归属同步后退出 | +| `npm run job:backup` | 调用本机 wikit CLI 做备份 | | `npm run prisma:generate` | 生成 Prisma Client | | `npm run prisma:push` | 同步 schema 到数据库 | | `npm run prisma:studio` | 打开 Prisma Studio 可视化管理数据库 | diff --git a/auto-attribution.js b/auto-attribution.js index b48cf8b..61083d2 100644 --- a/auto-attribution.js +++ b/auto-attribution.js @@ -1,148 +1,39 @@ /** - * WikitDB 归属资料独立定时任务 - * 与主爬虫(auto-crawler.js 逐页爬取)完全解耦: - * 每 30 分钟单独抓取各站点的「归属资料页」+ listpages 页面评分, - * 同步 author_attributions 表并聚合作者分数(author_score:{site})。 + * 一次性归属同步。跑完退出。 + * + * node auto-attribution.js + * node auto-attribution.js --for 300 + * node auto-attribution.js --slice + * node auto-attribution.js --force */ -const cron = require('node-cron'); -const http = require('http'); -const https = require('https'); -const fs = require('fs'); -const path = require('path'); -const prisma = require('./lib/prisma'); -const config = require('./wikitdb.config.js'); -const { fetchAttributionPage, aggregateAuthorScores, normalizePageKey } = require('./utils/attribution'); +const { runAttributionSlice } = require('./jobs/attribution'); -const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); -const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 }); -const request = require('axios').create({ - httpAgent, - httpsAgent, - timeout: 30000, - validateStatus: (s) => s >= 200 && s < 400 -}); - -const LOG_FILE = path.join(process.cwd(), 'crawler.log'); -const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); - -function logLine(...args) { - const line = args.map(String).join(' '); - console.log(line); - try { - fs.appendFileSync(LOG_FILE, `[${new Date().toLocaleString()}] ${line}\n`, 'utf8'); - } catch (e) { /* 忽略 */ } -} - -/** 获取站点全部页面及评分(listpages 分页) */ -async function fetchAllPages(siteConfig) { - const actualWikiName = siteConfig.URL.replace(/^https?:\/\//i, '').replace(/^www\./i, '').split('.')[0]; - const baseUrl = siteConfig.URL.replace(/\/$/, ''); - const pages = []; - let pageNum = 1; - let totalPages = 1; - - while (true) { - const res = await request.get(`https://wikit.unitreaty.org/listpages?wiki=${actualWikiName}&p=${pageNum}`); - const lines = String(res.data).split('\n').map(l => l.trim()).filter(Boolean); - - const totalMatch = lines.find(l => l.startsWith('Total Pages:'))?.match(/(\d+)/); - if (totalMatch) totalPages = parseInt(totalMatch[1], 10) || 1; - - let countThisPage = 0; - for (const line of lines) { - if (!line.startsWith('http')) continue; - const parts = line.split('|').map(s => s.trim()); - if (parts.length < 6) continue; - const pageSlug = parts[0].split('/').pop(); - pages.push({ - page: pageSlug, - title: parts[1] || '', - rating: parseInt(parts[3], 10) || 0, - upvotes: parseInt(parts[4], 10) || 0, - downvotes: parseInt(parts[5], 10) || 0 - }); - countThisPage++; - } - - if (pageNum >= totalPages || countThisPage === 0) break; - pageNum++; - await sleep(400); - } - return pages; -} - -/** 处理单个站点:抓归属页 + 页面评分 + 聚合作者分数 */ -async function processSite(siteConfig) { - const wikiParam = siteConfig.PARAM; - const attrRecords = await fetchAttributionPage(siteConfig, request); - if (attrRecords.length === 0) { - logLine(`[归属] ${wikiParam} 归属资料页无有效数据,跳过`); - return; - } - - // 1. 全量同步归属记录 - await prisma.$transaction([ - prisma.authorAttribution.deleteMany({ where: { siteParam: wikiParam } }), - prisma.authorAttribution.createMany({ - data: attrRecords.map(a => ({ siteParam: wikiParam, page: a.page, username: a.username, type: a.type, date: a.date })), - skipDuplicates: true - }) - ]); - - // 2. 获取页面评分(listpages) - const allPages = await fetchAllPages(siteConfig); - - // 3. 保存页面评分 - const pageScoreRows = allPages.map(p => ({ - page: p.page, title: p.title, rating: p.rating, - upvotes: p.upvotes, downvotes: p.downvotes - })); - await prisma.setting.upsert({ - where: { key: `page_scores:${wikiParam}` }, - update: { value: JSON.stringify(pageScoreRows) }, - create: { key: `page_scores:${wikiParam}`, value: JSON.stringify(pageScoreRows) } - }); - - // 4. 按归属聚合作者分数(每个归属用户获得页面全分) - const pageRatings = new Map(); - for (const pg of allPages) pageRatings.set(normalizePageKey(pg.page), { rating: pg.rating }); - const authorScores = aggregateAuthorScores(attrRecords, pageRatings); - await prisma.setting.upsert({ - where: { key: `author_score:${wikiParam}` }, - update: { value: JSON.stringify(authorScores) }, - create: { key: `author_score:${wikiParam}`, value: JSON.stringify(authorScores) } - }); - - logLine(`[归属] ${wikiParam} 归属 ${attrRecords.length} 条 | 页面评分 ${allPages.length} | 作者 ${Object.keys(authorScores).length} 位`); +function argNumber(flag) { + const index = process.argv.indexOf(flag); + if (index === -1 || process.argv[index + 1] == null) return null; + const value = Number(process.argv[index + 1]); + return Number.isFinite(value) ? value : null; } -let isRunning = false; - -async function runAttribution() { - if (isRunning) { - logLine('[归属] 上一轮尚未结束,跳过本次触发'); +async function main() { + const force = process.argv.includes('--force'); + if (process.argv.includes('--slice')) { + console.log(JSON.stringify(await runAttributionSlice({ force }))); return; } - isRunning = true; - const start = Date.now(); - try { - const sites = config.SUPPORT_WIKI.filter(w => w.ATTRIBUTION_PAGE); - logLine(`[归属] 开始执行(${sites.length} 个站点配置了归属资料页)...`); - for (const siteConfig of sites) { - try { - await processSite(siteConfig); - } catch (e) { - logLine(`[归属] ${siteConfig.PARAM} 处理失败: ${e.message}`); - } - } - logLine(`[归属] 执行完成,耗时 ${Math.round((Date.now() - start) / 1000)}s`); - } catch (e) { - logLine(`[归属] 发生异常: ${e.message}`); - } finally { - isRunning = false; + const forSec = argNumber('--for'); + const deadline = forSec == null ? Infinity : Date.now() + forSec * 1000; + let last = null; + while (Date.now() < deadline) { + last = await runAttributionSlice({ force }); + if (!last || last.skipped || last.done || last.error) break; } + console.log(JSON.stringify(last)); } -// 每 30 分钟执行一次;启动时立即执行一轮 -cron.schedule('*/30 * * * *', () => runAttribution()); -runAttribution(); +main() + .then(() => process.exit(process.exitCode || 0)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/auto-crawler.js b/auto-crawler.js index 096ae16..fac2710 100644 --- a/auto-crawler.js +++ b/auto-crawler.js @@ -1,506 +1,49 @@ -const cron = require('node-cron'); -const axios = require('axios'); -const cheerio = require('cheerio'); -const http = require('http'); -const https = require('https'); -const { spawn } = require('child_process'); -const prisma = require('./lib/prisma'); -const config = require('./wikitdb.config.js'); -const { buildBackupArgs, getWikiName } = require('./utils/wikitBackup'); -const fs = require('fs'); -const path = require('path'); - -const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); -const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 }); -const request = axios.create({ - httpAgent, - httpsAgent, - timeout: 15000 -}); - -const sleep = ms => new Promise(r => setTimeout(r, ms)); - -const CRAWLER_LOG_FILE = path.join(process.cwd(), 'crawler.log'); - -/** 同时输出到控制台和 crawler.log,方便管理后台查看 */ -function logLine(...args) { - const line = args.map(String).join(' '); - console.log(line); - try { - fs.appendFileSync(CRAWLER_LOG_FILE, `[${new Date().toLocaleString()}] ${line}\n`, 'utf8'); - } catch (e) { /* 日志文件写入失败不影响主流程 */ } -} - -const CRAWLER_STATUS_KEY = 'crawler:status'; - -let crawlStatus = { - running: false, - startedAt: null, - finishedAt: null, - currentSite: null, - currentStage: '', - overall: { totalSites: 0, doneSites: 0 }, - sites: [], - lastRun: null -}; - -/** 将当前爬取状态持久化到数据库(管理后台 /api/admin/crawler-status 读取) */ -async function persistCrawlStatus() { - try { - await prisma.setting.upsert({ - where: { key: CRAWLER_STATUS_KEY }, - update: { value: JSON.stringify(crawlStatus) }, - create: { key: CRAWLER_STATUS_KEY, value: JSON.stringify(crawlStatus) } - }); - } catch (e) { - console.error(`[crawler-status] 状态持久化失败: ${e.message}`); - } -} - -function buildSiteStatus(siteConfig) { - return { - param: siteConfig.PARAM, - name: siteConfig.NAME, - status: 'pending', - pagesFound: 0, - pagesProcessed: 0, - votes: 0, - discussions: 0, - errors: 0, - startedAt: null, - finishedAt: null, - lastRun: crawlStatus.lastRun, - error: null - }; -} - -let botCookieCache = null; - -async function getBotCookie() { - if (botCookieCache) return botCookieCache; - const user = process.env.WIKIDOT_BOT_USER; - const pass = process.env.WIKIDOT_BOT_PASS; - if (!user || !pass) { - console.log("未配置机器人账号,将以访客身份进行抓取..."); - return null; - } - - try { - const payload = new URLSearchParams({ login: user, password: pass, action: 'Login2Action', event: 'login' }); - const res = await axios.post('https://www.wikidot.com/default--flow/login__LoginPopupScreen', payload.toString(), { - headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'WikitDB-Bot/1.0' }, - maxRedirects: 0, - validateStatus: status => status >= 200 && status < 400 - }); - let sessionId = ''; - const cookies = res.headers['set-cookie'] || []; - for (const c of cookies) { - if (c.includes('WIKIDOT_SESSION_ID=')) { - sessionId = c.split('WIKIDOT_SESSION_ID=')[1].split(';')[0]; - } - } - if (sessionId) { - botCookieCache = `WIKIDOT_SESSION_ID=${sessionId}; wikidot_token7=123456;`; - console.log("机器人账号登录成功,已获取受限站点抓取权限。"); - } - } catch (e) { - console.error('获取 Bot Cookie 失败:', e.message); - } - return botCookieCache; +/** + * 一次性爬虫 / 备份入口。进程跑完就退出,不再常驻,也不再内置 node-cron。 + * + * node auto-crawler.js 把当前这一轮爬完(切片续跑)后退出 + * node auto-crawler.js --for 480 最多跑 480 秒,游标留下次继续 + * node auto-crawler.js --slice 只跑一个默认时间片(约 45 秒) + * node auto-crawler.js --backup 调 wikit CLI 做全站备份(不能在 Vercel 函数里跑) + * node auto-crawler.js --force 忽略 3 小时冷却,立刻开新一轮 + */ +const { runCrawlSlice, runBackup } = require('./jobs/crawler'); + +function argNumber(flag) { + const index = process.argv.indexOf(flag); + if (index === -1 || process.argv[index + 1] == null) return null; + const value = Number(process.argv[index + 1]); + return Number.isFinite(value) ? value : null; } -let isRunning = false; - -async function runCrawler() { - if (isRunning) { - logLine(`[${new Date().toLocaleString()}] 警告:上一轮爬虫尚未结束,跳过本次触发。`); +async function main() { + const force = process.argv.includes('--force'); + if (process.argv.includes('--backup')) { + const result = await runBackup(); + console.log(JSON.stringify(result)); + if (result && result.exitCode && result.exitCode !== 0) process.exitCode = result.exitCode; return; } - isRunning = true; - - try { - const botCookie = await getBotCookie(); - const baseHeaders = { 'User-Agent': 'Mozilla/5.0' }; - if (botCookie) baseHeaders['Cookie'] = botCookie; - - logLine(`\n[${new Date().toLocaleString()}] 开始执行全站数据(评分+讨论区)爬取...`); - crawlStatus = { - running: true, - startedAt: Date.now(), - finishedAt: null, - currentSite: null, - currentStage: '', - overall: { totalSites: config.SUPPORT_WIKI.length, doneSites: 0 }, - sites: config.SUPPORT_WIKI.map(buildSiteStatus), - lastRun: crawlStatus.lastRun - }; - await persistCrawlStatus(); - for (const siteConfig of config.SUPPORT_WIKI) { - const wikiParam = siteConfig.PARAM; - const actualWikiName = siteConfig.URL.replace(/^https?:\/\//i, '').split('.')[0]; - const baseUrl = siteConfig.URL.replace(/\/$/, ''); - - let siteVotes = 0, siteDiscussions = 0, siteErrors = 0; - let siteStatus = crawlStatus.sites.find(s => s.param === wikiParam); - if (siteStatus) { - siteStatus.status = 'running'; - siteStatus.startedAt = Date.now(); - siteStatus.error = null; - crawlStatus.currentSite = wikiParam; - crawlStatus.currentStage = 'list'; - await persistCrawlStatus(); - } - - let allPages = []; - let pageNum = 1; - let totalPages = 1; - let hasMore = true; - - while (hasMore) { - try { - process.stdout.write(`获取 [${wikiParam}] 清单 第 ${pageNum} 页... `); - const res = await request.get(`https://wikit.unitreaty.org/listpages?wiki=${actualWikiName}&p=${pageNum}`); - const lines = res.data.split('\n').map(l => l.trim()).filter(Boolean); - let countThisPage = 0; - - lines.forEach(line => { - if (line.startsWith('Total Pages:')) { - totalPages = parseInt(line.replace('Total Pages:', '').trim(), 10) || 1; - } else if (line.includes('http') && line.includes('|')) { - const parts = line.split('|').map(item => item.trim()); - if (parts.length >= 7) { - const url = parts[0]; - const pageSlug = url.split('/').pop(); - let author = parts[6] || '未知'; - const match = author.match(/^(.*?)\s*\(\d+\)$/); - if (match) author = match[1].trim(); - - allPages.push({ page: pageSlug, title: parts[1], author: author, wiki: wikiParam, rating: parseInt(parts[3], 10) || 0, upvotes: parseInt(parts[4], 10) || 0, downvotes: parseInt(parts[5], 10) || 0 }); - countThisPage++; - } - } - }); - logLine(`成功 ${countThisPage} 篇`); - if (pageNum >= totalPages) hasMore = false; - else pageNum++; - } catch (e) { - await sleep(3000); - } - await sleep(1000); - } - - if (siteStatus) { - siteStatus.pagesFound = allPages.length; - crawlStatus.currentStage = 'crawl'; - } - await persistCrawlStatus(); - - let userVotesMap = {}; - let count = 0; - const CONCURRENCY = 3; - - for (let i = 0; i < allPages.length; i += CONCURRENCY) { - const batch = allPages.slice(i, i + CONCURRENCY); - await Promise.all(batch.map(async (pageNode) => { - const secureUrl = `${baseUrl}/${pageNode.page}`; - let success = false, attempt = 0; - - while (!success && attempt < 3) { - attempt++; - try { - const { data: html } = await request.get(secureUrl, { headers: baseHeaders }); - - const $page = cheerio.load(html); - let threadId = null; - - let href = $page('#discuss-button').attr('href'); - if (!href) href = $page('#page-info a').filter((_, el) => ($page(el).attr('href') || '').includes('/forum/t-')).attr('href'); - if (!href) href = $page('#page-content').parent().find('a').filter((_, el) => { - const text = $page(el).text().toLowerCase(); - return (text.includes('discuss') || text.includes('讨论') || text.includes('评论')) && ($page(el).attr('href') || '').includes('/forum/t-'); - }).attr('href'); - - if (href) { - const match = href.match(/\/forum\/t-(\d+)/); - if (match) threadId = match[1]; - } - - const cacheKey = `forum_v7:${wikiParam}:${pageNode.page}`; - - if (threadId) { - siteDiscussions++; - try { - const forumUrl = `${baseUrl}/forum/t-${threadId}`; - const { data: forumHtml } = await request.get(forumUrl, { headers: baseHeaders }); - const $forum = cheerio.load(forumHtml); - const posts = []; - const userIdCache = {}; - - const postElements = $forum('.post').toArray(); - for (const el of postElements) { - const $el = $forum(el); - const postId = ($el.attr('id') || '').replace('post-', ''); - if (!postId) continue; - - let parentId = null; - const $parentContainer = $el.parent('.post-container').parent('.post-container'); - if ($parentContainer.length) { - const $parentPost = $parentContainer.children('.post').first(); - parentId = ($parentPost.attr('id') || '').replace('post-', ''); - } - - let author = '未知用户'; - const $printUser = $el.find('.head .printuser').length ? $el.find('.head .printuser').first() : $el.find('.info .printuser').first(); - - if ($printUser.length) { - const $links = $printUser.find('a'); - if ($links.length) author = $links.last().text().trim(); - else author = $printUser.text().trim(); - } else { - author = $el.find('.head .author, .info .author').first().text().trim() || '未知用户'; - } - author = author.replace(/[\r\n\t]+/g, '').trim(); - - let userid = null; - const headHtml = $el.find('.head').html() || $el.find('.info').html() || $el.html() || ''; - const srcMatch = headHtml.match(/avatar\.php\?userid=(\d+)/i); - const clickMatch = headHtml.match(/userInfo\(\s*(\d+)\s*\)/i); - const karmaMatch = headHtml.match(/userkarma\.php\?u=(\d+)/i); - - if (srcMatch) userid = srcMatch[1]; - else if (clickMatch) userid = clickMatch[1]; - else if (karmaMatch) userid = karmaMatch[1]; - - if (!userid && author !== '未知用户') { - if (userIdCache[author]) { - userid = userIdCache[author]; - } else { - try { - const lookupRes = await axios.get(`https://www.wikidot.com/quickmodule.php?module=UserLookupQModule&q=${encodeURIComponent(author)}`, { timeout: 5000 }); - if (lookupRes.data && lookupRes.data.users && lookupRes.data.users.length > 0) { - userid = lookupRes.data.users[0].user_id; - userIdCache[author] = userid; - } - } catch (lookupErr) { - // 忽略查询报错 - } - } - } - - let avatarUrl = ''; - if (userid) { - const currentTs = Math.floor(Date.now() / 1000); - avatarUrl = `https://www.wikidot.com/avatar.php?userid=${userid}×tamp=${currentTs}`; - } else { - avatarUrl = `https://www.wikidot.com/avatar.php?account=default`; - } - - const contentHtml = $el.find('.content').html() || ''; - const odate = $el.find('.odate').first(); - const odateClass = odate.attr('class') || ''; - const timeMatch = odateClass.match(/time_(\d+)/); - - let timestamp = odate.text().trim(); - if (timeMatch) { - const dateObj = new Date(parseInt(timeMatch[1]) * 1000); - timestamp = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')} ${String(dateObj.getHours()).padStart(2, '0')}:${String(dateObj.getMinutes()).padStart(2, '0')}`; - } else if (odate.attr('title')) timestamp = odate.attr('title'); - - posts.push({ postId, parentId, author, avatarUrl, timestamp, contentHtml, children: [] }); - } - - const postMap = {}; - const rootPosts = []; - posts.forEach(p => { p.children = []; postMap[p.postId] = p; }); - posts.forEach(p => { - if (p.parentId && postMap[p.parentId]) postMap[p.parentId].children.push(p); - else rootPosts.push(p); - }); - - const forumData = { threadId, url: forumUrl, total: posts.length, threads: rootPosts }; - - await prisma.setting.upsert({ - where: { key: cacheKey }, - update: { value: JSON.stringify(forumData) }, - create: { key: cacheKey, value: JSON.stringify(forumData) } - }); - console.log(`[成功] 讨论区入库: ${pageNode.page} (${posts.length}条)`); - } catch (forumErr) { - console.error(`[失败] ${pageNode.page} 讨论区抓取报错: ${forumErr.message}`); - } - } else { - const emptyData = { threadId: null, url: '', total: 0, threads: [] }; - await prisma.setting.upsert({ - where: { key: cacheKey }, - update: { value: JSON.stringify(emptyData) }, - create: { key: cacheKey, value: JSON.stringify(emptyData) } - }); - } - - let pageId = null; - const idMatch = html.match(/pageId\s*[:=]\s*['"]?(\d+)['"]?/i) || html.match(/page_id\s*[:=]\s*['"]?(\d+)['"]?/i); - if (idMatch) pageId = idMatch[1]; - if (!pageId) { success = true; return; } - - const origin = new URL(secureUrl).origin; - const ajaxUrl = `${origin}/ajax-module-connector.php`; - - const ajaxHeaders = { - 'User-Agent': 'Mozilla/5.0', - 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'Cookie': botCookie ? botCookie : 'wikidot_token7=123456;' - }; - const { data: rateData } = await request.post(ajaxUrl, `pageId=${pageId}&page_id=${pageId}&moduleName=pagerate/WhoRatedPageModule&wikidot_token7=123456`, { - headers: ajaxHeaders - }); - - if (rateData.status === 'ok' && rateData.body) { - const $rate = cheerio.load(rateData.body); - $rate('.printuser').each((_, el) => { - const user = $rate(el).text().trim(); - let vote = '+1', textAfter = '', curr = el.next; - while (curr) { - if (curr.type === 'tag' && (curr.tagName === 'br' || (curr.attribs && curr.attribs.class && curr.attribs.class.includes('printuser')))) break; - if (curr.type === 'text') textAfter += curr.data; - else if (curr.type === 'tag') textAfter += $rate(curr).text(); - curr = curr.next; - } - if (textAfter.includes('-')) vote = '-1'; - if (!userVotesMap[user]) userVotesMap[user] = []; - userVotesMap[user].push({ wiki: wikiParam, page: pageNode.page, title: pageNode.title, vote: vote, author: pageNode.author, date: Date.now() }); - siteVotes++; - }); - } - success = true; - } catch (err) { - siteErrors++; - if (attempt < 3) await sleep(2000); - } - } - })); - - count += batch.length; - logLine(`--- 当前进度: [${count}/${allPages.length}] ---`); - - if (count % 100 === 0 || count >= allPages.length) { - if (siteStatus) { - siteStatus.pagesProcessed = count; - siteStatus.votes = siteVotes; - siteStatus.discussions = siteDiscussions; - siteStatus.errors = siteErrors; - await persistCrawlStatus(); - } - for (const [user, newVotes] of Object.entries(userVotesMap)) { - const key = `user_votes_${user.toLowerCase().replace(/_/g, '-').replace(/ /g, '-')}`; - const record = await prisma.setting.findUnique({ where: { key } }); - let existingMap = new Map(); - if (record) { - // lib/prisma.js 的 setting 扩展已自动解析 value 为对象,避免二次 JSON.parse 崩溃 - const parsed = typeof record.value === 'string' ? JSON.parse(record.value) : record.value; - if (Array.isArray(parsed)) { - parsed.forEach(v => existingMap.set(`${v.wiki}:${v.page}`, v)); - } - } - - newVotes.forEach(nv => { - const id = `${nv.wiki}:${nv.page}`; - if (!existingMap.has(id)) existingMap.set(id, nv); - else if (existingMap.get(id).vote !== nv.vote) { - existingMap.get(id).vote = nv.vote; - existingMap.get(id).date = Date.now(); - } - }); - - const truncatedVotes = Array.from(existingMap.values()).sort((a, b) => b.date - a.date).slice(0, 800); - await prisma.setting.upsert({ - where: { key }, - update: { value: JSON.stringify(truncatedVotes) }, - create: { key, value: JSON.stringify(truncatedVotes) } - }); - } - userVotesMap = {}; - } - await sleep(2500); - } - - if (siteStatus) { - siteStatus.status = 'done'; - siteStatus.finishedAt = Date.now(); - siteStatus.lastRun = Date.now(); - siteStatus.pagesProcessed = count; - siteStatus.votes = siteVotes; - siteStatus.discussions = siteDiscussions; - siteStatus.errors = siteErrors; - crawlStatus.overall.doneSites = crawlStatus.sites.filter(s => s.status === 'done').length; - crawlStatus.currentSite = null; - crawlStatus.currentStage = 'done'; - await persistCrawlStatus(); - } - } - } catch (e) { - logLine(`发生异常: ${e.message}`); - crawlStatus.sites.forEach(s => { - if (s.status === 'running') { - s.status = 'error'; - s.error = e.message; - } - }); - await persistCrawlStatus(); - } finally { - isRunning = false; - crawlStatus.running = false; - crawlStatus.finishedAt = Date.now(); - crawlStatus.lastRun = Date.now(); - crawlStatus.currentSite = null; - crawlStatus.currentStage = ''; - crawlStatus.overall.doneSites = crawlStatus.sites.filter(s => s.status === 'done').length; - await persistCrawlStatus(); - } -} - -cron.schedule('0 */3 * * *', () => runCrawler()); -runCrawler(); -let isBackupRunning = false; - -async function runWikitBackup() { - if (isBackupRunning) { - logLine(`[${new Date().toLocaleString()}] Wikit backup is already running; skipping.`); + if (process.argv.includes('--slice')) { + const result = await runCrawlSlice({ force }); + console.log(JSON.stringify(result)); return; } - const wikiNames = config.SUPPORT_WIKI.map(getWikiName).filter(Boolean); - if (wikiNames.length === 0) { - logLine(`[${new Date().toLocaleString()}] No valid wikis configured for backup.`); - return; + const forSec = argNumber('--for'); + const deadline = forSec == null ? Infinity : Date.now() + forSec * 1000; + let last = null; + while (Date.now() < deadline) { + last = await runCrawlSlice({ force }); + if (!last || last.skipped || last.done || last.error) break; } - - isBackupRunning = true; - const args = buildBackupArgs({ wikiNames, keepRemoved: true }); - logLine(`[${new Date().toLocaleString()}] Starting wikit backup for ${wikiNames.length} wikis.`); - - await new Promise((resolve) => { - const child = spawn('wikit', args, { - cwd: process.cwd(), - env: process.env, - shell: false, - windowsHide: true, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - child.stdout.on('data', (chunk) => process.stdout.write(`[wikit] ${chunk}`)); - child.stderr.on('data', (chunk) => process.stderr.write(`[wikit] ${chunk}`)); - child.on('error', (error) => { - logLine(`[${new Date().toLocaleString()}] Failed to start wikit: ${error.message}`); - }); - child.on('close', (code) => { - logLine(`[${new Date().toLocaleString()}] Wikit backup finished with exit code ${code}.`); - resolve(); - }); - }); - - isBackupRunning = false; + console.log(JSON.stringify(last)); } -// Back up every configured wiki daily at 12:00 China Standard Time. -cron.schedule('0 12 * * *', () => runWikitBackup(), { timezone: 'Asia/Shanghai' }); +main() + .then(() => process.exit(process.exitCode || 0)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/deploy/wikitdb-attribution.service b/deploy/wikitdb-attribution.service index ecf67f3..5783990 100644 --- a/deploy/wikitdb-attribution.service +++ b/deploy/wikitdb-attribution.service @@ -1,19 +1,23 @@ +# DEPRECATED +# 无服务器部署不要启用这个单元。归属同步由 /api/cron/attribution +# 或 .github/workflows/serverless-jobs.yml 触发,进程跑完即退出。 +# 若仍在虚拟机上临时使用:不要 Restart=always,否则冷却期也会被立刻拉起。 + [Unit] -Description=WikitDB Attribution Sync (fetch attribution pages + compute author scores every 30min) +Description=WikitDB Attribution Sync (one-shot; scheduler should invoke this) After=network.target postgresql.service Wants=postgresql.service [Service] -Type=simple +Type=oneshot User=root WorkingDirectory=/opt/WikitDB-Server EnvironmentFile=/opt/WikitDB-Server/.env Environment=NODE_ENV=production Environment=NODE_OPTIONS=--max-old-space-size=1024 Environment=PATH=/root/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ExecStart=/usr/bin/node /opt/WikitDB-Server/auto-attribution.js -Restart=always -RestartSec=10 +ExecStart=/usr/bin/node /opt/WikitDB-Server/auto-attribution.js --for 480 +Restart=no [Install] WantedBy=multi-user.target diff --git a/jobs/attribution.js b/jobs/attribution.js new file mode 100644 index 0000000..238ab1e --- /dev/null +++ b/jobs/attribution.js @@ -0,0 +1,183 @@ +/** + * 归属资料同步。按站点、按清单页切片,避免一次函数调用爬完整站。 + * 调度间隔 30 分钟:一轮跑完后游标进入 idle,到期再开下一轮。 + */ +const http = require('http'); +const https = require('https'); +const prisma = require('../lib/prisma'); +const { loadEffectiveConfig } = require('../utils/siteConfig'); +const { fetchAttributionPage, aggregateAuthorScores, normalizePageKey } = require('../utils/attribution'); +const { appendJobLogs } = require('../utils/jobLog'); +const { withJobLock } = require('../utils/jobLock'); + +const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); +const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 }); +const request = require('axios').create({ + httpAgent, + httpsAgent, + timeout: 30000, + validateStatus: (s) => s >= 200 && s < 400, +}); + +const INTERVAL_MS = 30 * 60 * 1000; +const DEFAULT_BUDGET_MS = 45 * 1000; +const CURSOR_KEY = 'attribution:cursor'; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const logBuffer = []; + +function logLine(...args) { + const line = args.map(String).join(' '); + console.log(line); + logBuffer.push(`[${new Date().toLocaleString()}] ${line}`); +} + +async function flushLogs() { + if (logBuffer.length === 0) return; + const lines = logBuffer.splice(0, logBuffer.length); + await appendJobLogs('crawler', lines); +} + +function normalize(value) { + if (!value) return null; + if (typeof value === 'string') { + try { return JSON.parse(value); } catch (e) { return null; } + } + return value; +} + +function wikiHost(siteConfig) { + return siteConfig.URL.replace(/^https?:\/\//i, '').replace(/^www\./i, '').split('.')[0]; +} + +async function fetchListPage(cursor, siteConfig) { + const res = await request.get(`https://wikit.unitreaty.org/listpages?wiki=${wikiHost(siteConfig)}&p=${cursor.listPage}`); + const lines = String(res.data).split('\n').map((l) => l.trim()).filter(Boolean); + const totalMatch = lines.find((l) => l.startsWith('Total Pages:'))?.match(/(\d+)/); + if (totalMatch) cursor.totalPages = parseInt(totalMatch[1], 10) || 1; + let countThisPage = 0; + for (const line of lines) { + if (!line.startsWith('http')) continue; + const parts = line.split('|').map((s) => s.trim()); + if (parts.length < 6) continue; + cursor.pages.push({ + page: parts[0].split('/').pop(), + title: parts[1] || '', + rating: parseInt(parts[3], 10) || 0, + upvotes: parseInt(parts[4], 10) || 0, + downvotes: parseInt(parts[5], 10) || 0, + }); + countThisPage++; + } + if (cursor.listPage >= cursor.totalPages || countThisPage === 0) cursor.phase = 'commit'; + else cursor.listPage += 1; +} + +async function commitSite(cursor, siteConfig) { + const wikiParam = siteConfig.PARAM; + const attrRecords = cursor.records || []; + await prisma.$transaction([ + prisma.authorAttribution.deleteMany({ where: { siteParam: wikiParam } }), + prisma.authorAttribution.createMany({ + data: attrRecords.map((a) => ({ + siteParam: wikiParam, + page: a.page, + username: a.username, + type: a.type, + date: a.date, + })), + skipDuplicates: true, + }), + ]); + const pageScoreRows = (cursor.pages || []).map((p) => ({ + page: p.page, title: p.title, rating: p.rating, upvotes: p.upvotes, downvotes: p.downvotes, + })); + await prisma.setting.upsert({ + where: { key: `page_scores:${wikiParam}` }, + update: { value: pageScoreRows }, + create: { key: `page_scores:${wikiParam}`, value: pageScoreRows }, + }); + const pageRatings = new Map(); + for (const pg of cursor.pages || []) pageRatings.set(normalizePageKey(pg.page), { rating: pg.rating }); + const authorScores = aggregateAuthorScores(attrRecords, pageRatings); + await prisma.setting.upsert({ + where: { key: `author_score:${wikiParam}` }, + update: { value: authorScores }, + create: { key: `author_score:${wikiParam}`, value: authorScores }, + }); + logLine(`[归属] ${wikiParam} 归属 ${attrRecords.length} 条 | 页面评分 ${pageScoreRows.length} | 作者 ${Object.keys(authorScores).length} 位`); +} + +async function runAttributionSlice(opts = {}) { + const budgetMs = Number(opts.budgetMs) > 0 ? Number(opts.budgetMs) : DEFAULT_BUDGET_MS; + const started = Date.now(); + const over = () => Date.now() - started >= budgetMs; + + return withJobLock('attribution', budgetMs + 30 * 1000, async () => { + const sites = (((await loadEffectiveConfig()).SUPPORT_WIKI) || []).filter((w) => w.ATTRIBUTION_PAGE); + if (sites.length === 0) return { skipped: true, reason: 'no-sites' }; + + let cursor = normalize((await prisma.setting.findUnique({ where: { key: CURSOR_KEY } }))?.value); + if (!cursor || cursor.phase === 'idle') { + if (cursor && cursor.nextCycleAt && Date.now() < cursor.nextCycleAt && !opts.force) { + return { skipped: true, reason: 'cooldown', nextCycleAt: cursor.nextCycleAt }; + } + logLine(`[归属] 开始执行(${sites.length} 个站点配置了归属资料页)...`); + cursor = { phase: 'fetch', siteIndex: 0, listPage: 1, totalPages: 1, pages: [], records: [], nextCycleAt: 0 }; + } + + try { + while (!over()) { + if (cursor.siteIndex >= sites.length) { + cursor = { phase: 'idle', siteIndex: 0, pages: [], records: [], nextCycleAt: Date.now() + INTERVAL_MS }; + await prisma.setting.upsert({ + where: { key: CURSOR_KEY }, + update: { value: cursor }, + create: { key: CURSOR_KEY, value: cursor }, + }); + logLine('[归属] 本轮执行完成。'); + await flushLogs(); + return { done: true, phase: 'idle' }; + } + + const siteConfig = sites[cursor.siteIndex]; + if (cursor.phase === 'fetch') { + cursor.records = await fetchAttributionPage(siteConfig, request); + cursor.pages = []; + cursor.listPage = 1; + cursor.totalPages = 1; + if (!cursor.records || cursor.records.length === 0) { + logLine(`[归属] ${siteConfig.PARAM} 归属资料页无有效数据,跳过`); + cursor.siteIndex += 1; + cursor.phase = 'fetch'; + } else { + cursor.phase = 'list'; + } + } else if (cursor.phase === 'list') { + await fetchListPage(cursor, siteConfig); + await sleep(200); + } else if (cursor.phase === 'commit') { + await commitSite(cursor, siteConfig); + cursor.siteIndex += 1; + cursor.phase = 'fetch'; + cursor.pages = []; + cursor.records = []; + } + + await prisma.setting.upsert({ + where: { key: CURSOR_KEY }, + update: { value: cursor }, + create: { key: CURSOR_KEY, value: cursor }, + }); + } + } catch (e) { + logLine(`[归属] ${sites[cursor.siteIndex] ? sites[cursor.siteIndex].PARAM : '?'} 处理失败: ${e.message}`); + await flushLogs(); + return { done: false, error: e.message, phase: cursor.phase, siteIndex: cursor.siteIndex }; + } + + await flushLogs(); + return { done: false, phase: cursor.phase, siteIndex: cursor.siteIndex, listPage: cursor.listPage }; + }); +} + +module.exports = { runAttributionSlice, INTERVAL_MS }; diff --git a/jobs/crawler.js b/jobs/crawler.js new file mode 100644 index 0000000..4d802b7 --- /dev/null +++ b/jobs/crawler.js @@ -0,0 +1,576 @@ +/** + * 评分 / 讨论区爬虫,按时间片推进。 + * 每一片都把游标写进 settings,调用结束即可销毁进程。 + * 整轮间隔仍是 3 小时,由外部调度反复调用,而不是进程内 node-cron。 + */ +const axios = require('axios'); +const cheerio = require('cheerio'); +const http = require('http'); +const https = require('https'); +const { spawn } = require('child_process'); +const prisma = require('../lib/prisma'); +const { loadEffectiveConfig } = require('../utils/siteConfig'); +const { buildBackupArgs, getWikiName } = require('../utils/wikitBackup'); +const { appendJobLogs } = require('../utils/jobLog'); +const { withJobLock } = require('../utils/jobLock'); + +const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); +const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 }); +const request = axios.create({ httpAgent, httpsAgent, timeout: 15000 }); + +const CRAWL_INTERVAL_MS = 3 * 60 * 60 * 1000; +const DEFAULT_BUDGET_MS = 45 * 1000; +const CONCURRENCY = 3; +const CURSOR_KEY = 'crawler:cursor'; +const STATUS_KEY = 'crawler:status'; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const logBuffer = []; + +function logLine(...args) { + const line = args.map(String).join(' '); + console.log(line); + logBuffer.push(`[${new Date().toLocaleString()}] ${line}`); +} + +async function flushLogs() { + if (logBuffer.length === 0) return; + const lines = logBuffer.splice(0, logBuffer.length); + await appendJobLogs('crawler', lines); +} + +function normalize(value) { + if (!value) return null; + if (typeof value === 'string') { + try { return JSON.parse(value); } catch (e) { return null; } + } + return value; +} + +async function readJsonSetting(key) { + const row = await prisma.setting.findUnique({ where: { key } }); + return normalize(row && row.value); +} + +async function writeJsonSetting(key, value) { + await prisma.setting.upsert({ + where: { key }, + update: { value }, + create: { key, value }, + }); +} + +function buildSiteStatus(siteConfig) { + return { + param: siteConfig.PARAM, + name: siteConfig.NAME, + status: 'pending', + pagesFound: 0, + pagesProcessed: 0, + votes: 0, + discussions: 0, + errors: 0, + startedAt: null, + finishedAt: null, + lastRun: null, + error: null, + }; +} + +function freshCursor(sites, previous) { + return { + phase: 'list', + siteIndex: 0, + listPage: 1, + totalPages: 1, + listErrors: 0, + pages: [], + pageOffset: 0, + siteVotes: 0, + siteDiscussions: 0, + siteErrors: 0, + userVotesMap: {}, + nextCycleAt: 0, + status: { + running: true, + startedAt: Date.now(), + finishedAt: null, + currentSite: null, + currentStage: 'list', + overall: { totalSites: sites.length, doneSites: 0 }, + sites: sites.map(buildSiteStatus), + lastRun: previous && previous.status ? previous.status.lastRun : null, + }, + }; +} + +function summary(cursor, extra = {}) { + return { + phase: cursor ? cursor.phase : 'idle', + siteIndex: cursor ? cursor.siteIndex : 0, + pageOffset: cursor ? cursor.pageOffset : 0, + pages: cursor && cursor.pages ? cursor.pages.length : 0, + nextCycleAt: cursor ? cursor.nextCycleAt : null, + ...extra, + }; +} + +let botCookieCache = null; + +async function getBotCookie() { + if (botCookieCache) return botCookieCache; + const user = process.env.WIKIDOT_BOT_USER; + const pass = process.env.WIKIDOT_BOT_PASS; + if (!user || !pass) { + logLine('未配置机器人账号,将以访客身份进行抓取...'); + return null; + } + try { + const payload = new URLSearchParams({ login: user, password: pass, action: 'Login2Action', event: 'login' }); + const res = await axios.post('https://www.wikidot.com/default--flow/login__LoginPopupScreen', payload.toString(), { + headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'WikitDB-Bot/1.0' }, + maxRedirects: 0, + validateStatus: (status) => status >= 200 && status < 400, + }); + let sessionId = ''; + const cookies = res.headers['set-cookie'] || []; + for (const c of cookies) { + if (c.includes('WIKIDOT_SESSION_ID=')) sessionId = c.split('WIKIDOT_SESSION_ID=')[1].split(';')[0]; + } + if (sessionId) { + botCookieCache = `WIKIDOT_SESSION_ID=${sessionId}; wikidot_token7=123456;`; + logLine('机器人账号登录成功,已获取受限站点抓取权限。'); + } + } catch (e) { + console.error('获取 Bot Cookie 失败:', e.message); + } + return botCookieCache; +} + +function wikiHost(siteConfig) { + return siteConfig.URL.replace(/^https?:\/\//i, '').replace(/^www\./i, '').split('.')[0]; +} + +async function fetchListPage(cursor, siteConfig) { + const wikiParam = siteConfig.PARAM; + const actualWikiName = wikiHost(siteConfig); + try { + logLine(`获取 [${wikiParam}] 清单 第 ${cursor.listPage} 页...`); + const res = await request.get(`https://wikit.unitreaty.org/listpages?wiki=${actualWikiName}&p=${cursor.listPage}`); + const lines = String(res.data).split('\n').map((l) => l.trim()).filter(Boolean); + let countThisPage = 0; + lines.forEach((line) => { + if (line.startsWith('Total Pages:')) { + cursor.totalPages = parseInt(line.replace('Total Pages:', '').trim(), 10) || 1; + } else if (line.includes('http') && line.includes('|')) { + const parts = line.split('|').map((item) => item.trim()); + if (parts.length >= 7) { + const url = parts[0]; + const pageSlug = url.split('/').pop(); + let author = parts[6] || '未知'; + const match = author.match(/^(.*?)\s*\(\d+\)$/); + if (match) author = match[1].trim(); + cursor.pages.push({ + page: pageSlug, + title: parts[1], + author, + wiki: wikiParam, + rating: parseInt(parts[3], 10) || 0, + upvotes: parseInt(parts[4], 10) || 0, + downvotes: parseInt(parts[5], 10) || 0, + }); + countThisPage++; + } + } + }); + logLine(`成功 ${countThisPage} 篇`); + cursor.listErrors = 0; + await sleep(400); + if (cursor.listPage >= cursor.totalPages) { + cursor.phase = 'pages'; + cursor.pageOffset = 0; + const siteStatus = cursor.status.sites.find((s) => s.param === wikiParam); + if (siteStatus) { + siteStatus.pagesFound = cursor.pages.length; + cursor.status.currentStage = 'crawl'; + } + } else { + cursor.listPage += 1; + } + } catch (e) { + cursor.listErrors = (cursor.listErrors || 0) + 1; + logLine(`[${wikiParam}] 清单第 ${cursor.listPage} 页失败 (${cursor.listErrors}): ${e.message}`); + if (cursor.listErrors >= 3) { + cursor.listPage += 1; + cursor.listErrors = 0; + if (cursor.listPage > cursor.totalPages) { + cursor.phase = 'pages'; + cursor.pageOffset = 0; + } + } else { + await sleep(1500); + } + } +} + +async function flushVotes(userVotesMap) { + for (const [user, newVotes] of Object.entries(userVotesMap)) { + const key = `user_votes_${user.toLowerCase().replace(/_/g, '-').replace(/ /g, '-')}`; + const record = await prisma.setting.findUnique({ where: { key } }); + const existingMap = new Map(); + if (record) { + const parsed = typeof record.value === 'string' ? JSON.parse(record.value) : record.value; + if (Array.isArray(parsed)) parsed.forEach((v) => existingMap.set(`${v.wiki}:${v.page}`, v)); + } + newVotes.forEach((nv) => { + const id = `${nv.wiki}:${nv.page}`; + if (!existingMap.has(id)) existingMap.set(id, nv); + else if (existingMap.get(id).vote !== nv.vote) { + existingMap.get(id).vote = nv.vote; + existingMap.get(id).date = Date.now(); + } + }); + const truncatedVotes = Array.from(existingMap.values()).sort((a, b) => b.date - a.date).slice(0, 800); + await prisma.setting.upsert({ + where: { key }, + update: { value: truncatedVotes }, + create: { key, value: truncatedVotes }, + }); + } +} + +async function scrapeOne(pageNode, siteConfig, baseHeaders, cursor) { + const baseUrl = siteConfig.URL.replace(/\/$/, ''); + const secureUrl = `${baseUrl}/${pageNode.page}`; + const wikiParam = siteConfig.PARAM; + let success = false; + let attempt = 0; + while (!success && attempt < 3) { + attempt++; + try { + const { data: html } = await request.get(secureUrl, { headers: baseHeaders }); + const $page = cheerio.load(html); + let threadId = null; + let href = $page('#discuss-button').attr('href'); + if (!href) href = $page('#page-info a').filter((_, el) => ($page(el).attr('href') || '').includes('/forum/t-')).attr('href'); + if (!href) { + href = $page('#page-content').parent().find('a').filter((_, el) => { + const text = $page(el).text().toLowerCase(); + return (text.includes('discuss') || text.includes('讨论') || text.includes('评论')) && ($page(el).attr('href') || '').includes('/forum/t-'); + }).attr('href'); + } + if (href) { + const match = href.match(/\/forum\/t-(\d+)/); + if (match) threadId = match[1]; + } + const cacheKey = `forum_v7:${wikiParam}:${pageNode.page}`; + if (threadId) { + cursor.siteDiscussions++; + try { + const forumUrl = `${baseUrl}/forum/t-${threadId}`; + const { data: forumHtml } = await request.get(forumUrl, { headers: baseHeaders }); + const $forum = cheerio.load(forumHtml); + const posts = []; + const userIdCache = {}; + const postElements = $forum('.post').toArray(); + for (const el of postElements) { + const $el = $forum(el); + const postId = ($el.attr('id') || '').replace('post-', ''); + if (!postId) continue; + let parentId = null; + const $parentContainer = $el.parent('.post-container').parent('.post-container'); + if ($parentContainer.length) { + const $parentPost = $parentContainer.children('.post').first(); + parentId = ($parentPost.attr('id') || '').replace('post-', ''); + } + let author = '未知用户'; + const $printUser = $el.find('.head .printuser').length ? $el.find('.head .printuser').first() : $el.find('.info .printuser').first(); + if ($printUser.length) { + const $links = $printUser.find('a'); + author = $links.length ? $links.last().text().trim() : $printUser.text().trim(); + } else { + author = $el.find('.head .author, .info .author').first().text().trim() || '未知用户'; + } + author = author.replace(/[\r\n\t]+/g, '').trim(); + let userid = null; + const headHtml = $el.find('.head').html() || $el.find('.info').html() || $el.html() || ''; + const srcMatch = headHtml.match(/avatar\.php\?userid=(\d+)/i); + const clickMatch = headHtml.match(/userInfo\(\s*(\d+)\s*\)/i); + const karmaMatch = headHtml.match(/userkarma\.php\?u=(\d+)/i); + if (srcMatch) userid = srcMatch[1]; + else if (clickMatch) userid = clickMatch[1]; + else if (karmaMatch) userid = karmaMatch[1]; + if (!userid && author !== '未知用户') { + if (userIdCache[author]) userid = userIdCache[author]; + else { + try { + const lookupRes = await axios.get(`https://www.wikidot.com/quickmodule.php?module=UserLookupQModule&q=${encodeURIComponent(author)}`, { timeout: 5000 }); + if (lookupRes.data && lookupRes.data.users && lookupRes.data.users.length > 0) { + userid = lookupRes.data.users[0].user_id; + userIdCache[author] = userid; + } + } catch (lookupErr) { /* 忽略 */ } + } + } + const currentTs = Math.floor(Date.now() / 1000); + const avatarUrl = userid + ? `https://www.wikidot.com/avatar.php?userid=${userid}×tamp=${currentTs}` + : 'https://www.wikidot.com/avatar.php?account=default'; + const contentHtml = $el.find('.content').html() || ''; + const odate = $el.find('.odate').first(); + const odateClass = odate.attr('class') || ''; + const timeMatch = odateClass.match(/time_(\d+)/); + let timestamp = odate.text().trim(); + if (timeMatch) { + const dateObj = new Date(parseInt(timeMatch[1], 10) * 1000); + timestamp = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')} ${String(dateObj.getHours()).padStart(2, '0')}:${String(dateObj.getMinutes()).padStart(2, '0')}`; + } else if (odate.attr('title')) timestamp = odate.attr('title'); + posts.push({ postId, parentId, author, avatarUrl, timestamp, contentHtml, children: [] }); + } + const postMap = {}; + const rootPosts = []; + posts.forEach((p) => { p.children = []; postMap[p.postId] = p; }); + posts.forEach((p) => { + if (p.parentId && postMap[p.parentId]) postMap[p.parentId].children.push(p); + else rootPosts.push(p); + }); + await prisma.setting.upsert({ + where: { key: cacheKey }, + update: { value: { threadId, url: forumUrl, total: posts.length, threads: rootPosts } }, + create: { key: cacheKey, value: { threadId, url: forumUrl, total: posts.length, threads: rootPosts } }, + }); + logLine(`[成功] 讨论区入库: ${pageNode.page} (${posts.length}条)`); + } catch (forumErr) { + console.error(`[失败] ${pageNode.page} 讨论区抓取报错: ${forumErr.message}`); + } + } else { + await prisma.setting.upsert({ + where: { key: cacheKey }, + update: { value: { threadId: null, url: '', total: 0, threads: [] } }, + create: { key: cacheKey, value: { threadId: null, url: '', total: 0, threads: [] } }, + }); + } + + let pageId = null; + const idMatch = html.match(/pageId\s*[:=]\s*['"]?(\d+)['"]?/i) || html.match(/page_id\s*[:=]\s*['"]?(\d+)['"]?/i); + if (idMatch) pageId = idMatch[1]; + if (!pageId) { success = true; return; } + + const origin = new URL(secureUrl).origin; + const ajaxUrl = `${origin}/ajax-module-connector.php`; + const botCookie = baseHeaders.Cookie; + const { data: rateData } = await request.post( + ajaxUrl, + `pageId=${pageId}&page_id=${pageId}&moduleName=pagerate/WhoRatedPageModule&wikidot_token7=123456`, + { headers: { 'User-Agent': 'Mozilla/5.0', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', Cookie: botCookie || 'wikidot_token7=123456;' } }, + ); + if (rateData.status === 'ok' && rateData.body) { + const $rate = cheerio.load(rateData.body); + $rate('.printuser').each((_, el) => { + const user = $rate(el).text().trim(); + let vote = '+1'; + let textAfter = ''; + let curr = el.next; + while (curr) { + if (curr.type === 'tag' && (curr.tagName === 'br' || (curr.attribs && curr.attribs.class && curr.attribs.class.includes('printuser')))) break; + if (curr.type === 'text') textAfter += curr.data; + else if (curr.type === 'tag') textAfter += $rate(curr).text(); + curr = curr.next; + } + if (textAfter.includes('-')) vote = '-1'; + if (!cursor.userVotesMap[user]) cursor.userVotesMap[user] = []; + cursor.userVotesMap[user].push({ + wiki: wikiParam, page: pageNode.page, title: pageNode.title, vote, author: pageNode.author, date: Date.now(), + }); + cursor.siteVotes++; + }); + } + success = true; + } catch (err) { + cursor.siteErrors++; + if (attempt < 3) await sleep(1000); + } + } +} + +function touchSite(cursor, siteConfig, patch) { + const siteStatus = cursor.status.sites.find((s) => s.param === siteConfig.PARAM); + if (!siteStatus) return; + Object.assign(siteStatus, patch); +} + +async function finishSite(cursor, siteConfig) { + await flushVotes(cursor.userVotesMap || {}); + cursor.userVotesMap = {}; + touchSite(cursor, siteConfig, { + status: 'done', + finishedAt: Date.now(), + lastRun: Date.now(), + pagesProcessed: cursor.pageOffset, + pagesFound: cursor.pages.length, + votes: cursor.siteVotes, + discussions: cursor.siteDiscussions, + errors: cursor.siteErrors, + }); + cursor.status.overall.doneSites = cursor.status.sites.filter((s) => s.status === 'done').length; + cursor.status.currentSite = null; + cursor.status.currentStage = 'done'; + cursor.siteIndex += 1; + cursor.phase = 'list'; + cursor.listPage = 1; + cursor.totalPages = 1; + cursor.listErrors = 0; + cursor.pages = []; + cursor.pageOffset = 0; + cursor.siteVotes = 0; + cursor.siteDiscussions = 0; + cursor.siteErrors = 0; +} + +function finishCycle(cursor) { + cursor.phase = 'idle'; + cursor.pages = []; + cursor.userVotesMap = {}; + cursor.nextCycleAt = Date.now() + CRAWL_INTERVAL_MS; + cursor.status.running = false; + cursor.status.finishedAt = Date.now(); + cursor.status.lastRun = Date.now(); + cursor.status.currentSite = null; + cursor.status.currentStage = ''; + cursor.status.overall.doneSites = cursor.status.sites.filter((s) => s.status === 'done').length; + return cursor; +} + +async function runCrawlSlice(opts = {}) { + const budgetMs = Number(opts.budgetMs) > 0 ? Number(opts.budgetMs) : DEFAULT_BUDGET_MS; + const started = Date.now(); + const over = () => Date.now() - started >= budgetMs; + + return withJobLock('crawl', budgetMs + 30 * 1000, async () => { + const sites = ((await loadEffectiveConfig()).SUPPORT_WIKI) || []; + let cursor = await readJsonSetting(CURSOR_KEY); + if (!cursor || cursor.phase === 'idle' || !cursor.status) { + if (cursor && cursor.nextCycleAt && Date.now() < cursor.nextCycleAt && !opts.force) { + return summary(cursor, { skipped: true, reason: 'cooldown' }); + } + logLine(`\n[${new Date().toLocaleString()}] 开始执行全站数据(评分+讨论区)爬取...`); + cursor = freshCursor(sites, cursor); + } + + const botCookie = await getBotCookie(); + const baseHeaders = { 'User-Agent': 'Mozilla/5.0' }; + if (botCookie) baseHeaders.Cookie = botCookie; + + try { + while (!over()) { + if (cursor.siteIndex >= sites.length) { + cursor = finishCycle(cursor); + await writeJsonSetting(CURSOR_KEY, cursor); + await writeJsonSetting(STATUS_KEY, cursor.status); + logLine('本轮爬取结束。'); + await flushLogs(); + return summary(cursor, { done: true }); + } + + const siteConfig = sites[cursor.siteIndex]; + cursor.status.currentSite = siteConfig.PARAM; + const siteStatus = cursor.status.sites.find((s) => s.param === siteConfig.PARAM); + if (siteStatus && siteStatus.status === 'pending') { + siteStatus.status = 'running'; + siteStatus.startedAt = Date.now(); + siteStatus.error = null; + } + + if (cursor.phase === 'list') { + cursor.status.currentStage = 'list'; + await fetchListPage(cursor, siteConfig); + } else if (cursor.pageOffset >= cursor.pages.length) { + await finishSite(cursor, siteConfig); + } else { + cursor.status.currentStage = 'crawl'; + const batch = cursor.pages.slice(cursor.pageOffset, cursor.pageOffset + CONCURRENCY); + await Promise.all(batch.map((pageNode) => scrapeOne(pageNode, siteConfig, baseHeaders, cursor))); + cursor.pageOffset += batch.length; + touchSite(cursor, siteConfig, { + pagesProcessed: cursor.pageOffset, + pagesFound: cursor.pages.length, + votes: cursor.siteVotes, + discussions: cursor.siteDiscussions, + errors: cursor.siteErrors, + }); + if (cursor.pageOffset % 100 < CONCURRENCY || cursor.pageOffset >= cursor.pages.length) { + await flushVotes(cursor.userVotesMap || {}); + cursor.userVotesMap = {}; + } + logLine(`--- 当前进度: [${cursor.pageOffset}/${cursor.pages.length}] ${siteConfig.PARAM} ---`); + await sleep(2500); + } + + await writeJsonSetting(CURSOR_KEY, cursor); + await writeJsonSetting(STATUS_KEY, cursor.status); + } + } catch (e) { + logLine(`发生异常: ${e.message}`); + cursor.status.sites.forEach((s) => { + if (s.status === 'running') { + s.status = 'error'; + s.error = e.message; + } + }); + cursor.status.running = false; + await writeJsonSetting(CURSOR_KEY, cursor); + await writeJsonSetting(STATUS_KEY, cursor.status); + await flushLogs(); + return summary(cursor, { done: false, error: e.message }); + } + + await flushLogs(); + return summary(cursor, { done: false }); + }); +} + +function runBackup() { + if (process.env.VERCEL) { + return Promise.resolve({ + skipped: true, + reason: 'wikit CLI 需要本地二进制和磁盘,不能在无服务器函数里跑。请用 GitHub Actions 的备份工作流。', + }); + } + return withJobLock('backup', 6 * 60 * 60 * 1000, async () => { + const config = await loadEffectiveConfig(); + const wikiNames = (config.SUPPORT_WIKI || []).map(getWikiName).filter(Boolean); + if (wikiNames.length === 0) { + logLine(`[${new Date().toLocaleString()}] 没有可备份的站点。`); + await flushLogs(); + return { skipped: true, reason: 'no-wikis' }; + } + const args = buildBackupArgs({ wikiNames, keepRemoved: true, baseDir: process.env.WIKIT_BASE_DIR || undefined }); + logLine(`[${new Date().toLocaleString()}] 开始 wikit 备份,共 ${wikiNames.length} 个站点。`); + const code = await new Promise((resolve) => { + const child = spawn('wikit', args, { + cwd: process.cwd(), + env: process.env, + shell: false, + windowsHide: true, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout.on('data', (chunk) => logLine(`[wikit] ${chunk}`)); + child.stderr.on('data', (chunk) => logLine(`[wikit] ${chunk}`)); + child.on('error', (error) => { + logLine(`无法启动 wikit: ${error.message}`); + resolve(127); + }); + child.on('close', (exitCode) => resolve(exitCode)); + }); + logLine(`Wikit 备份结束,退出码 ${code}。`); + await flushLogs(); + return { done: true, exitCode: code }; + }); +} + +module.exports = { + runCrawlSlice, + runBackup, + CRAWL_INTERVAL_MS, + DEFAULT_BUDGET_MS, +}; diff --git a/lib/prisma.js b/lib/prisma.js index adcaac4..c248ee3 100644 --- a/lib/prisma.js +++ b/lib/prisma.js @@ -1,30 +1,42 @@ const { PrismaClient } = require('@prisma/client'); -let prismaBase; +/** + * 无服务器温启动会复用同一个实例。 + * 客户端必须挂在 globalThis 上,否则每次冷启动路径重复 new 会打满数据库连接。 + * 在 Vercel 上如果连接串没写 connection_limit,默认补 1,避免一个函数占一整池连接。 + */ +function serverlessDatabaseUrl(url) { + if (!url || !process.env.VERCEL) return url; + if (/[?&]connection_limit=/.test(url)) return url; + const join = url.includes('?') ? '&' : '?'; + return `${url}${join}connection_limit=1`; +} -if (process.env.NODE_ENV === 'production') { - prismaBase = new PrismaClient(); -} else { - if (!global.__prisma) { - global.__prisma = new PrismaClient(); - } - prismaBase = global.__prisma; +function createClient() { + const url = serverlessDatabaseUrl(process.env.DATABASE_URL); + const options = url && url !== process.env.DATABASE_URL + ? { datasources: { db: { url } } } + : undefined; + return new PrismaClient(options); } +const prismaBase = global.__prisma || createClient(); +global.__prisma = prismaBase; + const prisma = prismaBase.$extends({ query: { setting: { async findUnique({ args, query }) { const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, async findFirst({ args, query }) { const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, @@ -32,7 +44,7 @@ const prisma = prismaBase.$extends({ const results = await query(args); results.forEach(r => { if (r && r.value) { - try { r.value = JSON.parse(r.value); } catch {} + try { r.value = JSON.parse(r.value); } catch { /* 保持原字符串 */ } } }); return results; @@ -43,7 +55,7 @@ const prisma = prismaBase.$extends({ } const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, @@ -53,7 +65,7 @@ const prisma = prismaBase.$extends({ } const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, @@ -66,7 +78,7 @@ const prisma = prismaBase.$extends({ } const result = await query(args); if (result && result.value) { - try { result.value = JSON.parse(result.value); } catch {} + try { result.value = JSON.parse(result.value); } catch { /* 保持原字符串 */ } } return result; }, diff --git a/middleware.js b/middleware.js index 83c43a1..2b6a779 100644 --- a/middleware.js +++ b/middleware.js @@ -1,48 +1,69 @@ import { NextResponse } from 'next/server'; +import { Redis } from '@upstash/redis'; -const ipRequestCounts = new Map(); const WINDOW_MS = 60 * 1000; const MAX_REQUESTS_PER_MINUTE = 120; -function cleanupStore() { +// 没配 Upstash 时只在当前边缘实例里计数,跨实例不共享。 +const ipRequestCounts = new Map(); + +const redis = process.env.UPSTASH_REDIS_REST_URL && process.env.UPSTASH_REDIS_REST_TOKEN + ? new Redis({ + url: process.env.UPSTASH_REDIS_REST_URL, + token: process.env.UPSTASH_REDIS_REST_TOKEN, + }) + : null; + +function clientIp(request) { + const trustProxy = process.env.TRUST_PROXY === 'true'; + return (trustProxy && request.headers.get('x-forwarded-for')?.split(',')[0]?.trim()) + || (trustProxy && request.headers.get('x-real-ip')) + || request.ip + || 'unknown'; +} + +function tooMany() { + return NextResponse.json( + { error: '请求过于频繁,请稍后再试' }, + { status: 429 } + ); +} + +function memoryLimit(ip) { const now = Date.now(); + const record = ipRequestCounts.get(ip); + if (!record || now - record.windowStart > WINDOW_MS) { + ipRequestCounts.set(ip, { count: 1, windowStart: now }); + return false; + } + record.count += 1; if (ipRequestCounts.size > 5000) { for (const [key, data] of ipRequestCounts) { - if (now - data.windowStart > WINDOW_MS) { - ipRequestCounts.delete(key); - } + if (now - data.windowStart > WINDOW_MS) ipRequestCounts.delete(key); } } + return record.count > MAX_REQUESTS_PER_MINUTE; } -export function middleware(request) { - const trustProxy = process.env.TRUST_PROXY === 'true'; - // IP 解析优先级:反代头(需 TRUST_PROXY)→ Next.js request.ip → 兜底 'unknown' - // 修复:直连场景下避免所有请求共享 'unknown' 桶导致单点 DoS - const ip = (trustProxy && request.headers.get('x-forwarded-for')?.split(',')[0]?.trim()) - || (trustProxy && request.headers.get('x-real-ip')) - || request.ip - || 'unknown'; +export async function middleware(request) { + if (!request.nextUrl.pathname.startsWith('/api/')) return NextResponse.next(); - if (request.nextUrl.pathname.startsWith('/api/')) { - const now = Date.now(); - const record = ipRequestCounts.get(ip); - - if (!record || now - record.windowStart > WINDOW_MS) { - ipRequestCounts.set(ip, { count: 1, windowStart: now }); - } else { - record.count++; - if (record.count > MAX_REQUESTS_PER_MINUTE) { - return NextResponse.json( - { error: '请求过于频繁,请稍后再试' }, - { status: 429 } - ); - } - } + const ip = clientIp(request); - cleanupStore(); + if (redis) { + try { + const bucket = Math.floor(Date.now() / WINDOW_MS); + const key = `rl:${ip}:${bucket}`; + const count = await redis.incr(key); + if (count === 1) await redis.expire(key, 70); + if (count > MAX_REQUESTS_PER_MINUTE) return tooMany(); + return NextResponse.next(); + } catch (e) { + // Redis 故障时退回本实例计数,不把整个 API 打挂。 + } } + if (memoryLimit(ip)) return tooMany(); return NextResponse.next(); } diff --git a/next.config.js b/next.config.js index 3f3336d..ba87564 100644 --- a/next.config.js +++ b/next.config.js @@ -9,6 +9,9 @@ const ALLOWED_FRAME_DOMAINS = ['https://timer.backroomswiki.cn']; const isDev = process.env.NODE_ENV !== 'production'; const nextConfig = { + outputFileTracingIncludes: { + '/*': ['./wikitdb.config.js', './content/**/*'], + }, async rewrites() { return [ // Honeypot traps for common attack paths diff --git a/package-lock.json b/package-lock.json index 2c24375..3186b88 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,6 @@ "jsonwebtoken": "^9.0.2", "lightweight-charts": "^4.1.3", "next": "15.5.21", - "node-cron": "^4.2.1", "nodemailer": "^9.0.3", "nvm": "^0.0.4", "postcss": "^8.5.23", @@ -5640,15 +5639,6 @@ } } }, - "node_modules/node-cron": { - "version": "4.2.1", - "resolved": "https://registry.npmmirror.com/node-cron/-/node-cron-4.2.1.tgz", - "integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==", - "license": "ISC", - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/node-exports-info": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.2.tgz", diff --git a/package.json b/package.json index e271e10..6b576bf 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,9 @@ "test:ftml": "node scripts/test-ftml-runtime.mjs", "start": "next start", "worker": "node auto-crawler.js", + "job:crawl": "node auto-crawler.js", + "job:attribution": "node auto-attribution.js", + "job:backup": "node auto-crawler.js --backup", "lint": "next lint --quiet", "test": "node --test tests/*.test.js", "prisma:generate": "prisma generate", @@ -29,7 +32,6 @@ "jsonwebtoken": "^9.0.2", "lightweight-charts": "^4.1.3", "next": "15.5.21", - "node-cron": "^4.2.1", "nodemailer": "^9.0.3", "nvm": "^0.0.4", "postcss": "^8.5.23", diff --git a/pages/admin.js b/pages/admin.js index a43f778..765d9ea 100644 --- a/pages/admin.js +++ b/pages/admin.js @@ -201,7 +201,7 @@ export default function AdminDashboard() { body: JSON.stringify(isEdit ? { param: editingSiteParam, ...siteForm } : siteForm) }); setSites(d.sites || []); - setSiteMsg({ type: 'ok', text: isEdit ? (d.message || '站点已更新') : '站点已添加,配置已写入 wikitdb.config.js' }); + setSiteMsg({ type: 'ok', text: isEdit ? (d.message || '站点已更新') : '站点已写入数据库(磁盘可写时会同时刷新配置文件)' }); setEditingSiteParam(null); setSiteForm({ NAME: '', URL: '', PARAM: '', WIKIT_ID: '', ImgURL: '', GQL_API: '', AUTHOR_TAG: '', ATTRIBUTION_PAGE: '', FORUM_SYNC: false }); } catch (e) { @@ -235,7 +235,7 @@ export default function AdminDashboard() { }; const handleDeleteSite = async (param) => { - if (!confirm(`确定删除站点 "${param}"?此操作会立即修改 wikitdb.config.js。`)) return; + if (!confirm(`确定删除站点 "${param}"?此操作会写入数据库里的站点配置。`)) return; try { const d = await api('/api/admin/sites', { method: 'DELETE', body: JSON.stringify({ param }) }); setSites(d.sites || []); @@ -440,7 +440,7 @@ export default function AdminDashboard() { {activeTab === 'sites' && (
-
收录站点管理(写入 wikitdb.config.js)
+
收录站点管理(写入数据库)
@@ -514,7 +514,7 @@ export default function AdminDashboard() { {editingSiteParam && }
-
提示:站点变更会立即写入 wikitdb.config.js。已构建的静态页面与正在运行的爬虫进程(npm run worker)需重启后才会使用新的站点列表。
+
提示:站点变更写入数据库,爬虫下一片就会读到。已经打包进页面的导航列表要重新部署后才更新。无服务器环境写不了 wikitdb.config.js,这是预期行为。
)} @@ -523,7 +523,7 @@ export default function AdminDashboard() { {activeTab === 'crawler' && (
-
爬虫运行状态(每 3 小时自动执行)
+
爬虫运行状态(无服务器切片,每轮间隔 3 小时)
{crawlerLoading && 加载中...} @@ -806,11 +806,11 @@ export default function AdminDashboard() { {activeTab === 'logs' && (
-
服务器 / 爬虫文件日志
+
爬虫日志(数据库)
diff --git a/pages/api/admin/crawler-status.js b/pages/api/admin/crawler-status.js index ba4c40a..87f0e62 100644 --- a/pages/api/admin/crawler-status.js +++ b/pages/api/admin/crawler-status.js @@ -1,12 +1,10 @@ import prisma from '../../../lib/prisma'; import { withAdmin } from '../../../utils/withAdmin'; -import { loadSiteConfig } from '../../../utils/siteConfig'; +import { loadEffectiveConfig } from '../../../utils/siteConfig'; /** * 爬虫状态 API * GET /api/admin/crawler-status - * 返回:爬虫持久化的结构化状态(crawler:status), - * 并与当前配置中的站点列表合并,保证新增/删除站点后状态表始终完整。 */ async function handler(req, res) { if (req.method !== 'GET') return res.status(405).json({ error: 'Method not allowed' }); @@ -20,7 +18,7 @@ async function handler(req, res) { let configuredSites = []; try { - configuredSites = (loadSiteConfig().SUPPORT_WIKI) || []; + configuredSites = ((await loadEffectiveConfig()).SUPPORT_WIKI) || []; } catch (e) { configuredSites = []; } @@ -46,7 +44,6 @@ async function handler(req, res) { }; }); - // 用合并后的站点列表修正总进度统计,保证新增站点也能反映到进度里 if (status && typeof status === 'object' && !status.running) { status = { ...status, diff --git a/pages/api/admin/file-logs.js b/pages/api/admin/file-logs.js index 997c80f..dfe7c9c 100644 --- a/pages/api/admin/file-logs.js +++ b/pages/api/admin/file-logs.js @@ -1,47 +1,68 @@ import fs from 'fs'; import path from 'path'; import { withAdmin } from '../../../utils/withAdmin'; +import { readJobLogs } from '../../../utils/jobLog'; /** - * 服务器 / 爬虫文件日志查看 API - * GET /api/admin/file-logs?file=crawler|server|serverErr&lines=500 - * 返回指定日志文件末尾若干行。 + * 日志查看。 + * 爬虫日志在无服务器部署里来自数据库(logs:crawler)。 + * server.log / server-err.log 只在自建虚拟机上存在,读不到就返回说明。 */ -const LOG_FILES = { - crawler: 'crawler.log', +const FILE_LOGS = { server: 'server.log', serverErr: 'server-err.log', }; +async function readDisk(fileName, linesLimit) { + const filePath = path.join(process.cwd(), fileName); + if (!fs.existsSync(filePath)) return null; + const content = fs.readFileSync(filePath, 'utf8'); + const stat = fs.statSync(filePath); + const allLines = content.split(/\r?\n/).map(l => l.replace(/\r$/, '')).filter(l => l.trim().length > 0); + return { + lines: allLines.slice(-linesLimit), + totalLines: allLines.length, + size: stat.size, + mtime: stat.mtime, + }; +} + async function handler(req, res) { if (req.method !== 'GET') return res.status(405).json({ error: 'Method not allowed' }); const fileKey = String(req.query.file || 'crawler'); - const linesLimit = Math.min(parseInt(req.query.lines) || 500, 5000); - const fileName = LOG_FILES[fileKey]; - if (!fileName) return res.status(400).json({ error: '未知日志文件' }); - - const filePath = path.join(process.cwd(), fileName); + const linesLimit = Math.min(parseInt(req.query.lines, 10) || 500, 5000); try { - let content = ''; - let stat = null; - if (fs.existsSync(filePath)) { - content = fs.readFileSync(filePath, 'utf8'); - stat = fs.statSync(filePath); + if (fileKey === 'crawler') { + const stored = await readJobLogs('crawler', linesLimit); + return res.status(200).json({ + file: fileKey, + name: 'logs:crawler', + source: 'database', + lines: stored.lines, + totalLines: stored.totalLines, + size: stored.lines.join('\n').length, + mtime: stored.lines.length ? new Date().toISOString() : null, + }); } - const allLines = content.split(/\r?\n/); - const lines = allLines.map(l => l.replace(/\r$/, '')).filter(l => l.trim().length > 0).slice(-linesLimit); - - return res.status(200).json({ - file: fileKey, - name: fileName, - lines, - totalLines: allLines.length, - size: stat ? stat.size : 0, - mtime: stat ? stat.mtime : null, - }); + const fileName = FILE_LOGS[fileKey]; + if (!fileName) return res.status(400).json({ error: '未知日志文件' }); + const disk = await readDisk(fileName, linesLimit); + if (!disk) { + return res.status(200).json({ + file: fileKey, + name: fileName, + source: 'missing', + lines: [], + totalLines: 0, + size: 0, + mtime: null, + note: '无服务器部署没有常驻进程日志文件。爬虫输出在「crawler」里,来自数据库。', + }); + } + return res.status(200).json({ file: fileKey, name: fileName, source: 'file', ...disk }); } catch (error) { console.error('[file-logs] 读取失败:', error); return res.status(500).json({ error: `读取日志失败: ${error.message}` }); diff --git a/pages/api/admin/sites.js b/pages/api/admin/sites.js index cfa1815..f82250d 100644 --- a/pages/api/admin/sites.js +++ b/pages/api/admin/sites.js @@ -1,25 +1,27 @@ import { withAdmin } from '../../../utils/withAdmin'; -import { loadSiteConfig, saveSiteConfig, validateSite } from '../../../utils/siteConfig'; +import { loadEffectiveConfig, saveSiteConfig, validateSite } from '../../../utils/siteConfig'; /** * 站点管理 API - * GET /api/admin/sites -> 获取站点列表(读磁盘最新配置) + * GET /api/admin/sites -> 有效站点列表(数据库覆盖优先) * POST /api/admin/sites -> 添加站点 * PUT /api/admin/sites -> 编辑站点(body 传 { param: 原PARAM, ...站点字段 }) * DELETE /api/admin/sites -> 删除站点(body 或 query 传 { param }) + * + * 写入 settings.site:config。磁盘可写时同步刷新 wikitdb.config.js。 */ async function handler(req, res) { try { if (req.method === 'GET') { - const config = loadSiteConfig(); - return res.status(200).json({ sites: config.SUPPORT_WIKI }); + const config = await loadEffectiveConfig(); + return res.status(200).json({ sites: config.SUPPORT_WIKI, source: 'db-or-file' }); } if (req.method === 'POST') { const result = validateSite(req.body); if (result.error) return res.status(400).json({ error: result.error }); - const config = loadSiteConfig(); + const config = await loadEffectiveConfig(); if (config.SUPPORT_WIKI.some(s => s.PARAM === result.site.PARAM)) { return res.status(400).json({ error: `PARAM "${result.site.PARAM}" 已被使用` }); @@ -29,7 +31,7 @@ async function handler(req, res) { } config.SUPPORT_WIKI.push(result.site); - saveSiteConfig(config); + await saveSiteConfig(config); return res.status(200).json({ success: true, sites: config.SUPPORT_WIKI }); } @@ -40,11 +42,10 @@ async function handler(req, res) { const result = validateSite(req.body); if (result.error) return res.status(400).json({ error: result.error }); - const config = loadSiteConfig(); + const config = await loadEffectiveConfig(); const idx = config.SUPPORT_WIKI.findIndex(s => s.PARAM === param); if (idx === -1) return res.status(404).json({ error: '未找到该站点' }); - // 唯一性检查(排除自身) if (config.SUPPORT_WIKI.some((s, i) => i !== idx && s.PARAM === result.site.PARAM)) { return res.status(400).json({ error: `PARAM "${result.site.PARAM}" 已被其他站点使用` }); } @@ -53,14 +54,14 @@ async function handler(req, res) { } config.SUPPORT_WIKI[idx] = result.site; - saveSiteConfig(config); + await saveSiteConfig(config); return res.status(200).json({ success: true, message: `站点 "${result.site.NAME}" 已更新`, sites: config.SUPPORT_WIKI }); } if (req.method === 'DELETE') { const param = String((req.body && req.body.param) || req.query.param || '').trim(); - const config = loadSiteConfig(); + const config = await loadEffectiveConfig(); if (config.SUPPORT_WIKI.length <= 1) { return res.status(400).json({ error: '至少需要保留一个站点,不能删除全部站点' }); } @@ -69,14 +70,14 @@ async function handler(req, res) { if (idx === -1) return res.status(404).json({ error: '未找到该站点' }); config.SUPPORT_WIKI.splice(idx, 1); - saveSiteConfig(config); + await saveSiteConfig(config); return res.status(200).json({ success: true, sites: config.SUPPORT_WIKI }); } return res.status(405).json({ error: 'Method not allowed' }); } catch (error) { console.error('[sites] 处理失败:', error); - return res.status(500).json({ error: '站点管理失败,请检查 wikitdb.config.js 是否可写' }); + return res.status(500).json({ error: '站点管理失败。无服务器部署会把站点写进数据库,请确认 DATABASE_URL 可用。' }); } } diff --git a/pages/api/cron/attribution.js b/pages/api/cron/attribution.js new file mode 100644 index 0000000..8efb0d7 --- /dev/null +++ b/pages/api/cron/attribution.js @@ -0,0 +1,21 @@ +import { authorizeCron } from '../../../utils/cronAuth'; + +export const config = { maxDuration: 60 }; + +/** GET/POST /api/cron/attribution */ +export default async function handler(req, res) { + if (req.method !== 'GET' && req.method !== 'POST') { + return res.status(405).json({ error: 'Method not allowed' }); + } + if (!authorizeCron(req)) return res.status(401).json({ error: '未授权的调度请求' }); + + const force = req.query.force === '1' || req.query.force === 'true'; + try { + const { runAttributionSlice } = require('../../../jobs/attribution'); + const result = await runAttributionSlice({ force }); + return res.status(200).json({ ok: true, result }); + } catch (error) { + console.error('[cron/attribution]', error); + return res.status(500).json({ error: error.message || '归属同步失败' }); + } +} diff --git a/pages/api/cron/crawl.js b/pages/api/cron/crawl.js new file mode 100644 index 0000000..b472cf4 --- /dev/null +++ b/pages/api/cron/crawl.js @@ -0,0 +1,29 @@ +import { authorizeCron } from '../../../utils/cronAuth'; + +export const config = { maxDuration: 60 }; + +/** + * GET/POST /api/cron/crawl + * 无服务器调度入口。Vercel Cron 或 GitHub Actions 带 CRON_SECRET 调用。 + * 每次只推进一个时间片,游标在数据库里。 + */ +export default async function handler(req, res) { + if (req.method !== 'GET' && req.method !== 'POST') { + return res.status(405).json({ error: 'Method not allowed' }); + } + if (!authorizeCron(req)) return res.status(401).json({ error: '未授权的调度请求' }); + + const budget = Number(req.query.budget); + const force = req.query.force === '1' || req.query.force === 'true'; + try { + const { runCrawlSlice } = require('../../../jobs/crawler'); + const result = await runCrawlSlice({ + force, + budgetMs: Number.isFinite(budget) && budget > 0 ? Math.min(budget * 1000, 50 * 1000) : undefined, + }); + return res.status(200).json({ ok: true, result }); + } catch (error) { + console.error('[cron/crawl]', error); + return res.status(500).json({ error: error.message || '爬虫切片失败' }); + } +} diff --git a/pages/api/tools/delete-bot.js b/pages/api/tools/delete-bot.js index 521ceb1..2c8983d 100644 --- a/pages/api/tools/delete-bot.js +++ b/pages/api/tools/delete-bot.js @@ -70,6 +70,12 @@ function readLatestLog(lines = 40) { } async function handler(req, res) { + if (process.env.VERCEL) { + return res.status(501).json({ + success: false, + error: '宿主机删帖机器人依赖 systemd 和本地磁盘,无服务器部署请改用职员面板里的 BotAccount 自动扫描。', + }); + } // GET:状态 + 配置 + 日志 if (req.method === 'GET') { const active = run(`systemctl is-active ${SERVICE}`).toLowerCase() === 'active'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 049eeb0..cc8981c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,5 +1,6 @@ generator client { - provider = "prisma-client-js" + provider = "prisma-client-js" + binaryTargets = ["native", "rhel-openssl-3.0.x"] } datasource db { diff --git a/tests/cronAuth.test.js b/tests/cronAuth.test.js new file mode 100644 index 0000000..809a1a0 --- /dev/null +++ b/tests/cronAuth.test.js @@ -0,0 +1,42 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { authorizeCron } = require('../utils/cronAuth'); +const { normalizeLines } = require('../utils/jobLog'); + +function withEnv(patch, fn) { + const previous = {}; + for (const key of Object.keys(patch)) { + previous[key] = process.env[key]; + if (patch[key] == null) delete process.env[key]; + else process.env[key] = patch[key]; + } + try { + return fn(); + } finally { + for (const key of Object.keys(patch)) { + if (previous[key] == null) delete process.env[key]; + else process.env[key] = previous[key]; + } + } +} + +test('cron auth fails closed without a secret', () => { + withEnv({ CRON_SECRET: null, ALLOW_INSECURE_CRON: null, NODE_ENV: 'production', VERCEL: '1' }, () => { + assert.equal(authorizeCron({ headers: { authorization: 'Bearer whatever-token' } }), false); + }); +}); + +test('cron auth accepts the bearer secret and rejects a mismatch', () => { + withEnv({ CRON_SECRET: 'cron-secret-value-32chars!!', ALLOW_INSECURE_CRON: null, NODE_ENV: 'production', VERCEL: '1' }, () => { + assert.equal(authorizeCron({ headers: { authorization: 'Bearer cron-secret-value-32chars!!' } }), true); + assert.equal(authorizeCron({ headers: { 'x-cron-secret': 'cron-secret-value-32chars!!' } }), true); + assert.equal(authorizeCron({ headers: { authorization: 'Bearer nope' } }), false); + assert.equal(authorizeCron({ headers: {} }), false); + }); +}); + +test('job log normalizes stored lines', () => { + assert.deepEqual(normalizeLines({ lines: ['a', 'b'] }), ['a', 'b']); + assert.deepEqual(normalizeLines('["a"]'), ['a']); + assert.deepEqual(normalizeLines(null), []); +}); diff --git a/utils/cache.js b/utils/cache.js index c8a3e08..541ecf5 100644 --- a/utils/cache.js +++ b/utils/cache.js @@ -1,34 +1,14 @@ /** - * TTL 内存缓存 - * 支持不同过期时间,自动清理过期条目 + * 单实例 TTL 缓存。 + * 无服务器函数不能靠 setInterval 打扫(会拖住实例),读取时顺手丢掉过期项。 + * 命中范围只在当前温实例内;跨实例共享的数据放数据库或 Upstash。 */ const store = new Map(); -const DEFAULT_TTL = 5 * 60 * 1000; // 5 分钟 +const DEFAULT_TTL = 5 * 60 * 1000; const MAX_ENTRIES = 500; -let cleanupTimer = null; -function startCleanup() { - if (cleanupTimer) return; - cleanupTimer = setInterval(() => { - const now = Date.now(); - for (const [key, entry] of store) { - if (now > entry.expiresAt) store.delete(key); - } - if (store.size === 0) { - clearInterval(cleanupTimer); - cleanupTimer = null; - } - }, 60 * 1000); - if (cleanupTimer.unref) cleanupTimer.unref(); -} - -/** - * 获取缓存值 - * @param {string} key - * @returns {*} 缓存值或 undefined - */ function get(key) { const entry = store.get(key); if (!entry) return undefined; @@ -40,15 +20,10 @@ function get(key) { return entry.value; } -/** - * 设置缓存 - * @param {string} key - * @param {*} value - * @param {number} ttl 毫秒,默认 5 分钟 - */ function set(key, value, ttl = DEFAULT_TTL) { if (store.size >= MAX_ENTRIES) { - let oldest = null, oldestKey = null; + let oldest = null; + let oldestKey = null; for (const [k, v] of store) { if (!oldest || v.lastAccess < oldest.lastAccess) { oldest = v; @@ -58,29 +33,16 @@ function set(key, value, ttl = DEFAULT_TTL) { if (oldestKey) store.delete(oldestKey); } store.set(key, { value, expiresAt: Date.now() + ttl, lastAccess: Date.now() }); - startCleanup(); } -/** - * 删除缓存 - */ function del(key) { store.delete(key); } -/** - * 清空所有缓存 - */ function flush() { store.clear(); } -/** - * 带缓存的异步函数包装器 - * @param {string} key 缓存键 - * @param {Function} fn 异步函数 - * @param {number} ttl 缓存时间(毫秒) - */ async function cached(key, fn, ttl = DEFAULT_TTL) { const hit = get(key); if (hit !== undefined) return hit; diff --git a/utils/cronAuth.js b/utils/cronAuth.js new file mode 100644 index 0000000..bc01234 --- /dev/null +++ b/utils/cronAuth.js @@ -0,0 +1,38 @@ +/** + * 定时任务入口鉴权。 + * Vercel Cron 会带 Authorization: Bearer $CRON_SECRET。 + * 未配置 CRON_SECRET 时拒绝(避免把爬虫端点暴露到公网)。 + * 本地手动调试可设 ALLOW_INSECURE_CRON=1。 + */ +const crypto = require('crypto'); + +function header(req, name) { + const headers = (req && req.headers) || {}; + const direct = headers[name] ?? headers[name.toLowerCase()]; + if (Array.isArray(direct)) return String(direct[0] || ''); + return direct ? String(direct) : ''; +} + +function tokenFromRequest(req) { + const auth = header(req, 'authorization'); + if (/^Bearer\s+/i.test(auth)) return auth.replace(/^Bearer\s+/i, '').trim(); + return header(req, 'x-cron-secret').trim(); +} + +function safeEqual(a, b) { + const left = Buffer.from(String(a)); + const right = Buffer.from(String(b)); + if (left.length !== right.length) return false; + return crypto.timingSafeEqual(left, right); +} + +function authorizeCron(req) { + if (process.env.ALLOW_INSECURE_CRON === '1' && process.env.NODE_ENV !== 'production' && !process.env.VERCEL) { + return true; + } + const secret = process.env.CRON_SECRET || ''; + if (!secret || secret.length < 16) return false; + return safeEqual(tokenFromRequest(req), secret); +} + +module.exports = { authorizeCron, tokenFromRequest }; diff --git a/utils/jobLock.js b/utils/jobLock.js new file mode 100644 index 0000000..b52e643 --- /dev/null +++ b/utils/jobLock.js @@ -0,0 +1,53 @@ +/** + * 用 settings 行做互斥锁。 + * 无服务器平台会并发触发同一个 cron,锁保证同一任务只有一个切片在跑。 + */ +function normalize(value) { + if (!value) return null; + if (typeof value === 'string') { + try { return JSON.parse(value); } catch (e) { return null; } + } + return value; +} + +async function withJobLock(name, ttlMs, fn) { + const prisma = require('../lib/prisma'); + const key = `lock:${name}`; + const owner = `${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`; + const until = Date.now() + ttlMs; + + let state; + try { + state = await prisma.$transaction(async (tx) => { + const row = await tx.setting.findUnique({ where: { key } }); + const cur = normalize(row && row.value); + if (cur && Number(cur.until) > Date.now()) return 'locked'; + const value = { owner, until }; + await tx.setting.upsert({ + where: { key }, + update: { value }, + create: { key, value }, + }); + return 'ok'; + }, { isolationLevel: 'Serializable' }); + } catch (e) { + if (e.code === 'P2034') return { skipped: true, reason: 'locked' }; + throw e; + } + + if (state !== 'ok') return { skipped: true, reason: 'locked' }; + + try { + return await fn(); + } finally { + try { + const row = await prisma.setting.findUnique({ where: { key } }); + const cur = normalize(row && row.value); + if (cur && cur.owner === owner) { + await prisma.setting.delete({ where: { key } }); + } + } catch (e) { /* 锁过期由下一轮覆盖 */ } + } +} + +module.exports = { withJobLock }; diff --git a/utils/jobLog.js b/utils/jobLog.js new file mode 100644 index 0000000..a5339ff --- /dev/null +++ b/utils/jobLog.js @@ -0,0 +1,52 @@ +/** + * 任务日志写入 settings 表(logs:),替代 crawler.log 这类本地文件。 + * 无服务器实例的磁盘在调用结束后就没了,管理后台仍从数据库读最近若干行。 + */ +const MAX_LINES = 800; + +function normalizeLines(value) { + if (!value) return []; + const parsed = typeof value === 'string' ? safeJson(value) : value; + if (Array.isArray(parsed)) return parsed.map(String); + if (parsed && Array.isArray(parsed.lines)) return parsed.lines.map(String); + return []; +} + +function safeJson(text) { + try { return JSON.parse(text); } catch (e) { return null; } +} + +async function appendJobLogs(source, lines) { + const incoming = (Array.isArray(lines) ? lines : [lines]).map(String).filter(Boolean); + if (incoming.length === 0) return; + let prisma; + try { + prisma = require('../lib/prisma'); + } catch (e) { + return; + } + const key = `logs:${source}`; + try { + const row = await prisma.setting.findUnique({ where: { key } }); + const prev = normalizeLines(row && row.value); + const next = prev.concat(incoming).slice(-MAX_LINES); + await prisma.setting.upsert({ + where: { key }, + update: { value: { lines: next } }, + create: { key, value: { lines: next } }, + }); + } catch (e) { + console.error(`[job-log] ${source} 写入失败: ${e.message}`); + } +} + +async function readJobLogs(source, limit = 500) { + const prisma = require('../lib/prisma'); + const key = `logs:${source}`; + const row = await prisma.setting.findUnique({ where: { key } }); + const lines = normalizeLines(row && row.value); + const sliced = lines.slice(-Math.max(1, limit)); + return { lines: sliced, totalLines: lines.length }; +} + +module.exports = { appendJobLogs, readJobLogs, normalizeLines, MAX_LINES }; diff --git a/utils/siteConfig.js b/utils/siteConfig.js index 9baa4ac..2b0589b 100644 --- a/utils/siteConfig.js +++ b/utils/siteConfig.js @@ -1,7 +1,8 @@ /** - * wikitdb.config.js 运行时读写工具 - * 使用 vm 沙箱解析配置文件,避免 Next.js webpack 打包导致的 require 缓存问题, - * 确保读取到的始终是磁盘上的最新配置。 + * 站点配置。 + * 仓库里的 wikitdb.config.js 是默认值(会打进部署包)。 + * 管理后台的改动写入 settings.site:config,无服务器实例没有可写磁盘时也能生效。 + * 磁盘可写时仍会尝试刷新文件,方便自建虚拟机沿用原来的工作流。 */ const fs = require('fs'); @@ -9,13 +10,13 @@ const path = require('path'); const vm = require('vm'); const CONFIG_FILENAME = 'wikitdb.config.js'; +const CONFIG_KEY = 'site:config'; function getConfigPath() { return path.join(process.cwd(), CONFIG_FILENAME); } -/** 从磁盘加载配置(vm 沙箱执行,不污染全局) */ -function loadSiteConfig() { +function loadSiteConfigFromFile() { const filePath = getConfigPath(); const source = fs.readFileSync(filePath, 'utf8'); const sandbox = { module: { exports: {} }, exports: {} }; @@ -25,7 +26,37 @@ function loadSiteConfig() { return cfg; } -/** 使 Node 侧的 require 缓存失效(对 auto-crawler.js 等独立进程生效) */ +/** 同步读取磁盘配置。已打包的页面仍直接 require 该文件。 */ +function loadSiteConfig() { + return loadSiteConfigFromFile(); +} + +function normalizeStored(value) { + if (!value) return null; + if (typeof value === 'string') { + try { return JSON.parse(value); } catch (e) { return null; } + } + return value; +} + +/** 数据库覆盖优先,没有记录或数据库不可用时退回文件。 */ +async function loadEffectiveConfig() { + const fileCfg = loadSiteConfigFromFile(); + try { + const prisma = require('../lib/prisma'); + const row = await prisma.setting.findUnique({ where: { key: CONFIG_KEY } }); + const stored = normalizeStored(row && row.value); + if (stored && Array.isArray(stored.SUPPORT_WIKI)) { + return { + ...fileCfg, + ...stored, + SUPPORT_WIKI: stored.SUPPORT_WIKI, + }; + } + } catch (e) { /* 构建期或未连上数据库时用文件 */ } + return fileCfg; +} + function invalidateConfigCache() { try { const resolved = require.resolve(getConfigPath()); @@ -33,7 +64,6 @@ function invalidateConfigCache() { } catch (e) { /* 打包环境中可能无法解析,忽略 */ } } -/** 生成配置文件内容(保留站点模板注释,便于后续手工编辑) */ function buildConfigFileContent(config) { const lines = []; lines.push('module.exports = {'); @@ -61,10 +91,27 @@ function buildConfigFileContent(config) { return lines.join('\n') + '\n'; } -/** 写回配置文件并刷新缓存 */ -function saveSiteConfig(config) { - fs.writeFileSync(getConfigPath(), buildConfigFileContent(config), 'utf8'); - invalidateConfigCache(); +async function saveSiteConfig(config) { + const payload = { + SITE_NAME: config.SITE_NAME || 'WikitDB', + SITE_URL: config.SITE_URL || '', + SITE_SINCE: config.SITE_SINCE || '2026', + SITE_AUTHOR: config.SITE_AUTHOR || 'WikitDB Team', + SUPPORT_WIKI: config.SUPPORT_WIKI || [], + }; + const prisma = require('../lib/prisma'); + await prisma.setting.upsert({ + where: { key: CONFIG_KEY }, + update: { value: payload }, + create: { key: CONFIG_KEY, value: payload }, + }); + try { + fs.writeFileSync(getConfigPath(), buildConfigFileContent(payload), 'utf8'); + invalidateConfigCache(); + } catch (e) { + if (e.code !== 'EROFS' && e.code !== 'EACCES' && e.code !== 'EPERM' && e.code !== 'ENOENT') throw e; + } + return payload; } function isHttpUrl(value) { @@ -77,10 +124,8 @@ function isHttpUrl(value) { } } -/** 校验并规范化一个站点对象,返回 { site } 或 { error } */ function validateSite(input) { input = input || {}; - const site = {}; site.NAME = String(input.NAME || '').trim(); @@ -100,7 +145,6 @@ function validateSite(input) { return { error: 'WIKIT_ID 只能包含字母、数字、点、下划线和连字符(1-80 位)' }; } - // 可选字段 if (input.ImgURL) { const imgUrl = String(input.ImgURL).trim(); if (!isHttpUrl(imgUrl)) return { error: 'Logo 链接 (ImgURL) 必须是有效的 http(s) 链接' }; @@ -127,8 +171,11 @@ function validateSite(input) { module.exports = { getConfigPath, loadSiteConfig, + loadSiteConfigFromFile, + loadEffectiveConfig, saveSiteConfig, validateSite, buildConfigFileContent, invalidateConfigCache, + CONFIG_KEY, }; diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..3dff4a7 --- /dev/null +++ b/vercel.json @@ -0,0 +1,12 @@ +{ + "crons": [ + { + "path": "/api/cron/crawl", + "schedule": "0 18 * * *" + }, + { + "path": "/api/cron/attribution", + "schedule": "30 18 * * *" + } + ] +}