diff --git a/checks/_helpers/collectors.mjs b/checks/_helpers/collectors.mjs index 96a47006..2360abbc 100644 --- a/checks/_helpers/collectors.mjs +++ b/checks/_helpers/collectors.mjs @@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs' import { readdir, readFile } from 'node:fs/promises' import { basename, join } from 'node:path' import { readBoundedResponseText, runCheckCommand } from '@harlan-zw/nuxt-checkin/external' -import { ANALYTICS_ACCOUNT_TAG, buildCopyQuery, buildCopyTotalsQuery, buildWorkersQuery, collectWorkflowRuns, parseWorkflowName, runListArgs, summarizeCopies, summarizeWorkflowRuns, workersQueryFailure, workersWindow } from './observability.mjs' +import { ANALYTICS_ACCOUNT_TAG, buildCopyQuery, buildCopyTotalsQuery, buildWorkersQuery, collectWorkflowRuns, parseAnalyticsEngineResponse, parseWorkersResponse, parseWorkflowName, runListArgs, summarizeCopies, summarizeWorkflowRuns, workersWindow } from './observability.mjs' import { buildSignupQuery, summarizeSignupEvents } from './signup-analytics.ts' const resources = Object.fromEntries(['github-auth', 'production-ref', 'git', 'deploy', 'ci', 'd1', 'workers', 'analytics'].map(key => [key, {}])) @@ -323,15 +323,11 @@ export function collectWorkers(context) { throw new Error('Cloudflare token is unavailable.') const window = workersWindow(context.since.toISOString(), context.now.toISOString()) const response = await fetch('https://api.cloudflare.com/client/v4/graphql', { method: 'POST', redirect: 'error', signal: context.signal, headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ query: buildWorkersQuery(window.since, window.until) }) }) - const body = JSON.parse(await readBoundedResponseText(response, 2_097_152)) - const failure = workersQueryFailure(response.status, body) - if (failure) - throw new Error(failure) - const rows = body.data?.viewer?.accounts?.[0]?.workersInvocationsAdaptive - if (!Array.isArray(rows)) - throw new Error('Cloudflare Worker evidence is unavailable.') + const parsed = parseWorkersResponse(response.status, await readBoundedResponseText(response, 2_097_152)) + if (parsed._tag === 'Err') + throw new Error(parsed.reason) const outcomes = {} - for (const row of rows) { + for (const row of parsed.rows) { if (row.dimensions.scriptName === 'skilld-dev') outcomes[row.dimensions.status] = (outcomes[row.dimensions.status] ?? 0) + row.sum.requests } @@ -356,13 +352,10 @@ export function collectAnalytics(context) { throw new Error('Cloudflare token is unavailable.') const readRows = async (query) => { const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${ANALYTICS_ACCOUNT_TAG}/analytics_engine/sql`, { method: 'POST', redirect: 'error', signal: context.signal, headers: { Authorization: `Bearer ${token}` }, body: query }) - const text = await readBoundedResponseText(response, 2_097_152) - if (!response.ok) - throw new Error(`Analytics Engine returned HTTP ${response.status}.`) - const rows = JSON.parse(text).data - if (!Array.isArray(rows)) - throw new Error('Analytics Engine copy evidence is unavailable.') - return rows + const parsed = parseAnalyticsEngineResponse(response.status, await readBoundedResponseText(response, 2_097_152)) + if (parsed._tag === 'Err') + throw new Error(parsed.reason) + return parsed.rows } const [totalsRows, topRows, signupRows] = await Promise.all([ readRows(buildCopyTotalsQuery(context.since.toISOString(), context.now.toISOString())), diff --git a/checks/_helpers/observability.mjs b/checks/_helpers/observability.mjs index d7776e60..7c293879 100644 --- a/checks/_helpers/observability.mjs +++ b/checks/_helpers/observability.mjs @@ -65,15 +65,54 @@ export function workersWindow(sinceIso, untilIso) { return { _tag: 'clamped', since: new Date(earliest).toISOString(), until: untilIso, requestedSince: sinceIso } } -/** The failure a Workers analytics response reports, or null when it carries data. */ -export function workersQueryFailure(status, body) { - const messages = (body?.errors ?? []).map(error => error?.message).filter(Boolean) - if (status >= 200 && status < 300 && !messages.length) - return null - // GraphQL reports query errors with HTTP 200, so the status alone hid the cause. - return messages.length - ? `Cloudflare GraphQL returned HTTP ${status}: ${messages.join('; ')}` - : `Cloudflare GraphQL returned HTTP ${status}.` +// The reason lands in the archive and the published report, so keep it short. +const FAILURE_DETAIL_LIMIT = 300 + +function failure(service, status, detail) { + const text = detail.trim() + if (!text) + return { _tag: 'Err', reason: `${service} returned HTTP ${status}.` } + const bounded = text.length > FAILURE_DETAIL_LIMIT ? `${text.slice(0, FAILURE_DETAIL_LIMIT)}…` : text + return { _tag: 'Err', reason: `${service} returned HTTP ${status}: ${bounded}` } +} + +function parseJson(text) { + try { + return { _tag: 'Ok', value: JSON.parse(text) } + } + catch { + // A proxy error page is HTML. The caller reports the status instead. + return { _tag: 'Err' } + } +} + +const isSuccess = status => status >= 200 && status < 300 + +/** + * Parse a Workers analytics response into its rows, or the reason it failed. + * GraphQL reports query errors inside an HTTP 200 body, so any `errors` entry + * fails the read, and its messages become the reason. + */ +export function parseWorkersResponse(status, text) { + const json = parseJson(text) + const errors = json._tag === 'Ok' ? json.value?.errors : undefined + const hasErrors = errors != null && (!Array.isArray(errors) || errors.length > 0) + if (!isSuccess(status) || json._tag === 'Err' || hasErrors) { + const messages = Array.isArray(errors) ? errors.map(error => error?.message).filter(message => typeof message === 'string') : [] + return failure('Cloudflare GraphQL', status, messages.join('; ')) + } + const rows = json.value?.data?.viewer?.accounts?.[0]?.workersInvocationsAdaptive + return Array.isArray(rows) ? { _tag: 'Ok', rows } : { _tag: 'Err', reason: 'Cloudflare Worker evidence is unavailable.' } +} + +/** Parse an Analytics Engine SQL response into its rows, or the reason it failed. */ +export function parseAnalyticsEngineResponse(status, text) { + // The SQL API answers a rejected query with a plain-text message. + if (!isSuccess(status)) + return failure('Analytics Engine', status, text) + const json = parseJson(text) + const rows = json._tag === 'Ok' ? json.value?.data : undefined + return Array.isArray(rows) ? { _tag: 'Ok', rows } : { _tag: 'Err', reason: 'Analytics Engine copy evidence is unavailable.' } } // The archive records `since` but until #195 nothing derived a gap from it, so diff --git a/shared/checkin-external.ts b/shared/checkin-external.ts index 456482ad..17f4b513 100644 --- a/shared/checkin-external.ts +++ b/shared/checkin-external.ts @@ -13,6 +13,7 @@ Interpret this site evidence: - Any stable non-200 \`skilld.dev\` front door response is RED. A retry flap is a note. - Read every \`ci.workflows\` state. \`failure\` means the gate is broken. \`pending\` after a prior failure must be followed until complete. \`missing\` is an observability gap. Never infer overall CI health from only the deploy workflow. - Any \`workers.nonOk\`, new Sentry issue, failed digest, failed job, stale reserved job, stale scheduled task, or missing expected table needs an explicit verdict. + - Worker counts cover \`workers.window\`, which is at most 32 days. If its \`_tag\` is \`clamped\`, compute Worker rates over \`workers.window.since\` to \`workers.window.until\`, never over \`gapHours\`. - \`d1.migrations.localHead !== d1.migrations.prodHead\` is migration drift. Never assume deployment applied D1 migrations. - \`skilld.baseline\` result evidence records the window against the daily cadence. \`stale\` means the saved baseline is old; report the \`gapHours\` figure and compare rates per day, never as overnight step changes. The shared CLI rejects invalid or future state before collecting evidence. Repair the state or supply an explicit \`--since\` boundary. - Use failed-job \`first_failed_at\` and \`last_failed_at\`, plus \`d1.recentJobBatches\` and \`d1.registryMaintenance\`, to distinguish an active incident from a recovered burst. Do not call a window clean because the latest batch passed, or active because an older batch failed. diff --git a/test/unit/daily-checkin-observability.test.ts b/test/unit/daily-checkin-observability.test.ts index 5b1a26a8..f105589f 100644 --- a/test/unit/daily-checkin-observability.test.ts +++ b/test/unit/daily-checkin-observability.test.ts @@ -6,13 +6,14 @@ import { buildWorkersQuery, collectWorkflowRuns, deriveBaselineFlag, + parseAnalyticsEngineResponse, + parseWorkersResponse, parseWorkflowName, readMigrationState, refreshProductionRef, runListArgs, summarizeCopies, summarizeWorkflowRuns, - workersQueryFailure, workersWindow, } from '../../checks/_helpers/observability.mjs' @@ -130,16 +131,39 @@ describe('daily check-in observability', () => { }) it('names the GraphQL error that a 200 response carries', () => { - expect(workersQueryFailure(200, { errors: [{ message: 'cannot request a time range wider than 4w4d' }] })) - .toBe('Cloudflare GraphQL returned HTTP 200: cannot request a time range wider than 4w4d') + expect(parseWorkersResponse(200, JSON.stringify({ errors: [{ message: 'cannot request a time range wider than 4w4d' }] }))) + .toEqual({ _tag: 'Err', reason: 'Cloudflare GraphQL returned HTTP 200: cannot request a time range wider than 4w4d' }) }) - it('names the status of a failed response without errors', () => { - expect(workersQueryFailure(403, {})).toBe('Cloudflare GraphQL returned HTTP 403.') + it('fails a response whose errors carry no message', () => { + expect(parseWorkersResponse(200, JSON.stringify({ errors: [{}], data: null }))) + .toEqual({ _tag: 'Err', reason: 'Cloudflare GraphQL returned HTTP 200.' }) }) - it('accepts a 200 response without errors', () => { - expect(workersQueryFailure(200, { data: {} })).toBeNull() + it('keeps the status of a response that is not JSON', () => { + expect(parseWorkersResponse(502, 'Bad gateway')) + .toEqual({ _tag: 'Err', reason: 'Cloudflare GraphQL returned HTTP 502.' }) + }) + + it('bounds the length of a failure reason', () => { + const result = parseWorkersResponse(200, JSON.stringify({ errors: [{ message: 'x'.repeat(5_000) }] })) + expect(result._tag === 'Err' && result.reason.length).toBeLessThanOrEqual(400) + }) + + it('returns the Worker rows of a successful response', () => { + const rows = [{ dimensions: { scriptName: 'skilld-dev', status: 'success' }, sum: { requests: 3 } }] + expect(parseWorkersResponse(200, JSON.stringify({ data: { viewer: { accounts: [{ workersInvocationsAdaptive: rows }] } } }))) + .toEqual({ _tag: 'Ok', rows }) + }) + + it('names the Analytics Engine message of a failed query', () => { + expect(parseAnalyticsEngineResponse(422, 'unknown column: blob9')) + .toEqual({ _tag: 'Err', reason: 'Analytics Engine returned HTTP 422: unknown column: blob9' }) + }) + + it('returns the Analytics Engine rows of a successful query', () => { + expect(parseAnalyticsEngineResponse(200, JSON.stringify({ data: [{ copies: 2 }] }))) + .toEqual({ _tag: 'Ok', rows: [{ copies: 2 }] }) }) // The exact shape from issue #195: the routine ran on 2026-09-09 and not