Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 9 additions & 16 deletions checks/_helpers/collectors.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs'
import { readdir, readFile } from 'node:fs/promises'
import { basename, join } from 'node:path'
import { readBoundedResponseText, runCheckCommand } from '@harlan-zw/nuxt-checkin/external'
import { ANALYTICS_ACCOUNT_TAG, buildCopyQuery, buildCopyTotalsQuery, buildWorkersQuery, collectWorkflowRuns, parseWorkflowName, runListArgs, summarizeCopies, summarizeWorkflowRuns, workersQueryFailure, workersWindow } from './observability.mjs'
import { ANALYTICS_ACCOUNT_TAG, buildCopyQuery, buildCopyTotalsQuery, buildWorkersQuery, collectWorkflowRuns, parseAnalyticsEngineResponse, parseWorkersResponse, parseWorkflowName, runListArgs, summarizeCopies, summarizeWorkflowRuns, workersWindow } from './observability.mjs'
import { buildSignupQuery, summarizeSignupEvents } from './signup-analytics.ts'

const resources = Object.fromEntries(['github-auth', 'production-ref', 'git', 'deploy', 'ci', 'd1', 'workers', 'analytics'].map(key => [key, {}]))
Expand Down Expand Up @@ -323,15 +323,11 @@ export function collectWorkers(context) {
throw new Error('Cloudflare token is unavailable.')
const window = workersWindow(context.since.toISOString(), context.now.toISOString())
const response = await fetch('https://api.cloudflare.com/client/v4/graphql', { method: 'POST', redirect: 'error', signal: context.signal, headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ query: buildWorkersQuery(window.since, window.until) }) })
const body = JSON.parse(await readBoundedResponseText(response, 2_097_152))
const failure = workersQueryFailure(response.status, body)
if (failure)
throw new Error(failure)
const rows = body.data?.viewer?.accounts?.[0]?.workersInvocationsAdaptive
if (!Array.isArray(rows))
throw new Error('Cloudflare Worker evidence is unavailable.')
const parsed = parseWorkersResponse(response.status, await readBoundedResponseText(response, 2_097_152))
if (parsed._tag === 'Err')
throw new Error(parsed.reason)
const outcomes = {}
for (const row of rows) {
for (const row of parsed.rows) {
if (row.dimensions.scriptName === 'skilld-dev')
outcomes[row.dimensions.status] = (outcomes[row.dimensions.status] ?? 0) + row.sum.requests
}
Expand All @@ -356,13 +352,10 @@ export function collectAnalytics(context) {
throw new Error('Cloudflare token is unavailable.')
const readRows = async (query) => {
const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${ANALYTICS_ACCOUNT_TAG}/analytics_engine/sql`, { method: 'POST', redirect: 'error', signal: context.signal, headers: { Authorization: `Bearer ${token}` }, body: query })
const text = await readBoundedResponseText(response, 2_097_152)
if (!response.ok)
throw new Error(`Analytics Engine returned HTTP ${response.status}.`)
const rows = JSON.parse(text).data
if (!Array.isArray(rows))
throw new Error('Analytics Engine copy evidence is unavailable.')
return rows
const parsed = parseAnalyticsEngineResponse(response.status, await readBoundedResponseText(response, 2_097_152))
if (parsed._tag === 'Err')
throw new Error(parsed.reason)
return parsed.rows
}
const [totalsRows, topRows, signupRows] = await Promise.all([
readRows(buildCopyTotalsQuery(context.since.toISOString(), context.now.toISOString())),
Expand Down
57 changes: 48 additions & 9 deletions checks/_helpers/observability.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,54 @@ export function workersWindow(sinceIso, untilIso) {
return { _tag: 'clamped', since: new Date(earliest).toISOString(), until: untilIso, requestedSince: sinceIso }
}

/** The failure a Workers analytics response reports, or null when it carries data. */
export function workersQueryFailure(status, body) {
const messages = (body?.errors ?? []).map(error => error?.message).filter(Boolean)
if (status >= 200 && status < 300 && !messages.length)
return null
// GraphQL reports query errors with HTTP 200, so the status alone hid the cause.
return messages.length
? `Cloudflare GraphQL returned HTTP ${status}: ${messages.join('; ')}`
: `Cloudflare GraphQL returned HTTP ${status}.`
// The reason lands in the archive and the published report, so keep it short.
const FAILURE_DETAIL_LIMIT = 300

function failure(service, status, detail) {
const text = detail.trim()
if (!text)
return { _tag: 'Err', reason: `${service} returned HTTP ${status}.` }
const bounded = text.length > FAILURE_DETAIL_LIMIT ? `${text.slice(0, FAILURE_DETAIL_LIMIT)}…` : text
return { _tag: 'Err', reason: `${service} returned HTTP ${status}: ${bounded}` }
}

function parseJson(text) {
try {
return { _tag: 'Ok', value: JSON.parse(text) }
}
catch {
// A proxy error page is HTML. The caller reports the status instead.
return { _tag: 'Err' }
}
}

const isSuccess = status => status >= 200 && status < 300

/**
* Parse a Workers analytics response into its rows, or the reason it failed.
* GraphQL reports query errors inside an HTTP 200 body, so any `errors` entry
* fails the read, and its messages become the reason.
*/
export function parseWorkersResponse(status, text) {
const json = parseJson(text)
const errors = json._tag === 'Ok' ? json.value?.errors : undefined
const hasErrors = errors != null && (!Array.isArray(errors) || errors.length > 0)
if (!isSuccess(status) || json._tag === 'Err' || hasErrors) {
const messages = Array.isArray(errors) ? errors.map(error => error?.message).filter(message => typeof message === 'string') : []
return failure('Cloudflare GraphQL', status, messages.join('; '))
}
const rows = json.value?.data?.viewer?.accounts?.[0]?.workersInvocationsAdaptive
return Array.isArray(rows) ? { _tag: 'Ok', rows } : { _tag: 'Err', reason: 'Cloudflare Worker evidence is unavailable.' }
}

/** Parse an Analytics Engine SQL response into its rows, or the reason it failed. */
export function parseAnalyticsEngineResponse(status, text) {
// The SQL API answers a rejected query with a plain-text message.
if (!isSuccess(status))
return failure('Analytics Engine', status, text)
const json = parseJson(text)
const rows = json._tag === 'Ok' ? json.value?.data : undefined
return Array.isArray(rows) ? { _tag: 'Ok', rows } : { _tag: 'Err', reason: 'Analytics Engine copy evidence is unavailable.' }
}

// The archive records `since` but until #195 nothing derived a gap from it, so
Expand Down
1 change: 1 addition & 0 deletions shared/checkin-external.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ Interpret this site evidence:
- Any stable non-200 \`skilld.dev\` front door response is RED. A retry flap is a note.
- Read every \`ci.workflows\` state. \`failure\` means the gate is broken. \`pending\` after a prior failure must be followed until complete. \`missing\` is an observability gap. Never infer overall CI health from only the deploy workflow.
- Any \`workers.nonOk\`, new Sentry issue, failed digest, failed job, stale reserved job, stale scheduled task, or missing expected table needs an explicit verdict.
- Worker counts cover \`workers.window\`, which is at most 32 days. If its \`_tag\` is \`clamped\`, compute Worker rates over \`workers.window.since\` to \`workers.window.until\`, never over \`gapHours\`.
- \`d1.migrations.localHead !== d1.migrations.prodHead\` is migration drift. Never assume deployment applied D1 migrations.
- \`skilld.baseline\` result evidence records the window against the daily cadence. \`stale\` means the saved baseline is old; report the \`gapHours\` figure and compare rates per day, never as overnight step changes. The shared CLI rejects invalid or future state before collecting evidence. Repair the state or supply an explicit \`--since\` boundary.
- Use failed-job \`first_failed_at\` and \`last_failed_at\`, plus \`d1.recentJobBatches\` and \`d1.registryMaintenance\`, to distinguish an active incident from a recovered burst. Do not call a window clean because the latest batch passed, or active because an older batch failed.
Expand Down
38 changes: 31 additions & 7 deletions test/unit/daily-checkin-observability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@ import {
buildWorkersQuery,
collectWorkflowRuns,
deriveBaselineFlag,
parseAnalyticsEngineResponse,
parseWorkersResponse,
parseWorkflowName,
readMigrationState,
refreshProductionRef,
runListArgs,
summarizeCopies,
summarizeWorkflowRuns,
workersQueryFailure,
workersWindow,
} from '../../checks/_helpers/observability.mjs'

Expand Down Expand Up @@ -130,16 +131,39 @@ describe('daily check-in observability', () => {
})

it('names the GraphQL error that a 200 response carries', () => {
expect(workersQueryFailure(200, { errors: [{ message: 'cannot request a time range wider than 4w4d' }] }))
.toBe('Cloudflare GraphQL returned HTTP 200: cannot request a time range wider than 4w4d')
expect(parseWorkersResponse(200, JSON.stringify({ errors: [{ message: 'cannot request a time range wider than 4w4d' }] })))
.toEqual({ _tag: 'Err', reason: 'Cloudflare GraphQL returned HTTP 200: cannot request a time range wider than 4w4d' })
})

it('names the status of a failed response without errors', () => {
expect(workersQueryFailure(403, {})).toBe('Cloudflare GraphQL returned HTTP 403.')
it('fails a response whose errors carry no message', () => {
expect(parseWorkersResponse(200, JSON.stringify({ errors: [{}], data: null })))
.toEqual({ _tag: 'Err', reason: 'Cloudflare GraphQL returned HTTP 200.' })
})

it('accepts a 200 response without errors', () => {
expect(workersQueryFailure(200, { data: {} })).toBeNull()
it('keeps the status of a response that is not JSON', () => {
expect(parseWorkersResponse(502, '<html>Bad gateway</html>'))
.toEqual({ _tag: 'Err', reason: 'Cloudflare GraphQL returned HTTP 502.' })
})

it('bounds the length of a failure reason', () => {
const result = parseWorkersResponse(200, JSON.stringify({ errors: [{ message: 'x'.repeat(5_000) }] }))
expect(result._tag === 'Err' && result.reason.length).toBeLessThanOrEqual(400)
})

it('returns the Worker rows of a successful response', () => {
const rows = [{ dimensions: { scriptName: 'skilld-dev', status: 'success' }, sum: { requests: 3 } }]
expect(parseWorkersResponse(200, JSON.stringify({ data: { viewer: { accounts: [{ workersInvocationsAdaptive: rows }] } } })))
.toEqual({ _tag: 'Ok', rows })
})

it('names the Analytics Engine message of a failed query', () => {
expect(parseAnalyticsEngineResponse(422, 'unknown column: blob9'))
.toEqual({ _tag: 'Err', reason: 'Analytics Engine returned HTTP 422: unknown column: blob9' })
})

it('returns the Analytics Engine rows of a successful query', () => {
expect(parseAnalyticsEngineResponse(200, JSON.stringify({ data: [{ copies: 2 }] })))
.toEqual({ _tag: 'Ok', rows: [{ copies: 2 }] })
})

// The exact shape from issue #195: the routine ran on 2026-09-09 and not
Expand Down
Loading