From 2c56c363c5af109fcdca26876a9d7df76d019d8f Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 15:29:19 +0000 Subject: [PATCH 001/102] feat(credential-proxy): add bounded bearer forwarding --- .../credential-proxy/src/bearer-forwarder.mjs | 151 ++++++++++++++++++ .../src/destination-policy.mjs | 11 +- .../test/bearer-forwarder.test.mjs | 77 +++++++++ .../test/destination-policy.test.mjs | 2 +- 4 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 packages/credential-proxy/src/bearer-forwarder.mjs create mode 100644 packages/credential-proxy/test/bearer-forwarder.test.mjs diff --git a/packages/credential-proxy/src/bearer-forwarder.mjs b/packages/credential-proxy/src/bearer-forwarder.mjs new file mode 100644 index 00000000..5fc6ee80 --- /dev/null +++ b/packages/credential-proxy/src/bearer-forwarder.mjs @@ -0,0 +1,151 @@ +import https from "node:https"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createOriginConnector } from "./destination-policy.mjs"; + +export const DEFAULT_FORWARD_LIMITS = Object.freeze({ + connectMs: 10_000, + headersMs: 60_000, + idleMs: 120_000, + requestMs: 15 * 60_000, + requestBytes: 32 * 1024 * 1024, + responseBytes: 64 * 1024 * 1024, + headerBytes: 32 * 1024, +}); + +const HOP_BY_HOP = new Set([ + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailer", "transfer-encoding", "upgrade", "host", "authorization", + "content-length", "expect", +]); + +function safeHeaders(input, credential, maxBytes) { + const source = input instanceof Headers ? [...input.entries()] : Object.entries(input ?? {}); + const connectionTokens = new Set(); + for (const [name, value] of source) { + if (name.toLowerCase() === "connection") { + for (const token of String(value).split(",")) connectionTokens.add(token.trim().toLowerCase()); + } + } + const headers = Object.create(null); + let size = 0; + for (const [name, value] of source) { + const lower = name.toLowerCase(); + if (HOP_BY_HOP.has(lower) || lower.startsWith("x-forwarded-") || lower === "forwarded" || connectionTokens.has(lower)) continue; + const text = Array.isArray(value) ? value.join(", ") : String(value); + size += Buffer.byteLength(name) + Buffer.byteLength(text) + 4; + if (size > maxBytes) throw new RangeError("request headers exceed configured limit"); + headers[lower] = text; + } + headers.authorization = `Bearer ${credential}`; + return headers; +} + +function byteLimit(limit, label, onLimit) { + let bytes = 0; + return new Transform({ + transform(chunk, _encoding, callback) { + bytes += chunk.length; + if (bytes > limit) { + onLimit?.(); + callback(new RangeError(`${label} exceeds configured limit`)); + } else callback(null, chunk); + }, + }); +} + +function sendHttpsRequest({ socket, url, method, headers, body, limits, signal }) { + return new Promise((resolve, reject) => { + let settled = false; + let responseTimer; + let idleTimer; + const finishError = (error) => { + if (settled) return; + settled = true; + clearTimeout(responseTimer); + clearTimeout(idleTimer); + socket.destroy(error); + reject(error); + }; + const request = https.request(url, { + method, + headers, + agent: false, + createConnection: () => socket, + maxHeaderSize: limits.headerBytes, + signal, + }, (response) => { + clearTimeout(responseTimer); + let responseBytes = 0; + const chunks = []; + const resetIdle = () => { + clearTimeout(idleTimer); + idleTimer = setTimeout(() => finishError(new Error("upstream response idle deadline exceeded")), limits.idleMs); + idleTimer.unref?.(); + }; + response.on("data", (chunk) => { + responseBytes += chunk.length; + if (responseBytes > limits.responseBytes) return finishError(new RangeError("upstream response exceeds configured limit")); + chunks.push(chunk); + resetIdle(); + }); + response.once("error", finishError); + if (response.statusCode >= 300 && response.statusCode < 400) { + response.resume(); + return finishError(new Error("upstream redirects are not permitted")); + } + response.once("end", () => { + if (settled) return; + settled = true; + clearTimeout(idleTimer); + resolve({ status: response.statusCode, headers: response.headers, body: Buffer.concat(chunks) }); + }); + resetIdle(); + }); + request.once("error", finishError); + responseTimer = setTimeout(() => finishError(new Error("upstream response headers deadline exceeded")), limits.headersMs); + responseTimer.unref?.(); + if (!body) request.end(); + else { + const input = body[Symbol.asyncIterator] ? body : [body]; + pipeline(input, byteLimit(limits.requestBytes, "request body", () => request.destroy()), request, { signal }).catch(finishError); + } + }); +} + +/** + * Forward one request through a grant selected by authenticated connection state. + * The caller supplies the decrypted integration record; agent-controlled data + * cannot select either its credential or origin. + */ +export function createBearerForwarder({ connectorFactory = createOriginConnector, request = sendHttpsRequest, limits = DEFAULT_FORWARD_LIMITS } = {}) { + const active = new Set(); + return Object.freeze({ + async forward(integration, input, { signal } = {}) { + if (!integration || typeof integration.origin !== "string" || typeof integration.credential !== "string" || !integration.credential) { + throw new TypeError("integration grant is unavailable"); + } + if (active.size >= 16) throw new Error("concurrent request limit exceeded"); + const controller = new AbortController(); + const abort = () => controller.abort(signal?.reason); + signal?.addEventListener("abort", abort, { once: true }); + const timeout = setTimeout(() => controller.abort(new Error("request duration limit exceeded")), limits.requestMs); + timeout.unref?.(); + const key = {}; + active.add(key); + try { + const connect = connectorFactory(integration.origin, { timeoutMs: limits.connectMs }); + const { socket, url } = await connect(input.url); + const method = String(input.method ?? "GET").toUpperCase(); + if (!/^[A-Z!#$%&'*+.^_`|~-]+$/.test(method)) throw new TypeError("invalid HTTP method"); + const headers = safeHeaders(input.headers, integration.credential, limits.headerBytes); + return await request({ socket, url, method, headers, body: input.body, limits, signal: controller.signal }); + } finally { + clearTimeout(timeout); + signal?.removeEventListener("abort", abort); + active.delete(key); + } + }, + get activeRequests() { return active.size; }, + }); +} diff --git a/packages/credential-proxy/src/destination-policy.mjs b/packages/credential-proxy/src/destination-policy.mjs index 0b71dbbb..e0bbba9e 100644 --- a/packages/credential-proxy/src/destination-policy.mjs +++ b/packages/credential-proxy/src/destination-policy.mjs @@ -109,7 +109,7 @@ async function resolveAll(hostname) { return lookup(hostname, { all: true, verbatim: true }); } -function openTlsSocket({ hostname, port, address }) { +function openTlsSocket({ hostname, port, address, timeoutMs = 10_000 }) { return new Promise((resolve, reject) => { const socket = tlsConnect({ host: address, @@ -117,9 +117,12 @@ function openTlsSocket({ hostname, port, address }) { servername: isIP(hostname) ? undefined : hostname, checkServerIdentity: (_server, certificate) => checkServerIdentity(hostname, certificate), }); - const failed = (error) => { socket.destroy(); reject(error); }; + const timer = setTimeout(() => failed(new Error("upstream connect deadline exceeded")), timeoutMs); + timer.unref?.(); + const failed = (error) => { clearTimeout(timer); socket.destroy(); reject(error); }; socket.once("error", failed); socket.once("secureConnect", () => { + clearTimeout(timer); socket.removeListener("error", failed); resolve(socket); }); @@ -132,7 +135,7 @@ function openTlsSocket({ hostname, port, address }) { * so a second DNS lookup cannot change the destination between validation and * connection. Paths, queries, methods, and bodies remain available to callers. */ -export function createOriginConnector(configuredOrigin, { resolve = resolveAll, connect = openTlsSocket } = {}) { +export function createOriginConnector(configuredOrigin, { resolve = resolveAll, connect = openTlsSocket, timeoutMs = 10_000 } = {}) { const configured = parseHttpsUrl(configuredOrigin, "configured origin"); if (configured.url.pathname !== "/" || configured.url.search || configured.url.hash) { throw new TypeError("configured origin must contain only an HTTPS origin"); @@ -148,7 +151,7 @@ export function createOriginConnector(configuredOrigin, { resolve = resolveAll, if (!answer || !isPublicAddress(answer.address)) throw new Error("origin resolved to a non-public address"); } const address = answers[0].address; - const socket = await connect({ hostname: target.hostname, port: target.port, address, servername: target.hostname }); + const socket = await connect({ hostname: target.hostname, port: target.port, address, servername: target.hostname, timeoutMs }); const rawPeerAddress = socket?.remoteAddress ?? ""; const peerAddress = normalizeAddress(rawPeerAddress); if (!peerAddress || peerAddress !== normalizeAddress(address) || !isPublicAddress(rawPeerAddress)) { diff --git a/packages/credential-proxy/test/bearer-forwarder.test.mjs b/packages/credential-proxy/test/bearer-forwarder.test.mjs new file mode 100644 index 00000000..db1a2154 --- /dev/null +++ b/packages/credential-proxy/test/bearer-forwarder.test.mjs @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; + +const integration = { id: "integration-1", origin: "https://api.example.test", credential: "synthetic-token" }; + +test("forwards only through the granted origin and replaces unsafe routing and authorization headers", async () => { + const connections = []; + const calls = []; + const forwarder = createBearerForwarder({ + connectorFactory: origin => async url => { + connections.push({ origin, url }); + return { socket: { fixture: true }, url: new URL(url) }; + }, + request: async request => { calls.push(request); return { status: 201, body: Buffer.from("created") }; }, + }); + const result = await forwarder.forward(integration, { + method: "post", + url: "https://api.example.test/v1/items?source=job", + headers: { + Authorization: "Bearer attacker-token", + Host: "attacker.example", + Connection: "keep-alive, x-routing-hint", + "X-Routing-Hint": "attacker", + "X-Forwarded-Host": "attacker.example", + "Content-Type": "application/json", + }, + body: Buffer.from('{"ok":true}'), + }); + assert.deepEqual(result, { status: 201, body: Buffer.from("created") }); + assert.deepEqual(connections, [{ origin: integration.origin, url: "https://api.example.test/v1/items?source=job" }]); + assert.equal(calls[0].method, "POST"); + assert.equal(calls[0].headers.authorization, "Bearer synthetic-token"); + assert.equal(calls[0].headers.host, undefined); + assert.equal(calls[0].headers["x-routing-hint"], undefined); + assert.equal(calls[0].headers["x-forwarded-host"], undefined); + assert.equal(calls[0].headers["content-type"], "application/json"); +}); + +test("rejects requests outside the grant origin without sending them upstream", async () => { + let sent = false; + const forwarder = createBearerForwarder({ + connectorFactory: () => async () => { throw new TypeError("request URL does not match the configured HTTPS origin"); }, + request: async () => { sent = true; }, + }); + await assert.rejects(forwarder.forward(integration, { url: "https://169.254.169.254/latest/meta-data" }), /configured HTTPS origin/); + assert.equal(sent, false); + assert.equal(forwarder.activeRequests, 0); +}); + +test("applies an explicit per-proxy concurrent request bound", async () => { + let release; + const gate = new Promise(resolve => { release = resolve; }); + const forwarder = createBearerForwarder({ + connectorFactory: () => async url => ({ socket: {}, url: new URL(url) }), + request: async () => gate, + limits: { requestMs: 10_000 }, + }); + const started = []; + for (let i = 0; i < 16; i++) started.push(forwarder.forward(integration, { url: "https://api.example.test/" })); + await new Promise(resolve => setImmediate(resolve)); + await assert.rejects(forwarder.forward(integration, { url: "https://api.example.test/" }), /concurrent request limit/); + release({ status: 200 }); + await Promise.all(started); + assert.equal(forwarder.activeRequests, 0); +}); + +test("rejects oversized caller headers before upstream dispatch", async () => { + let sent = false; + const forwarder = createBearerForwarder({ + connectorFactory: () => async url => ({ socket: {}, url: new URL(url) }), + request: async () => { sent = true; }, + limits: { headerBytes: 20 }, + }); + await assert.rejects(forwarder.forward(integration, { url: "https://api.example.test/", headers: { "x-large": "x".repeat(50) } }), /headers exceed/); + assert.equal(sent, false); +}); diff --git a/packages/credential-proxy/test/destination-policy.test.mjs b/packages/credential-proxy/test/destination-policy.test.mjs index d6266632..264a3055 100644 --- a/packages/credential-proxy/test/destination-policy.test.mjs +++ b/packages/credential-proxy/test/destination-policy.test.mjs @@ -18,7 +18,7 @@ test("URL policy permits paths and queries only on the configured HTTPS origin", const result = await connect("https://api.example.test:8443/v1/items?limit=5"); assert.equal(result.url.pathname, "/v1/items"); assert.equal(result.url.search, "?limit=5"); - assert.deepEqual(calls, [{ hostname: "api.example.test", port: 8443, address: "93.184.216.34", servername: "api.example.test" }]); + assert.deepEqual(calls, [{ hostname: "api.example.test", port: 8443, address: "93.184.216.34", servername: "api.example.test", timeoutMs: 10_000 }]); }); test("rejects unsafe and malformed request URLs before resolution", async () => { From 41d09a34220b287f8342b48bb93640f16cd3f7c0 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 15:42:04 +0000 Subject: [PATCH 002/102] feat(credential-proxy): stream authorized tunnel requests --- .../credential-proxy/src/bearer-forwarder.mjs | 71 +++++++---- packages/credential-proxy/src/main.mjs | 8 +- .../credential-proxy/src/proxy-listener.mjs | 117 ++++++++++++++++++ packages/credential-proxy/src/runtime.mjs | 52 ++++++-- .../test/bearer-forwarder.test.mjs | 6 +- .../test/fixtures/upstream-cert.pem | 20 +++ .../test/fixtures/upstream-key.pem | 28 +++++ .../test/proxy-listener.e2e.test.mjs | 117 ++++++++++++++++++ 8 files changed, 379 insertions(+), 40 deletions(-) create mode 100644 packages/credential-proxy/src/proxy-listener.mjs create mode 100644 packages/credential-proxy/test/fixtures/upstream-cert.pem create mode 100644 packages/credential-proxy/test/fixtures/upstream-key.pem create mode 100644 packages/credential-proxy/test/proxy-listener.e2e.test.mjs diff --git a/packages/credential-proxy/src/bearer-forwarder.mjs b/packages/credential-proxy/src/bearer-forwarder.mjs index 5fc6ee80..37ea1854 100644 --- a/packages/credential-proxy/src/bearer-forwarder.mjs +++ b/packages/credential-proxy/src/bearer-forwarder.mjs @@ -59,48 +59,58 @@ function sendHttpsRequest({ socket, url, method, headers, body, limits, signal } let settled = false; let responseTimer; let idleTimer; + let responseStream; + const agent = new https.Agent({ keepAlive: false }); + agent.createConnection = () => socket; const finishError = (error) => { - if (settled) return; + if (settled) { + responseStream?.destroy(error); + return; + } settled = true; clearTimeout(responseTimer); clearTimeout(idleTimer); socket.destroy(error); + agent.destroy(); reject(error); }; const request = https.request(url, { method, headers, - agent: false, - createConnection: () => socket, + agent, maxHeaderSize: limits.headerBytes, signal, }, (response) => { clearTimeout(responseTimer); let responseBytes = 0; - const chunks = []; const resetIdle = () => { clearTimeout(idleTimer); - idleTimer = setTimeout(() => finishError(new Error("upstream response idle deadline exceeded")), limits.idleMs); + idleTimer = setTimeout(() => responseStream.destroy(new Error("upstream response idle deadline exceeded")), limits.idleMs); idleTimer.unref?.(); }; - response.on("data", (chunk) => { - responseBytes += chunk.length; - if (responseBytes > limits.responseBytes) return finishError(new RangeError("upstream response exceeds configured limit")); - chunks.push(chunk); - resetIdle(); + responseStream = new Transform({ + transform(chunk, _encoding, callback) { + responseBytes += chunk.length; + if (responseBytes > limits.responseBytes) return callback(new RangeError("upstream response exceeds configured limit")); + resetIdle(); + callback(null, chunk); + }, + }); + response.once("error", (error) => responseStream.destroy(error)); + responseStream.once("error", (error) => { + response.destroy(error); + request.destroy(error); }); - response.once("error", finishError); + responseStream.once("close", () => clearTimeout(idleTimer)); + responseStream.once("close", () => agent.destroy()); if (response.statusCode >= 300 && response.statusCode < 400) { response.resume(); return finishError(new Error("upstream redirects are not permitted")); } - response.once("end", () => { - if (settled) return; - settled = true; - clearTimeout(idleTimer); - resolve({ status: response.statusCode, headers: response.headers, body: Buffer.concat(chunks) }); - }); resetIdle(); + response.pipe(responseStream); + settled = true; + resolve({ status: response.statusCode, headers: response.headers, body: responseStream }); }); request.once("error", finishError); responseTimer = setTimeout(() => finishError(new Error("upstream response headers deadline exceeded")), limits.headersMs); @@ -133,17 +143,34 @@ export function createBearerForwarder({ connectorFactory = createOriginConnector timeout.unref?.(); const key = {}; active.add(key); + let released = false; + const release = () => { + if (released) return; + released = true; + clearTimeout(timeout); + signal?.removeEventListener("abort", abort); + active.delete(key); + }; try { const connect = connectorFactory(integration.origin, { timeoutMs: limits.connectMs }); const { socket, url } = await connect(input.url); + if (controller.signal.aborted) { + socket.destroy(controller.signal.reason); + throw controller.signal.reason ?? new Error("request cancelled"); + } const method = String(input.method ?? "GET").toUpperCase(); if (!/^[A-Z!#$%&'*+.^_`|~-]+$/.test(method)) throw new TypeError("invalid HTTP method"); const headers = safeHeaders(input.headers, integration.credential, limits.headerBytes); - return await request({ socket, url, method, headers, body: input.body, limits, signal: controller.signal }); - } finally { - clearTimeout(timeout); - signal?.removeEventListener("abort", abort); - active.delete(key); + const result = await request({ socket, url, method, headers, body: input.body, limits, signal: controller.signal }); + if (result?.body && typeof result.body.once === "function") { + result.body.once("end", release); + result.body.once("close", release); + result.body.once("error", release); + } else release(); + return result; + } catch (error) { + release(); + throw error; } }, get activeRequests() { return active.size; }, diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 20c32377..28cd1f65 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -10,10 +10,10 @@ function respond(value) { process.stdout.write(`${JSON.stringify(value)}\n`); } -function stop() { +async function stop() { if (closing) return; closing = true; - runtime.shutdown(); + await runtime.shutdown(); lines.close(); clearTimeout(shutdownDeadline); process.exit(0); @@ -25,7 +25,7 @@ shutdownDeadline = setTimeout(() => process.exit(1), 2_000); shutdownDeadline.unref(); respond({ type: "ready", state: runtime.state }); -lines.on("line", (line) => { +lines.on("line", async (line) => { if (closing) return; let command; try { @@ -35,7 +35,7 @@ lines.on("line", (line) => { runtime.unlock(command.key); respond({ type: "status", state: runtime.state }); } else if (command?.type === "lock") { - runtime.lock(); + await runtime.lock(); respond({ type: "status", state: runtime.state }); } else if (command?.type === "stop") stop(); else respond({ type: "error", error: "unsupported command" }); diff --git a/packages/credential-proxy/src/proxy-listener.mjs b/packages/credential-proxy/src/proxy-listener.mjs new file mode 100644 index 00000000..3def1524 --- /dev/null +++ b/packages/credential-proxy/src/proxy-listener.mjs @@ -0,0 +1,117 @@ +import { createServer } from "node:http"; +import { pipeline } from "node:stream/promises"; +import { createBearerForwarder } from "./bearer-forwarder.mjs"; + +const RESPONSE_HOP_BY_HOP = new Set([ + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", + "trailer", "transfer-encoding", "upgrade", +]); + +function grantTable(grants) { + if (!Array.isArray(grants)) throw new TypeError("authenticated connection grants must be an array"); + const table = new Map(); + for (const grant of grants) { + if (!grant || typeof grant.id !== "string" || !grant.id || table.has(grant.id) + || typeof grant.origin !== "string" || typeof grant.credential !== "string" || !grant.credential) { + throw new TypeError("authenticated connection contains an invalid integration grant"); + } + table.set(grant.id, Object.freeze({ id: grant.id, origin: grant.origin, credential: grant.credential })); + } + return table; +} + +function requestTarget(origin, suffix, search) { + const base = new URL(origin); + if (base.protocol !== "https:" || base.pathname !== "/" || base.search || base.hash) { + throw new TypeError("integration grant origin must be an HTTPS origin"); + } + const path = suffix || "/"; + if (!path.startsWith("/") || path.startsWith("//") || path.includes("\\")) { + throw new TypeError("invalid integration request path"); + } + return `${base.origin}${path}${search}`; +} + +/** + * Start one loopback HTTP endpoint for a single already-authenticated tunnel + * connection. The connection's consumed grants are held in this closure and + * can never be selected or supplied by an HTTP caller. + */ +export async function createAuthenticatedProxyListener({ connection, forwarder = createBearerForwarder(), host = "127.0.0.1", port = 0 } = {}) { + if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true + || !connection.connectionId || !connection.runId || !connection.vmId) { + throw new TypeError("proxy listener requires an authenticated consumed connection"); + } + const grants = grantTable(connection.grants); + const controllers = new Set(); + const server = createServer(async (request, response) => { + const controller = new AbortController(); + controllers.add(controller); + const cancel = () => { + if (!response.writableEnded) controller.abort(new Error("tunnel request cancelled")); + }; + request.once("aborted", cancel); + response.once("close", cancel); + try { + const parsed = new URL(request.url ?? "/", "http://proxy.loopback"); + const match = /^\/integrations\/([^/]+)(\/.*)?$/.exec(parsed.pathname); + if (!match) { + response.writeHead(404).end(); + return; + } + let integrationId; + try { integrationId = decodeURIComponent(match[1]); } catch { + response.writeHead(400).end(); + return; + } + if (!integrationId || integrationId.includes("/") || integrationId.includes("\\")) { + response.writeHead(400).end(); + return; + } + const integration = grants.get(integrationId); + if (!integration) { + response.writeHead(403).end(); + return; + } + const target = requestTarget(integration.origin, match[2] ?? "/", parsed.search); + const result = await forwarder.forward(integration, { + url: target, + method: request.method, + headers: request.headers, + body: request, + }, { signal: controller.signal }); + const headers = Object.create(null); + for (const [name, value] of Object.entries(result.headers ?? {})) { + if (value !== undefined && !RESPONSE_HOP_BY_HOP.has(name.toLowerCase())) headers[name] = value; + } + response.writeHead(result.status, headers); + await pipeline(result.body, response, { signal: controller.signal }); + } catch (error) { + if (!response.headersSent && !response.destroyed) { + response.writeHead(controller.signal.aborted ? 499 : 502, { "content-type": "text/plain; charset=utf-8" }); + response.end(controller.signal.aborted ? "request cancelled" : "upstream request failed"); + } else if (!response.destroyed) response.destroy(error); + } finally { + controllers.delete(controller); + request.removeListener("aborted", cancel); + response.removeListener("close", cancel); + } + }); + server.on("connect", (_request, socket) => { socket.end("HTTP/1.1 405 Method Not Allowed\r\nConnection: close\r\n\r\n"); }); + server.on("upgrade", (_request, socket) => socket.destroy()); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { + server.removeListener("error", reject); + resolve(); + }); + }); + return Object.freeze({ + connectionId: connection.connectionId, + address: server.address(), + close: () => { + for (const controller of controllers) controller.abort(new Error("authenticated tunnel disconnected")); + return new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + }, + }); +} diff --git a/packages/credential-proxy/src/runtime.mjs b/packages/credential-proxy/src/runtime.mjs index e85401c7..4627acb9 100644 --- a/packages/credential-proxy/src/runtime.mjs +++ b/packages/credential-proxy/src/runtime.mjs @@ -1,17 +1,23 @@ +import { createAuthenticatedProxyListener } from "./proxy-listener.mjs"; + /** - * Internal adapter contracts for the follow-up record, enrollment and forwarding - * slices. This runtime deliberately does not call or implement these adapters. - * - * @typedef {object} ProxyAdapters - * @property {(identity: object) => Promise} loadRecords - * @property {(request: object) => Promise} createEnrollment - * @property {(request: object) => Promise} forwardRequest + * Process-local runtime for decrypted records and authorized tunnel listeners. + * The transport adapter must pass a connection only after it has consumed and + * authenticated its enrollment against the current run and proxy boot epoch. */ -export function createProxyRuntime() { +export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyListener } = {}) { let unlockMaterial; let state = "locked"; let shutdown = false; + let locking = false; + const listeners = new Set(); + + async function closeListeners() { + const current = [...listeners]; + listeners.clear(); + await Promise.allSettled(current.map((listener) => listener.close())); + } return Object.freeze({ get state() { @@ -19,19 +25,41 @@ export function createProxyRuntime() { }, unlock(key) { if (shutdown) throw new Error("runtime is shutting down"); + if (locking) throw new Error("proxy is locking"); if (typeof key !== "string" || key.length === 0) throw new TypeError("unlock key must be a non-empty string"); unlockMaterial?.fill(0); unlockMaterial = Buffer.from(key, "utf8"); state = "unlocked"; }, - lock() { + async lock() { + locking = true; + state = "locked"; unlockMaterial?.fill(0); unlockMaterial = undefined; - if (!shutdown) state = "locked"; + try { await closeListeners(); } + finally { locking = false; } + }, + async acceptAuthenticatedConnection(connection) { + if (shutdown) throw new Error("runtime is shutting down"); + if (state !== "unlocked") throw new Error("proxy is locked"); + // The transport adapter calls this only after atomically consuming and + // authenticating enrollment. Grants are the consumed, current run state. + const listener = await listenerFactory({ connection }); + if (state !== "unlocked" || shutdown) { + await listener.close(); + throw new Error("proxy locked while accepting the connection"); + } + listeners.add(listener); + return listener; + }, + async disconnect(connectionId) { + const matching = [...listeners].filter((listener) => listener.connectionId === connectionId); + await Promise.allSettled(matching.map((listener) => listener.close())); + for (const listener of matching) listeners.delete(listener); }, - shutdown() { - this.lock(); + async shutdown() { shutdown = true; + await this.lock(); state = "stopped"; }, }); diff --git a/packages/credential-proxy/test/bearer-forwarder.test.mjs b/packages/credential-proxy/test/bearer-forwarder.test.mjs index db1a2154..9284b49a 100644 --- a/packages/credential-proxy/test/bearer-forwarder.test.mjs +++ b/packages/credential-proxy/test/bearer-forwarder.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import { Readable } from "node:stream"; import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; const integration = { id: "integration-1", origin: "https://api.example.test", credential: "synthetic-token" }; @@ -12,7 +13,7 @@ test("forwards only through the granted origin and replaces unsafe routing and a connections.push({ origin, url }); return { socket: { fixture: true }, url: new URL(url) }; }, - request: async request => { calls.push(request); return { status: 201, body: Buffer.from("created") }; }, + request: async request => { calls.push(request); return { status: 201, headers: {}, body: Readable.from([Buffer.from("created")]) }; }, }); const result = await forwarder.forward(integration, { method: "post", @@ -27,7 +28,8 @@ test("forwards only through the granted origin and replaces unsafe routing and a }, body: Buffer.from('{"ok":true}'), }); - assert.deepEqual(result, { status: 201, body: Buffer.from("created") }); + assert.equal(result.status, 201); + assert.deepEqual(Buffer.concat(await Array.fromAsync(result.body)), Buffer.from("created")); assert.deepEqual(connections, [{ origin: integration.origin, url: "https://api.example.test/v1/items?source=job" }]); assert.equal(calls[0].method, "POST"); assert.equal(calls[0].headers.authorization, "Bearer synthetic-token"); diff --git a/packages/credential-proxy/test/fixtures/upstream-cert.pem b/packages/credential-proxy/test/fixtures/upstream-cert.pem new file mode 100644 index 00000000..685d5670 --- /dev/null +++ b/packages/credential-proxy/test/fixtures/upstream-cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNDCCAhygAwIBAgIUWvCDWltyenSy441ZnfAvqkjgIwUwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQYXBpLmV4YW1wbGUudGVzdDAeFw0yNjExMTExNTQwNTha +Fw0zNjExMDgxNTQwNThaMBsxGTAXBgNVBAMMEGFwaS5leGFtcGxlLnRlc3QwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDClgGTbXnob9GHbVxMt1lSmsfW +6lSfuOEBEuE5n96bKcfSsVKMRKvw3j8PyGasF+DTQTwqFlcavHzmmVkOBpQYLSz3 +bvD0RnKJ8CXhZwcRSA77BENRJQJ3TRr9WIAVrz3dgkAEl5Tv4Y+0KILRqZMx+vg1 +R0CuAzt6JbhGTvFugRAgeSJootV/O9Uu+z4SqtEbOkpj7isRUJa1uirAmKoFucIU +sa/4wTBkWIL74Z2iEMdmNM0HQ0xTsoVmFVJb+yxCoCr5rzIdPuGDyQeqWaWtju7Z +kVJjUskyYgSrycUivxAElY9aedgmyZxoBIHiwdFsSt/Nu5Gj9mOahqAfAPdjAgMB +AAGjcDBuMB0GA1UdDgQWBBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAfBgNVHSMEGDAW +gBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAPBgNVHRMBAf8EBTADAQH/MBsGA1UdEQQU +MBKCEGFwaS5leGFtcGxlLnRlc3QwDQYJKoZIhvcNAQELBQADggEBAFOW5O4OKDe8 +6MWztml7Yt9Rvn6zcDTM4xsMKhMDITAlorXuAfu51JGgFOsqiqNP9L9cpN57k5JS +5wYA9Fy+cFiiXtkAqImSvjSf1gvlzKpHCh1s0dGjIYnoStoWbE0PzVweSQ2zCQy/ +aQC11E3/CyS2+toZR+i1zffaBLjbA6T4GToOKCrXdGl8Wv7vCyTbEMA6ouEspkJO +U/glcrkekGgIZVC10di70LI+LkgqsnuDpFIALb7qDMtZQuqnIEU/wyapORpQn6Lm +WjHPEcWSHe5dlHEQWO0P3XXEA2mbr+fboVWWqQ/5gt7aBTLOVfV9rSkQqGpvriPQ +RmX+kW6AbK4= +-----END CERTIFICATE----- diff --git a/packages/credential-proxy/test/fixtures/upstream-key.pem b/packages/credential-proxy/test/fixtures/upstream-key.pem new file mode 100644 index 00000000..69f0bcaa --- /dev/null +++ b/packages/credential-proxy/test/fixtures/upstream-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDClgGTbXnob9GH +bVxMt1lSmsfW6lSfuOEBEuE5n96bKcfSsVKMRKvw3j8PyGasF+DTQTwqFlcavHzm +mVkOBpQYLSz3bvD0RnKJ8CXhZwcRSA77BENRJQJ3TRr9WIAVrz3dgkAEl5Tv4Y+0 +KILRqZMx+vg1R0CuAzt6JbhGTvFugRAgeSJootV/O9Uu+z4SqtEbOkpj7isRUJa1 +uirAmKoFucIUsa/4wTBkWIL74Z2iEMdmNM0HQ0xTsoVmFVJb+yxCoCr5rzIdPuGD +yQeqWaWtju7ZkVJjUskyYgSrycUivxAElY9aedgmyZxoBIHiwdFsSt/Nu5Gj9mOa +hqAfAPdjAgMBAAECggEAHCixGbCiD6z+2WZOERwHZRjZCpTmoiaTCSC+TyR/hlzJ +eggHNh1J9JV08sfBc+4QrwGlv0F0YX4dZRGT/Z+eFtGSzEvVusPnF2cVVEUglLOk +11aKl1ePv8v8t+TyxDCT91Y+BWwaNYyiyyOBqu2rS3RSLwOcrJHzLX3/6MzLy33I +1++WK9wGUKEyzfQ8k363yUv1CmhN/4kJs9a00OLc9pvBKO/LLxMxGZ3lMUsVhVUc +3h0yjlRvhvYDvZeLJKGhWVhn1wPfa6BNPwMNatJ6O5ubdA1qegOOxi+VDheSg05H +MVILqoFWt6B7AY+iMHbzbR1YJxSYWNVA9X9KWqtWoQKBgQD7EnZ1137IBFeC0UwK +bvjnh92lPCg3Ek8iBaH+tHga08KZr4xR10tsbqXIKxoGBeE0m3uOQbnZ0cDYSlSP +MsEXk3hthnpSNT1Ln/K1sc5ht7wb0xhDzbVsRjhtH0Ae64LrExhn/wlTirrtwDcx +6Dw1zymbr7a6LflUtHNB7V28VwKBgQDGZ7kWt7w7yDZlLQPkEmJ3BIXTvDKPsfCv +1R4Xwn/ZHaXA8J6CHZ3tEzOSWSPU7TOCSwu3mrMS+CGK5GINOYJ54YfNcgbQJ+7G +Yzw7yUFidbhJ6TBulhmCO8gdR743CRweC0K28bVe5suPnNNdlJDyowwtboua9Vn5 +H0sxcNz11QKBgQDmqxDAtYqaUzrfSlE1aL0vYqklLP65kg238Tri2oVaCGHKdvlR +G58E8SdInyjajb5Zadv7V7FbSJWHYcsMgRHcCjXjUmDBdH5rI3mGquQ07OxbF/c8 +HBtpbZ5byxFLFw8B0Fpk+XBlG5kf47ASHw5qVXkhuy8DIEfpJZya3mx3IwKBgFNN +IU0Ef6ep/AymxcdwYJ8P6xBnJ+NlFmT82dGn+/2wkPNdE3AV90SelzyO6Jhxrvq5 +f4XrEdyZCZxDH1fmGX4KzvXXNj0XRw1MhPGlIiqwultt6MPFiZhAeVix0o9Q59lF +aRFekoPx6u21LplP0zvMcURx8XNyKACMvwlXFa6xAoGAK5qaim0rJzwrtlr8T1z/ +HxzM4ZAZJchb8fN7LExj2ZrzCmZY2pnpS8oH3ae9iopV9LKUR+b6OwgnV8eWv1qx +Ldc3l7R/8AD9MOmuFwU36VozB/esox6r6MKaBl8m009wceouZG7uV4KvKr2E2Kry +dfnJ6oBHEnM8GcAQjZ3tqJA= +-----END PRIVATE KEY----- diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs new file mode 100644 index 00000000..34bfae01 --- /dev/null +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -0,0 +1,117 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { createServer as createHttpsServer } from "node:https"; +import { once } from "node:events"; +import { connect as tlsConnect } from "node:tls"; +import { test } from "node:test"; +import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; +import { createAuthenticatedProxyListener } from "../src/proxy-listener.mjs"; +import { createProxyRuntime } from "../src/runtime.mjs"; + +const fixtures = new URL("./fixtures/", import.meta.url); +const key = await readFile(new URL("upstream-key.pem", fixtures)); +const cert = await readFile(new URL("upstream-cert.pem", fixtures)); + +async function waitForEvent(emitter, event) { + const pending = once(emitter, event); + return pending; +} + +test("authenticated tunnel listener streams over TLS, confines grants and cancels upstream work", { timeout: 5_000 }, async () => { + let observedAuthorization; + let upstreamEnded = false; + let cancelUpstream; + let finishUpstream; + let cancelStartedResolve; + const cancelStarted = new Promise(resolve => { cancelStartedResolve = resolve; }); + const upstream = createHttpsServer({ key, cert }, async (request, response) => { + observedAuthorization = request.headers.authorization; + if (request.url === "/cancel") { + const closed = new Promise(resolve => request.socket.once("close", resolve)); + response.writeHead(200, { "content-type": "text/plain" }); + response.write("first chunk"); + cancelUpstream = closed; + cancelStartedResolve(); + return; + } + const received = []; + for await (const chunk of request) received.push(chunk); + assert.equal(Buffer.concat(received).toString(), "request-body"); + response.writeHead(200, { "content-type": "text/plain" }); + response.write("first chunk"); + response.once("close", () => { upstreamEnded = true; }); + await new Promise(resolve => { finishUpstream = resolve; }); + response.end(" second chunk"); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + const forwarder = createBearerForwarder({ + connectorFactory: origin => async requestUrl => { + assert.equal(origin, `https://api.example.test:${upstreamPort}`); + const url = new URL(requestUrl); + const socket = tlsConnect({ host: "127.0.0.1", port: upstreamPort, servername: url.hostname, ca: cert }); + await once(socket, "secureConnect"); + return { socket, url }; + }, + }); + const runtime = createProxyRuntime({ + listenerFactory: ({ connection }) => createAuthenticatedProxyListener({ connection, forwarder }), + }); + let listener; + try { + runtime.unlock("synthetic-unlock-key"); + listener = await runtime.acceptAuthenticatedConnection({ + authenticated: true, + enrollmentConsumed: true, + connectionId: "consumed-enrollment-1", + runId: "run-1", + vmId: "vm-1", + grants: [{ id: "tool-1", origin: `https://api.example.test:${upstreamPort}`, credential: "synthetic-bearer" }], + }); + const address = listener.address; + assert.equal(address.address, "127.0.0.1"); + const base = `http://127.0.0.1:${address.port}`; + const response = await fetch(`${base}/integrations/tool-1/echo?source=vm`, { + method: "POST", + headers: { authorization: "Bearer caller-token", "content-type": "text/plain" }, + body: "request-body", + }); + assert.equal(response.status, 200); + assert.equal(observedAuthorization, "Bearer synthetic-bearer"); + const reader = response.body.getReader(); + const first = await reader.read(); + assert.equal(Buffer.from(first.value).toString(), "first chunk"); + assert.equal(upstreamEnded, false, "response was delivered before the upstream finished"); + assert.equal(forwarder.activeRequests, 1, "streaming response retains its concurrency slot"); + finishUpstream(); + const second = await reader.read(); + assert.equal(Buffer.from(second.value).toString(), " second chunk"); + assert.equal((await reader.read()).done, true); + assert.equal(forwarder.activeRequests, 0); + + const denied = await fetch(`${base}/integrations/other-tool/echo`); + assert.equal(denied.status, 403); + assert.equal(observedAuthorization, "Bearer synthetic-bearer", "ungranted request never reached the upstream"); + + const abort = new AbortController(); + const cancelled = fetch(`${base}/integrations/tool-1/cancel`, { signal: abort.signal }); + await cancelStarted; + abort.abort(); + await assert.rejects(cancelled, error => error.name === "AbortError"); + assert.ok(cancelUpstream, "upstream cancellation hook was installed"); + await Promise.race([cancelUpstream, new Promise((_, reject) => setTimeout(() => reject(new Error("upstream was not cancelled")), 1_000))]); + assert.equal(forwarder.activeRequests, 0); + } finally { + finishUpstream?.(); + if (listener) await runtime.disconnect("consumed-enrollment-1"); + await runtime.shutdown(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); + +test("listener refuses an unconsumed or unauthenticated connection", async () => { + await assert.rejects(createAuthenticatedProxyListener({ connection: { + authenticated: true, connectionId: "connection", runId: "run", vmId: "vm", grants: [], + } }), /authenticated consumed connection/); +}); From f00b2f072268d55d27441ce13c91c432c72d3849 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 16:00:46 +0000 Subject: [PATCH 003/102] feat(credential-proxy): connect authenticated VM tunnel --- .../credential-proxy/src/tunnel-transport.mjs | 131 ++++++++++++++++++ .../test/proxy-listener.e2e.test.mjs | 66 +++++++++ 2 files changed, 197 insertions(+) create mode 100644 packages/credential-proxy/src/tunnel-transport.mjs diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs new file mode 100644 index 00000000..b3a68363 --- /dev/null +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -0,0 +1,131 @@ +import { connect as connectTls, createServer as createTlsServer } from "node:tls"; +import { connect as connectTcp, createServer as createTcpServer } from "node:net"; +import { once } from "node:events"; + +const HANDSHAKE_LIMIT = 4096; +const HANDSHAKE_TIMEOUT_MS = 10_000; + +function peerIdentity(socket) { + const certificate = socket.getPeerCertificate(); + if (!socket.authorized || !certificate?.subject?.CN) throw new Error("tunnel client certificate is not authorized"); + return certificate.subject.CN; +} + +/** + * Accept only a TLS client with a trusted VM certificate, consume its one-use + * enrollment, and bridge that stream to the connection-scoped HTTP listener. + * The bridge does not expose a shell or arbitrary destination forwarding. + */ +export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnrollment, host = "127.0.0.1", port = 0 } = {}) { + if (!tls || !runtime || typeof authenticateEnrollment !== "function") throw new TypeError("tunnel server requires TLS, runtime and enrollment authentication"); + const connections = new Set(); + const server = createTlsServer({ ...tls, requestCert: true, rejectUnauthorized: true }, (socket) => { + connections.add(socket); + let handshake = Buffer.alloc(0); + let settled = false; + const timeout = setTimeout(() => socket.destroy(new Error("tunnel enrollment timed out")), HANDSHAKE_TIMEOUT_MS); + timeout.unref(); + const reject = () => { + if (settled) return; + settled = true; + clearTimeout(timeout); + socket.destroy(); + }; + socket.on("data", async (chunk) => { + if (settled) return; + handshake = Buffer.concat([handshake, chunk]); + const end = handshake.indexOf("\n"); + if (end < 0) return handshake.length > HANDSHAKE_LIMIT ? reject() : undefined; + if (end > HANDSHAKE_LIMIT) return reject(); + settled = true; + clearTimeout(timeout); + socket.pause(); + const line = handshake.subarray(0, end).toString("utf8").replace(/\r$/, ""); + const rest = handshake.subarray(end + 1); + const match = /^FACTORIZE-TUNNEL\/1 ([A-Za-z0-9_-]{32,128})$/.exec(line); + if (!match) return reject(); + let listener; + let target; + try { + const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket) }); + if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true) throw new Error("enrollment was not consumed"); + listener = await runtime.acceptAuthenticatedConnection(connection); + const address = listener.address; + target = connectTcp(address.port, address.address); + await once(target, "connect"); + if (rest.length) target.write(rest); + socket.pipe(target); + target.pipe(socket); + socket.resume(); + const close = () => { + socket.unpipe(target); + target.unpipe(socket); + target.destroy(); + void runtime.disconnect(connection.connectionId); + }; + socket.once("close", close); + target.once("close", close); + } catch { + listener && await runtime.disconnect(listener.connectionId); + socket.destroy(); + } + }); + socket.once("close", () => { clearTimeout(timeout); connections.delete(socket); }); + socket.once("error", () => {}); + }); + server.listen(port, host); + return { + server, + listening: once(server, "listening"), + address: () => server.address(), + close: async () => { + for (const socket of connections) socket.destroy(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + }, + }; +} + +/** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ +export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPort, host = "127.0.0.1", port = 0 } = {}) { + if (!tls || typeof enrollment !== "function" || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { + throw new TypeError("VM tunnel helper requires TLS, enrollment and proxy address"); + } + const sockets = new Set(); + const server = createTcpServer((local) => { + sockets.add(local); + void (async () => { + let remote; + try { + const grant = await enrollment(); + if (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token)) throw new Error("fresh enrollment unavailable"); + remote = connectTls({ ...tls, host: proxyHost, port: proxyPort, servername: tls.servername ?? proxyHost, rejectUnauthorized: true }); + sockets.add(remote); + const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); + try { await once(remote, "secureConnect"); } + finally { clearTimeout(connectTimer); } + remote.write(`FACTORIZE-TUNNEL/1 ${grant.token}\n`); + local.pipe(remote); + remote.pipe(local); + local.resume(); + remote.once("close", () => { sockets.delete(remote); local.destroy(); }); + local.once("close", () => remote.destroy()); + } catch { + local.destroy(); + remote?.destroy(); + } + })(); + const remove = () => { sockets.delete(local); }; + local.once("close", remove); + local.once("error", remove); + }); + server.listen(port, host); + await once(server, "listening"); + const address = server.address(); + return Object.freeze({ + address, + close: async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + }, + }); +} diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 34bfae01..fadcf5bb 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -7,6 +7,7 @@ import { test } from "node:test"; import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; import { createAuthenticatedProxyListener } from "../src/proxy-listener.mjs"; import { createProxyRuntime } from "../src/runtime.mjs"; +import { createAuthenticatedTunnelServer, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; const fixtures = new URL("./fixtures/", import.meta.url); const key = await readFile(new URL("upstream-key.pem", fixtures)); @@ -115,3 +116,68 @@ test("listener refuses an unconsumed or unauthenticated connection", async () => authenticated: true, connectionId: "connection", runId: "run", vmId: "vm", grants: [], } }), /authenticated consumed connection/); }); + +test("VM helper uses mutual TLS and consumed enrollment to reach only its granted integration listener", { timeout: 5_000 }, async () => { + let observedAuthorization; + let enrollmentCount = 0; + let authenticationCount = 0; + const upstream = createHttpsServer({ key, cert }, (request, response) => { + observedAuthorization = request.headers.authorization; + response.end("transport-ok"); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + const forwarder = createBearerForwarder({ + connectorFactory: origin => async requestUrl => { + assert.equal(origin, `https://api.example.test:${upstreamPort}`); + const url = new URL(requestUrl); + const socket = tlsConnect({ host: "127.0.0.1", port: upstreamPort, servername: url.hostname, ca: cert }); + await once(socket, "secureConnect"); + return { socket, url }; + }, + }); + const runtime = createProxyRuntime({ listenerFactory: ({ connection }) => createAuthenticatedProxyListener({ connection, forwarder }) }); + const tunnel = createAuthenticatedTunnelServer({ + tls: { key, cert, ca: cert }, + runtime, + authenticateEnrollment: async ({ token, vmIdentity }) => { + authenticationCount++; + assert.equal(token, (authenticationCount === 1 ? "A" : "B").repeat(43)); + assert.equal(vmIdentity, "api.example.test"); + return { + authenticated: true, + enrollmentConsumed: true, + connectionId: "transport-enrollment-1", + runId: "run-transport-1", + vmId: vmIdentity, + grants: [{ id: "tool-1", origin: `https://api.example.test:${upstreamPort}`, credential: "synthetic-bearer" }], + }; + }, + }); + await tunnel.listening; + runtime.unlock("synthetic-unlock-key"); + const local = await startVmTunnelHelper({ + tls: { key, cert, ca: cert, servername: "api.example.test" }, + enrollment: async () => ({ token: (++enrollmentCount === 1 ? "A" : "B").repeat(43) }), + proxyHost: "127.0.0.1", + proxyPort: tunnel.address().port, + }); + try { + const response = await fetch(`http://127.0.0.1:${local.address.port}/integrations/tool-1/echo`, { + headers: { authorization: "Bearer caller-token" }, + }); + assert.equal(response.status, 200); + assert.equal(await response.text(), "transport-ok"); + assert.equal(observedAuthorization, "Bearer synthetic-bearer"); + const denied = await fetch(`http://127.0.0.1:${local.address.port}/integrations/other-tool/echo`); + assert.equal(denied.status, 403); + assert.equal(enrollmentCount, 2, "each local connection gets fresh one-use enrollment"); + assert.equal(authenticationCount, 2, "each enrollment is independently consumed by the proxy transport"); + } finally { + await local.close(); + await tunnel.close(); + await runtime.shutdown(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); From 08fddf2e7e47a51f89312a41589f26b2f940c7c3 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 16:13:28 +0000 Subject: [PATCH 004/102] Bind enrollment acceptance to proxy boot epoch --- packages/credential-proxy/src/main.mjs | 2 +- packages/credential-proxy/src/runtime.mjs | 9 +++++++- .../credential-proxy/src/tunnel-transport.mjs | 2 +- .../test/proxy-listener.e2e.test.mjs | 23 ++++++++++++++++++- .../test/runtime-process.test.mjs | 7 +++++- 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 28cd1f65..d74a19c2 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -23,7 +23,7 @@ process.once("SIGTERM", stop); process.once("SIGINT", stop); shutdownDeadline = setTimeout(() => process.exit(1), 2_000); shutdownDeadline.unref(); -respond({ type: "ready", state: runtime.state }); +respond({ type: "ready", state: runtime.state, bootEpoch: runtime.bootEpoch }); lines.on("line", async (line) => { if (closing) return; diff --git a/packages/credential-proxy/src/runtime.mjs b/packages/credential-proxy/src/runtime.mjs index 4627acb9..06cf340c 100644 --- a/packages/credential-proxy/src/runtime.mjs +++ b/packages/credential-proxy/src/runtime.mjs @@ -1,4 +1,5 @@ import { createAuthenticatedProxyListener } from "./proxy-listener.mjs"; +import { randomUUID } from "node:crypto"; /** * Process-local runtime for decrypted records and authorized tunnel listeners. @@ -6,7 +7,10 @@ import { createAuthenticatedProxyListener } from "./proxy-listener.mjs"; * authenticated its enrollment against the current run and proxy boot epoch. */ -export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyListener } = {}) { +export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyListener, bootEpoch = randomUUID() } = {}) { + if (typeof bootEpoch !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(bootEpoch)) { + throw new TypeError("proxy boot epoch must be a UUID v4"); + } let unlockMaterial; let state = "locked"; let shutdown = false; @@ -20,6 +24,8 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL } return Object.freeze({ + // A fresh process identity invalidates enrollments from every prior boot. + bootEpoch, get state() { return state; }, @@ -42,6 +48,7 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL async acceptAuthenticatedConnection(connection) { if (shutdown) throw new Error("runtime is shutting down"); if (state !== "unlocked") throw new Error("proxy is locked"); + if (connection?.bootEpoch !== bootEpoch) throw new Error("enrollment belongs to a different proxy boot"); // The transport adapter calls this only after atomically consuming and // authenticating enrollment. Grants are the consumed, current run state. const listener = await listenerFactory({ connection }); diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index b3a68363..5777b185 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -47,7 +47,7 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro let listener; let target; try { - const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket) }); + const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket), bootEpoch: runtime.bootEpoch }); if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true) throw new Error("enrollment was not consumed"); listener = await runtime.acceptAuthenticatedConnection(connection); const address = listener.address; diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index fadcf5bb..7ba9eef0 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -65,6 +65,7 @@ test("authenticated tunnel listener streams over TLS, confines grants and cancel listener = await runtime.acceptAuthenticatedConnection({ authenticated: true, enrollmentConsumed: true, + bootEpoch: runtime.bootEpoch, connectionId: "consumed-enrollment-1", runId: "run-1", vmId: "vm-1", @@ -117,6 +118,24 @@ test("listener refuses an unconsumed or unauthenticated connection", async () => } }), /authenticated consumed connection/); }); +test("runtime rejects a consumed enrollment from an earlier proxy boot", async () => { + const runtime = createProxyRuntime({ bootEpoch: "11111111-1111-4111-8111-111111111111" }); + runtime.unlock("synthetic-unlock-key"); + try { + await assert.rejects(runtime.acceptAuthenticatedConnection({ + authenticated: true, + enrollmentConsumed: true, + bootEpoch: "22222222-2222-4222-8222-222222222222", + connectionId: "stale-enrollment", + runId: "run-1", + vmId: "vm-1", + grants: [], + }), /different proxy boot/); + } finally { + await runtime.shutdown(); + } +}); + test("VM helper uses mutual TLS and consumed enrollment to reach only its granted integration listener", { timeout: 5_000 }, async () => { let observedAuthorization; let enrollmentCount = 0; @@ -141,13 +160,15 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante const tunnel = createAuthenticatedTunnelServer({ tls: { key, cert, ca: cert }, runtime, - authenticateEnrollment: async ({ token, vmIdentity }) => { + authenticateEnrollment: async ({ token, vmIdentity, bootEpoch }) => { authenticationCount++; assert.equal(token, (authenticationCount === 1 ? "A" : "B").repeat(43)); assert.equal(vmIdentity, "api.example.test"); + assert.equal(bootEpoch, runtime.bootEpoch); return { authenticated: true, enrollmentConsumed: true, + bootEpoch, connectionId: "transport-enrollment-1", runId: "run-transport-1", vmId: vmIdentity, diff --git a/packages/credential-proxy/test/runtime-process.test.mjs b/packages/credential-proxy/test/runtime-process.test.mjs index 64fb7fe7..03944bf9 100644 --- a/packages/credential-proxy/test/runtime-process.test.mjs +++ b/packages/credential-proxy/test/runtime-process.test.mjs @@ -38,9 +38,13 @@ async function startProxy() { }); child.stderr.setEncoding("utf8").on("data", (chunk) => { stderr += chunk; }); await waitFor(() => messages.length > 0, "proxy startup"); - assert.deepEqual(messages.shift(), { type: "ready", state: "locked" }); + const ready = messages.shift(); + assert.equal(ready.type, "ready"); + assert.equal(ready.state, "locked"); + assert.match(ready.bootEpoch, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); return { child, + bootEpoch: ready.bootEpoch, get logs() { return `${log}${stderr}`; }, command(command) { child.stdin.write(`${JSON.stringify(command)}\n`); @@ -65,6 +69,7 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts const restarted = await startProxy(); processes.push(restarted); + assert.notEqual(restarted.bootEpoch, first.bootEpoch, "every process restart gets a fresh enrollment epoch"); assert.deepEqual(await restarted.command({ type: "status" }), { type: "status", state: "locked" }); assert.deepEqual(await restarted.command({ type: "unlock", key: tokenMarker }), { type: "status", state: "unlocked" }); restarted.child.stdin.write('{"type":"stop"}\n'); From adfefc59b76a75f6a7ec683ffd708cda80149cea Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 16:52:22 +0000 Subject: [PATCH 005/102] Bind tunnel enrollment to authenticated VM peer --- .../credential-proxy/src/tunnel-transport.mjs | 10 +++++++++- .../test/proxy-listener.e2e.test.mjs | 20 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index 5777b185..2ad21a67 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -48,7 +48,15 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro let target; try { const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket), bootEpoch: runtime.bootEpoch }); - if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true) throw new Error("enrollment was not consumed"); + // The production control plane returns the consumed enrollment binding. + // Compare its VM and boot identities with the authenticated TLS peer and + // this process so an adapter cannot accidentally attach another run's + // grants to the tunnel. + if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true + || connection.vmIdentity !== peerIdentity(socket) || connection.bootEpoch !== runtime.bootEpoch + || typeof connection.connectionId !== "string" || connection.connectionId.length === 0) { + throw new Error("enrollment binding does not match the authenticated tunnel"); + } listener = await runtime.acceptAuthenticatedConnection(connection); const address = listener.address; target = connectTcp(address.port, address.address); diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 7ba9eef0..c8c6a940 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -162,12 +162,13 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante runtime, authenticateEnrollment: async ({ token, vmIdentity, bootEpoch }) => { authenticationCount++; - assert.equal(token, (authenticationCount === 1 ? "A" : "B").repeat(43)); + assert.equal(token, (authenticationCount === 1 ? "A" : authenticationCount === 2 ? "B" : "C").repeat(43)); assert.equal(vmIdentity, "api.example.test"); assert.equal(bootEpoch, runtime.bootEpoch); return { authenticated: true, enrollmentConsumed: true, + vmIdentity: token === "C".repeat(43) ? "other-vm" : vmIdentity, bootEpoch, connectionId: "transport-enrollment-1", runId: "run-transport-1", @@ -180,10 +181,11 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante runtime.unlock("synthetic-unlock-key"); const local = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, - enrollment: async () => ({ token: (++enrollmentCount === 1 ? "A" : "B").repeat(43) }), + enrollment: async () => ({ token: (++enrollmentCount === 1 ? "A" : enrollmentCount === 2 ? "B" : "C").repeat(43) }), proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, }); + let mismatchLocal; try { const response = await fetch(`http://127.0.0.1:${local.address.port}/integrations/tool-1/echo`, { headers: { authorization: "Bearer caller-token" }, @@ -195,7 +197,21 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante assert.equal(denied.status, 403); assert.equal(enrollmentCount, 2, "each local connection gets fresh one-use enrollment"); assert.equal(authenticationCount, 2, "each enrollment is independently consumed by the proxy transport"); + mismatchLocal = await startVmTunnelHelper({ + tls: { key, cert, ca: cert, servername: "api.example.test" }, + enrollment: async () => ({ token: (++enrollmentCount === 3 ? "C" : "D").repeat(43) }), + proxyHost: "127.0.0.1", + proxyPort: tunnel.address().port, + }); + await assert.rejects( + fetch(`http://127.0.0.1:${mismatchLocal.address.port}/integrations/tool-1/echo`), + /fetch failed/, + "mismatched VM binding is rejected", + ); + assert.equal(enrollmentCount, 3); + assert.equal(authenticationCount, 3); } finally { + await mismatchLocal?.close(); await local.close(); await tunnel.close(); await runtime.shutdown(); From 5b44328b5ceae8697de9fe8fff46b11d1fec51c7 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 17:53:46 +0000 Subject: [PATCH 006/102] Add authenticated proxy boot registration --- .../0036_proxy_control_credentials.sql | 10 ++++ .../src/postgres/proxy-control-repository.ts | 56 ++++++++++++++++++ packages/api/src/postgres/schema.ts | 3 +- packages/api/src/proxy-control.ts | 38 ++++++++++++ packages/api/src/worker.ts | 2 + .../api/test/proxy-control-repository.test.ts | 51 ++++++++++++++++ packages/api/test/proxy-control.test.ts | 58 +++++++++++++++++++ .../credential-proxy/src/control-client.mjs | 25 ++++++++ packages/credential-proxy/src/main.mjs | 10 ++++ .../test/control-client.test.mjs | 25 ++++++++ 10 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 packages/api/migrations/0036_proxy_control_credentials.sql create mode 100644 packages/api/src/postgres/proxy-control-repository.ts create mode 100644 packages/api/src/proxy-control.ts create mode 100644 packages/api/test/proxy-control-repository.test.ts create mode 100644 packages/api/test/proxy-control.test.ts create mode 100644 packages/credential-proxy/src/control-client.mjs create mode 100644 packages/credential-proxy/test/control-client.test.mjs diff --git a/packages/api/migrations/0036_proxy_control_credentials.sql b/packages/api/migrations/0036_proxy_control_credentials.sql new file mode 100644 index 00000000..6c7d8b38 --- /dev/null +++ b/packages/api/migrations/0036_proxy_control_credentials.sql @@ -0,0 +1,10 @@ +CREATE TABLE app.credential_proxy_control_credentials ( + tenant_id uuid NOT NULL REFERENCES app.tenants(id) ON DELETE CASCADE, + proxy_identity text NOT NULL, + boot_epoch uuid NOT NULL, + token_digest bytea NOT NULL UNIQUE, + revoked_at timestamptz, + consumed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (tenant_id, proxy_identity) +); diff --git a/packages/api/src/postgres/proxy-control-repository.ts b/packages/api/src/postgres/proxy-control-repository.ts new file mode 100644 index 00000000..2dee38ae --- /dev/null +++ b/packages/api/src/postgres/proxy-control-repository.ts @@ -0,0 +1,56 @@ +import type { Database } from "./database"; + +export type RegisteredProxy = { tenantId: string; proxyIdentity: string; bootEpoch: string }; +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const digest = async (token: string) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(token))); + +/** Credentials for the Factorize-to-proxy control plane. These are never job run credentials. */ +export class ProxyControlRepository { + constructor(private database: Database) {} + + /** Called only from the authenticated infrastructure provisioning path; the token is returned once. */ + async provisionCredential(tenantId: string, proxyIdentity: string, bootEpoch: string): Promise { + if (!uuid.test(tenantId) || !proxyIdentity.trim() || proxyIdentity.length > 255 || !uuid.test(bootEpoch)) throw new TypeError("Invalid proxy identity or boot epoch"); + const token = `fzc_${Buffer.from(crypto.getRandomValues(new Uint8Array(32))).toString("base64url")}`; + await this.database.pool.query(`INSERT INTO app.credential_proxy_control_credentials(tenant_id,proxy_identity,boot_epoch,token_digest,revoked_at,consumed_at) + VALUES ($1,$2,$3,$4,NULL,NULL) ON CONFLICT (tenant_id,proxy_identity) DO UPDATE SET boot_epoch=excluded.boot_epoch,token_digest=excluded.token_digest,revoked_at=NULL,consumed_at=NULL,created_at=now()`, + [tenantId, proxyIdentity, bootEpoch, Buffer.from(await digest(token))]); + return token; + } + + async authenticate(token: string): Promise | null> { + if (typeof token !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return null; + const result = await this.database.pool.query<{ tenant_id: string; proxy_identity: string }>(`SELECT tenant_id,proxy_identity + FROM app.credential_proxy_control_credentials WHERE token_digest=$1 AND revoked_at IS NULL`, [Buffer.from(await digest(token))]); + const row = result.rows[0]; + return row ? { tenantId: row.tenant_id, proxyIdentity: row.proxy_identity } : null; + } + + async registerBoot(token: string, bootEpoch: string): Promise { + if (!uuid.test(bootEpoch)) throw new TypeError("bootEpoch must be a UUID"); + const tokenDigest = Buffer.from(await digest(token)); + return this.database.transaction(async client => { + const credential = await client.query<{ tenant_id: string; proxy_identity: string }>(`SELECT tenant_id,proxy_identity + FROM app.credential_proxy_control_credentials WHERE token_digest=$1 AND boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL FOR UPDATE`, [tokenDigest, bootEpoch]); + const identity = credential.rows[0]; + if (!identity) return null; + const saved = await client.query(`INSERT INTO app.credential_proxy_states(tenant_id,proxy_identity,boot_epoch) + VALUES ($1,$2,$3) ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity,boot_epoch=excluded.boot_epoch,updated_at=now() + WHERE app.credential_proxy_states.proxy_identity=excluded.proxy_identity + AND app.credential_proxy_states.disabled_at IS NULL AND app.credential_proxy_states.revoked_at IS NULL RETURNING tenant_id`, + [identity.tenant_id, identity.proxy_identity, bootEpoch]); + if (!saved.rowCount) return null; + await client.query(`UPDATE app.credential_proxy_control_credentials SET consumed_at=now() + WHERE token_digest=$1 AND boot_epoch=$2 AND consumed_at IS NULL`, [tokenDigest, bootEpoch]); + await client.query(`UPDATE app.credential_run_grants SET revoked_at=now() + WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch<>$3 AND revoked_at IS NULL`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); + return { tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch }; + }); + } + + async revokeCredential(tenantId: string, proxyIdentity: string): Promise { + const result = await this.database.pool.query(`UPDATE app.credential_proxy_control_credentials SET revoked_at=now() + WHERE tenant_id=$1 AND proxy_identity=$2 AND revoked_at IS NULL`, [tenantId, proxyIdentity]); + return Boolean(result.rowCount); + } +} diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 5bb398ab..4d5e9d50 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -53,6 +53,7 @@ export const integrationChanges = app.table("integration_changes", { export const credentialProxyStates = app.table("credential_proxy_states", { tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), disabledAt: timestamp("disabled_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), updatedAt: updatedAt(), }); +export const credentialProxyControlCredentials = app.table("credential_proxy_control_credentials", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), tokenDigest: bytea("token_digest").notNull().unique(), revokedAt: timestamp("revoked_at", { withTimezone: true }), consumedAt: timestamp("consumed_at", { withTimezone: true }), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.proxyIdentity] })]); export const credentialRunGrants = app.table("credential_run_grants", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), vmIdentity: text("vm_identity").notNull(), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), grantRevision: bigint("grant_revision", { mode: "number" }).notNull(), revokedAt: timestamp("revoked_at", { withTimezone: true }), createdAt: createdAt(), }, t => [primaryKey({ columns: [t.tenantId, t.runId] }), foreignKey({ columns: [t.tenantId, t.runId], foreignColumns: [jobRuns.tenantId, jobRuns.id] }).onDelete("cascade"), check("credential_run_grants_revision_check", sql`${t.grantRevision} > 0`)]); @@ -155,7 +156,7 @@ export const runRelations = relations(runs, ({ one, many }) => ({ export const databaseSchema = { schemaMigrations, tenants, oauthConnections, oauthConsentPreviews, authUsers, authAccounts, authResetTokens, authSessions, oauthDeviceAuthorizations, members, accessTokens, - connections, integrationCollections, integrationRecords, integrationChanges, credentialProxyStates, credentialRunGrants, credentialRunEnrollments, linearWorkspaces, githubInstallations, githubSetupStates, projects, jobs, vmImageVerifications, skills, jobSkills, triggers, webhookConditionsCutover, invocations, manualInvocationPermissions, jobRuns, runCredentials, runs, runArtifacts, + connections, integrationCollections, integrationRecords, integrationChanges, credentialProxyStates, credentialProxyControlCredentials, credentialRunGrants, credentialRunEnrollments, linearWorkspaces, githubInstallations, githubSetupStates, projects, jobs, vmImageVerifications, skills, jobSkills, triggers, webhookConditionsCutover, invocations, manualInvocationPermissions, jobRuns, runCredentials, runs, runArtifacts, runTraceEvents, runActivity, activeClaims, scheduleState, automaticWakes, lifecycleDeliveries, lifecycleEvents, jobEditDeliveries, jobEvents, webhookDeliveries, webhookDeliveryEvents, pendingVerifications, tailFingerprints, wakeHints, runTraceCursors, runTraceChunks, runTraceProjections, traceReplayLimits, traceReplayOperations, diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts new file mode 100644 index 00000000..3cd1632b --- /dev/null +++ b/packages/api/src/proxy-control.ts @@ -0,0 +1,38 @@ +import { databaseFor } from "./postgres/database"; +import { ProxyControlRepository } from "./postgres/proxy-control-repository"; + +const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "Cache-Control": "no-store" } }); + +/** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ +export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { + const url = new URL(request.url); + if (url.pathname !== "/internal/credential-proxy/register-boot") return json({ error: "not_found" }, 404); + if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); + if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); + const authorization = request.headers.get("authorization") ?? ""; + const token = authorization.startsWith("Bearer ") ? authorization.slice(7) : ""; + if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return json({ error: "unauthorized" }, 401); + const declaredLength = Number(request.headers.get("content-length") ?? "0"); + if (declaredLength > 1024) return json({ error: "invalid_request" }, 413); + let body: unknown; + try { + const reader = request.body?.getReader(); + const chunks: Uint8Array[] = []; let length = 0; + if (reader) for (;;) { + const { done, value } = await reader.read(); if (done) break; + length += value.byteLength; + if (length > 1024) { await reader.cancel(); return json({ error: "invalid_request" }, 413); } + chunks.push(value); + } + const bytes = new Uint8Array(length); let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + body = JSON.parse(new TextDecoder().decode(bytes)); + } catch { return json({ error: "invalid_request" }, 400); } + if (!body || typeof body !== "object" || Array.isArray(body) + || Object.keys(body).some(key => key !== "bootEpoch") + || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); + const repository = new ProxyControlRepository(databaseFor(env)); + const registration = await repository.registerBoot(token, (body as any).bootEpoch); + if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); + return json({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); +} diff --git a/packages/api/src/worker.ts b/packages/api/src/worker.ts index d7868347..e32c339b 100644 --- a/packages/api/src/worker.ts +++ b/packages/api/src/worker.ts @@ -14,6 +14,7 @@ import { artifactUpload } from "./artifact-upload"; import { traceChunkUpload } from "./trace-chunk-upload"; import { databaseFor } from "./postgres/database"; import { IdentityRepository } from "./postgres/identity-repository"; +import { proxyControlFetch } from "./proxy-control"; const scopes = [...ACCESS_SCOPES]; // Keep accepting grants issued by the pre-resource-scope protocol. New @@ -85,6 +86,7 @@ function provider(env: Env) { return new OAuthProvider(providerOptions(env) export { AlarmCoordinator } from "./alarm-coordinator"; const backend = { async fetch(request: Request, env: Env, ctx: ExecutionContext) { + if (new URL(request.url).pathname.startsWith("/internal/credential-proxy/")) return proxyControlFetch(request, env); const staticResponse = !["/authorize", "/device"].includes(new URL(request.url).pathname) ? await staticApp(request, env) : null; if (staticResponse) return staticResponse; const oauth = provider(env); diff --git a/packages/api/test/proxy-control-repository.test.ts b/packages/api/test/proxy-control-repository.test.ts new file mode 100644 index 00000000..20f1934c --- /dev/null +++ b/packages/api/test/proxy-control-repository.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from "vitest"; +import { ProxyControlRepository } from "../src/postgres/proxy-control-repository"; + +function database(query: ReturnType, transactionQuery = query) { + return { pool: { query }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) } as any; +} + +describe("dedicated credential proxy control identity", () => { + it("issues a scoped one-time secret and stores only its digest", async () => { + const query = vi.fn().mockResolvedValue({ rows: [], rowCount: 1 }); + const repository = new ProxyControlRepository(database(query)); + const token = await repository.provisionCredential("11111111-1111-4111-8111-111111111111", "proxy-a", "22222222-2222-4222-8222-222222222222"); + expect(token).toMatch(/^fzc_[A-Za-z0-9_-]{43}$/); + const stored = query.mock.calls[0][1][3] as Buffer; + expect(stored).toBeInstanceOf(Buffer); + expect(stored.toString()).not.toContain(token); + expect(stored.byteLength).toBe(32); + }); + + it("authenticates only a live credential and never accepts a guest run token", async () => { + const query = vi.fn().mockResolvedValue({ rows: [{ tenant_id: "tenant-a", proxy_identity: "proxy-a" }], rowCount: 1 }); + const repository = new ProxyControlRepository(database(query)); + expect(await repository.authenticate("fzt_run_" + "a".repeat(43))).toBeNull(); + expect(query).not.toHaveBeenCalled(); + expect(await repository.authenticate("fzc_" + "a".repeat(43))).toEqual({ tenantId: "tenant-a", proxyIdentity: "proxy-a" }); + }); + + it("registers a fresh boot for its credential identity and rejects revoked, disabled, or mismatched identities", async () => { + const identity = { tenant_id: "tenant-a", proxy_identity: "proxy-a" }; + const credentialQuery = vi.fn().mockResolvedValue({ rows: [identity], rowCount: 1 }); + const stateQuery = vi.fn().mockResolvedValue({ rows: [{ tenant_id: identity.tenant_id }], rowCount: 1 }); + const repository = new ProxyControlRepository(database(vi.fn(), vi.fn())); + const tx = { query: vi.fn().mockImplementation((sql: string) => sql.includes("credential_proxy_control_credentials") ? credentialQuery() : sql.includes("INSERT INTO app.credential_proxy_states") ? stateQuery() : Promise.resolve({ rows: [], rowCount: 1 })) }; + (repository as any).database = { transaction: (work: (client: any) => Promise) => work(tx) }; + const epoch = "22222222-2222-4222-8222-222222222222"; + expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch)).toEqual({ tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch: epoch }); + expect(tx.query.mock.calls.map(call => call[0])).toHaveLength(4); + expect(tx.query.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); + expect(tx.query.mock.calls[2][0]).toContain("consumed_at=now()"); + expect(tx.query.mock.calls[3][0]).toContain("boot_epoch<>$3"); + credentialQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 }); + expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch)).toBeNull(); + }); + + it("rejects malformed boot epochs before opening a transaction", async () => { + const transaction = vi.fn(); + const repository = new ProxyControlRepository({ transaction, pool: { query: vi.fn() } } as any); + await expect(repository.registerBoot("fzc_" + "a".repeat(43), "old-boot")).rejects.toThrow("bootEpoch must be a UUID"); + expect(transaction).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts new file mode 100644 index 00000000..734cfff4 --- /dev/null +++ b/packages/api/test/proxy-control.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi } from "vitest"; +import { proxyControlFetch } from "../src/proxy-control"; + +const token = `fzc_${"A".repeat(43)}`; +const bootEpoch = "22222222-2222-4222-8222-222222222222"; + +function envFor({ credentialFound = true, stateSaved = true } = {}) { + const transactionQuery = vi.fn(async (sql: string) => { + if (sql.includes("credential_proxy_control_credentials")) return { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture" }] : [], rowCount: credentialFound ? 1 : 0 }; + if (sql.includes("INSERT INTO app.credential_proxy_states")) return { rows: stateSaved ? [{ tenant_id: "11111111-1111-4111-8111-111111111111" }] : [], rowCount: stateSaved ? 1 : 0 }; + return { rows: [], rowCount: 1 }; + }); + return { DATABASE: { pool: { query: vi.fn() }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery } as any; +} + +function request({ authorization, body = { bootEpoch }, method = "POST" }: { authorization?: string; body?: unknown; method?: string } = {}) { + return new Request("https://factorize.example/internal/credential-proxy/register-boot", { method, headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}) }, body: method === "POST" ? JSON.stringify(body) : undefined }); +} + +describe("dedicated proxy boot registration endpoint", () => { + it("binds registration to the proxy identity owned by its distinct control credential", async () => { + const env = envFor(); + const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch }); + expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(4); + expect(env.transactionQuery.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); + expect(env.transactionQuery.mock.calls[2][0]).toContain("consumed_at=now()"); + }); + + it("rejects guest job tokens before database access", async () => { + const env = envFor(); + const response = await proxyControlFetch(request({ authorization: `Bearer fzt_run_${"B".repeat(32)}` }), env); + expect(response.status).toBe(401); + expect(env.transactionQuery).not.toHaveBeenCalled(); + }); + + it("rejects revoked credentials, disabled proxy identities and tenant or proxy substitution", async () => { + for (const env of [envFor({ credentialFound: false }), envFor({ stateSaved: false })]) { + const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); + expect(response.status).toBe(403); + } + }); + + it("validates the endpoint, method and exact boot payload", async () => { + const env = envFor(); + expect((await proxyControlFetch(new Request("https://factorize.example/internal/credential-proxy/register-boot", { method: "GET" }), env)).status).toBe(405); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, body: { bootEpoch, tenantId: "attacker-tenant" } }), env)).status).toBe(400); + expect(env.transactionQuery).not.toHaveBeenCalled(); + }); + + it("bounds registration payloads before parsing", async () => { + const env = envFor(); + const response = await proxyControlFetch(request({ authorization: `Bearer ${token}`, body: { bootEpoch, padding: "x".repeat(2_000) } }), env); + expect(response.status).toBe(413); + expect(env.transactionQuery).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs new file mode 100644 index 00000000..77a430e2 --- /dev/null +++ b/packages/credential-proxy/src/control-client.mjs @@ -0,0 +1,25 @@ +const bootEpochPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +/** Register one proxy process boot using a provisioning-scoped credential. The token is only held in this call's memory. */ +export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetchImpl = fetch }) { + const origin = new URL(controlOrigin); + if (origin.protocol !== "https:" || origin.pathname !== "/" || origin.username || origin.password || origin.search || origin.hash) throw new TypeError("proxy control origin must be an HTTPS origin"); + if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token ?? "")) throw new TypeError("invalid proxy control credential"); + if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); + let credential = token; + try { + const response = await fetchImpl(new URL("/internal/credential-proxy/register-boot", origin), { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${credential}` }, + body: JSON.stringify({ bootEpoch }), + signal: AbortSignal.timeout(10_000), + redirect: "error", + }); + if (!response.ok) throw new Error("proxy boot registration rejected"); + const result = await response.json(); + if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string") throw new Error("invalid proxy boot registration response"); + return { proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch }; + } finally { + credential = ""; + } +} diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index d74a19c2..f5409b16 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -1,5 +1,6 @@ import { createInterface } from "node:readline"; import { createProxyRuntime } from "./runtime.mjs"; +import { registerProxyBoot } from "./control-client.mjs"; const runtime = createProxyRuntime(); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); @@ -37,6 +38,15 @@ lines.on("line", async (line) => { } else if (command?.type === "lock") { await runtime.lock(); respond({ type: "status", state: runtime.state }); + } else if (command?.type === "register-boot") { + let credential = command.credential; + command.credential = undefined; + try { + const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + respond({ type: "boot-registered", ...registration }); + } catch { + respond({ type: "error", error: "proxy boot registration failed" }); + } finally { credential = ""; } } else if (command?.type === "stop") stop(); else respond({ type: "error", error: "unsupported command" }); } catch { diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs new file mode 100644 index 00000000..7f59aba7 --- /dev/null +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { registerProxyBoot } from "../src/control-client.mjs"; + +const token = `fzc_${"A".repeat(43)}`; +const bootEpoch = "22222222-2222-4222-8222-222222222222"; + +test("boot registration uses a one-call control credential, HTTPS, exact path and fresh process epoch", async () => { + let request; + const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async (url, init) => { + request = { url: String(url), init }; + return Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch }); + } }); + assert.deepEqual(result, { proxyIdentity: "proxy-fixture", bootEpoch }); + assert.equal(request.url, "https://factorize.example/internal/credential-proxy/register-boot"); + assert.equal(request.init.headers.Authorization, `Bearer ${token}`); + assert.equal(request.init.redirect, "error"); + assert.deepEqual(JSON.parse(request.init.body), { bootEpoch }); +}); + +test("boot registration rejects unsafe origins, invalid credentials and mismatched response bindings", async () => { + await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch }), /HTTPS origin/); + await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); + await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async () => Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch: "33333333-3333-4333-8333-333333333333" }) }), /invalid proxy boot registration response/); +}); From 1ed566e897337b2a1e89cba74dff12ce78b517c7 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 18:15:56 +0000 Subject: [PATCH 007/102] Add secure proxy bootstrap provisioning boundary --- packages/api/src/proxy-provisioner.ts | 37 +++++++++++++++++ packages/api/test/proxy-provisioner.test.ts | 45 +++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 packages/api/src/proxy-provisioner.ts create mode 100644 packages/api/test/proxy-provisioner.test.ts diff --git a/packages/api/src/proxy-provisioner.ts b/packages/api/src/proxy-provisioner.ts new file mode 100644 index 00000000..f6d4957a --- /dev/null +++ b/packages/api/src/proxy-provisioner.ts @@ -0,0 +1,37 @@ +import type { ProxyControlRepository } from "./postgres/proxy-control-repository"; + +export type ProxyBootstrapProvisioning = { + tenantId: string; + proxyIdentity: string; + /** A new UUID for every process boot. */ + bootEpoch: string; + controlOrigin: string; +}; + +/** + * Connects one-use bootstrap credential issuance to the infrastructure + * provisioner's private control channel. Implementations must deliver the + * credential directly to proxy stdin (or an equally short-lived memory-only + * channel), never through Machine config, environment, metadata, disk, or logs. + */ +export async function provisionCredentialProxy( + input: ProxyBootstrapProvisioning, + dependencies: { + control: Pick; + startProxy: (request: ProxyBootstrapProvisioning & { credential: string }) => Promise; + }, +): Promise<{ proxyIdentity: string; bootEpoch: string }> { + const origin = new URL(input.controlOrigin); + if (origin.protocol !== "https:" || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) { + throw new TypeError("controlOrigin must be an HTTPS origin"); + } + const credential = await dependencies.control.provisionCredential(input.tenantId, input.proxyIdentity, input.bootEpoch); + try { + await dependencies.startProxy({ ...input, controlOrigin: origin.origin, credential }); + } catch (error) { + // A failed/ambiguous start must not leave a valid bootstrap credential. + await dependencies.control.revokeCredential(input.tenantId, input.proxyIdentity).catch(() => false); + throw error; + } + return { proxyIdentity: input.proxyIdentity, bootEpoch: input.bootEpoch }; +} diff --git a/packages/api/test/proxy-provisioner.test.ts b/packages/api/test/proxy-provisioner.test.ts new file mode 100644 index 00000000..1a107b53 --- /dev/null +++ b/packages/api/test/proxy-provisioner.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it, vi } from "vitest"; +import { provisionCredentialProxy } from "../src/proxy-provisioner"; + +const input = { + tenantId: "11111111-1111-4111-8111-111111111111", + proxyIdentity: "fly-app/proxy-machine", + bootEpoch: "22222222-2222-4222-8222-222222222222", + controlOrigin: "https://factorize.example/", +}; + +describe("credential proxy bootstrap provisioning", () => { + it("issues an epoch-bound credential only to the provisioner's private start channel", async () => { + const control = { + provisionCredential: vi.fn().mockResolvedValue("fzc_one-use-secret"), + revokeCredential: vi.fn().mockResolvedValue(true), + }; + const startProxy = vi.fn().mockResolvedValue(undefined); + + await expect(provisionCredentialProxy(input, { control, startProxy })).resolves.toEqual({ + proxyIdentity: input.proxyIdentity, + bootEpoch: input.bootEpoch, + }); + + expect(control.provisionCredential).toHaveBeenCalledWith(input.tenantId, input.proxyIdentity, input.bootEpoch); + expect(startProxy).toHaveBeenCalledWith({ ...input, controlOrigin: "https://factorize.example", credential: "fzc_one-use-secret" }); + expect(control.revokeCredential).not.toHaveBeenCalled(); + }); + + it("revokes the one-use credential if proxy startup fails", async () => { + const control = { + provisionCredential: vi.fn().mockResolvedValue("fzc_one-use-secret"), + revokeCredential: vi.fn().mockResolvedValue(true), + }; + const failure = new Error("startup failed"); + await expect(provisionCredentialProxy(input, { control, startProxy: vi.fn().mockRejectedValue(failure) })).rejects.toBe(failure); + expect(control.revokeCredential).toHaveBeenCalledWith(input.tenantId, input.proxyIdentity); + }); + + it("rejects non-HTTPS or non-origin control URLs before issuing a credential", async () => { + const control = { provisionCredential: vi.fn(), revokeCredential: vi.fn() }; + await expect(provisionCredentialProxy({ ...input, controlOrigin: "http://factorize.example" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); + await expect(provisionCredentialProxy({ ...input, controlOrigin: "https://factorize.example/path" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); + expect(control.provisionCredential).not.toHaveBeenCalled(); + }); +}); From edf98ea8ea826727381f5b2f92c53cd3d2dfbf91 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 18:37:32 +0000 Subject: [PATCH 008/102] Provision credential proxy with isolated bootstrap --- .../src/fly-credential-proxy-provisioner.ts | 134 ++++++++++++++++++ .../fly-credential-proxy-provisioner.test.ts | 93 ++++++++++++ .../src/bootstrap-control.mjs | 67 +++++++++ packages/credential-proxy/src/main.mjs | 8 +- packages/credential-proxy/src/startup.mjs | 46 ++++++ .../test/bootstrap-control.test.mjs | 56 ++++++++ .../test/runtime-process.test.mjs | 12 +- 7 files changed, 409 insertions(+), 7 deletions(-) create mode 100644 packages/api/src/fly-credential-proxy-provisioner.ts create mode 100644 packages/api/test/fly-credential-proxy-provisioner.test.ts create mode 100644 packages/credential-proxy/src/bootstrap-control.mjs create mode 100644 packages/credential-proxy/src/startup.mjs create mode 100644 packages/credential-proxy/test/bootstrap-control.test.mjs diff --git a/packages/api/src/fly-credential-proxy-provisioner.ts b/packages/api/src/fly-credential-proxy-provisioner.ts new file mode 100644 index 00000000..ad8751d4 --- /dev/null +++ b/packages/api/src/fly-credential-proxy-provisioner.ts @@ -0,0 +1,134 @@ +import { provisionCredentialProxy, type ProxyBootstrapProvisioning } from "./proxy-provisioner"; +import type { ProxyControlRepository } from "./postgres/proxy-control-repository"; +import type { FlyMachine, FlyMachineConfig } from "./fly-machines-client"; +import { createFlyGuestTransport } from "./fly-guest-transport"; + +const IMAGE_DIGEST = /@sha256:[a-f0-9]{64}$/i; +const UUID_V4 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +export type FlyCredentialProxyConfig = { + tenantId: string; + app: string; + /** Only reviewed credential-proxy images pinned by digest are accepted. */ + image: string; + region: string; + size: string; + memoryMb?: number; + controlOrigin: string; + /** Stable across retries of one provisioning operation. */ + operationId: string; +}; + +export type FlyCredentialProxyMachineClient = { + createMachine(app: string, config: FlyMachineConfig, operationId: string): Promise; + startMachine(app: string, machine: string): Promise; + destroyMachine(app: string, machine: string): Promise; +}; + +export type FlyCredentialProxyGuest = { + /** The reviewed image startup command consumes this payload from stdin only. */ + startProxy(input: ProxyBootstrapProvisioning & { credential: string }): Promise; +}; + +/** Provider exec adapter: the bootstrap record is the command's stdin payload. */ +export function createFlyCredentialProxyGuest(options: { + app: string; + machine: string; + token: string; + cliPath?: string; + spawnProcess?: typeof import("node:child_process").spawn; +}): FlyCredentialProxyGuest { + const transport = createFlyGuestTransport({ ...options, timeoutMs: 30_000, outputLimitBytes: 16 * 1024 }); + return { + async startProxy(input) { + const result = await transport.run(["/usr/local/bin/factorize-proxy-startup"], `${JSON.stringify({ + controlOrigin: input.controlOrigin, + bootEpoch: input.bootEpoch, + credential: input.credential, + })}\n`); + if (!result.ok) throw new Error("Credential proxy startup command failed"); + if (result.stdout.trim() !== "Credential proxy started") throw new Error("Credential proxy startup was not confirmed"); + }, + }; +} + +export type FlyCredentialProxyProvisionerOptions = { + control: Pick; + machines: FlyCredentialProxyMachineClient; + guest: (machine: FlyMachine) => FlyCredentialProxyGuest; + /** Injectable to keep boot identity deterministic in boundary tests. */ + newBootEpoch?: () => string; +}; + +function validate(input: FlyCredentialProxyConfig): void { + if (!input.tenantId || !input.app || !input.operationId || input.operationId.length > 200) throw new TypeError("Invalid credential proxy provisioning identity"); + if (!IMAGE_DIGEST.test(input.image)) throw new TypeError("Credential proxy image must be pinned by sha256 digest"); + if (!/^[a-z0-9]{3}$/.test(input.region)) throw new TypeError("Invalid credential proxy region"); + if (!/^[a-z0-9-]{1,64}$/.test(input.size)) throw new TypeError("Invalid credential proxy size"); + if (input.memoryMb !== undefined && (!Number.isInteger(input.memoryMb) || input.memoryMb < 256 || input.memoryMb > 8192)) throw new TypeError("Credential proxy memory must be between 256 and 8192 MiB"); +} + +/** + * Provisions one dedicated Fly Machine for the credential proxy and wires its + * startup through the one-use bootstrap boundary. The bootstrap credential is + * never part of the Machine config; the guest adapter must deliver it over its + * private stdin startup channel. + */ +export function createFlyCredentialProxyProvisioner(options: FlyCredentialProxyProvisionerOptions) { + return { + async start(input: FlyCredentialProxyConfig): Promise<{ proxyIdentity: string; bootEpoch: string; machine: FlyMachine }> { + validate(input); + const origin = new URL(input.controlOrigin); + if (origin.protocol !== "https:" || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) { + throw new TypeError("controlOrigin must be an HTTPS origin"); + } + const candidateBootEpoch = (options.newBootEpoch ?? (() => crypto.randomUUID()))(); + if (!UUID_V4.test(candidateBootEpoch)) throw new TypeError("boot epoch must be a UUID v4"); + const config: FlyMachineConfig = { + image: input.image, + region: input.region, + cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", candidateBootEpoch], + guest: { cpu_kind: "shared", cpus: 1, memory_mb: input.memoryMb ?? 512 }, + metadata: { + factorize_role: "credential-proxy", + factorize_tenant_id: input.tenantId, + factorize_operation_id: input.operationId, + factorize_boot_epoch: candidateBootEpoch, + }, + // The proxy must be explicitly started and must not be restarted after + // process failure with a stale one-use bootstrap credential. + auto_start: false, + auto_stop: false, + restart: { policy: "no" }, + }; + const machine = await options.machines.createMachine(input.app, config, input.operationId); + if (!machine.id) throw new Error("Fly returned a credential proxy Machine without an id"); + const bootEpoch = machine.config?.metadata?.factorize_boot_epoch ?? candidateBootEpoch; + if (!UUID_V4.test(bootEpoch)) throw new Error("Fly returned a credential proxy Machine with an invalid boot epoch"); + const proxyIdentity = `fly:${input.app}:${machine.id}`; + const bootstrap = { + tenantId: input.tenantId, + proxyIdentity, + bootEpoch, + controlOrigin: origin.origin, + }; + try { + await provisionCredentialProxy(bootstrap, { + control: options.control, + startProxy: async (request) => { + // Fly Machine startup is explicit. Machine config and metadata only + // hold public identity/configuration; the token crosses stdin. + await options.machines.startMachine(input.app, machine.id); + await options.guest(machine).startProxy(request); + }, + }); + } catch (error) { + // Preserve the sanitized original provisioning failure while making a + // best-effort cleanup of a machine that never completed startup. + await options.machines.destroyMachine(input.app, machine.id).catch(() => undefined); + throw error; + } + return { proxyIdentity, bootEpoch, machine }; + }, + }; +} diff --git a/packages/api/test/fly-credential-proxy-provisioner.test.ts b/packages/api/test/fly-credential-proxy-provisioner.test.ts new file mode 100644 index 00000000..aeecf1ca --- /dev/null +++ b/packages/api/test/fly-credential-proxy-provisioner.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; +import { createFlyCredentialProxyProvisioner, createFlyCredentialProxyGuest } from "../src/fly-credential-proxy-provisioner"; + +const epoch = "11111111-1111-4111-8111-111111111111"; +const machine = { id: "machine-1", state: "created" }; +const input = { + tenantId: "11111111-1111-4111-8111-111111111112", + app: "factorize-proxies", + image: `ghcr.io/generationsoftware/credential-proxy@sha256:${"a".repeat(64)}`, + region: "iad", + size: "shared-cpu-1x", + controlOrigin: "https://factorize.example", + operationId: "tenant-proxy-start-1", +}; + +function fixture() { + const control = { + provisionCredential: vi.fn(async () => `fzc_${"b".repeat(43)}`), + revokeCredential: vi.fn(async () => true), + }; + const machines = { + createMachine: vi.fn(async () => machine), + startMachine: vi.fn(async () => undefined), + destroyMachine: vi.fn(async () => undefined), + }; + const guest = vi.fn(() => ({ startProxy: vi.fn(async () => undefined) })); + const provisioner = createFlyCredentialProxyProvisioner({ control, machines, guest, newBootEpoch: () => epoch }); + return { control, machines, guest, provisioner }; +} + +describe("Fly credential proxy startup", () => { + it("creates a digest-pinned dedicated Machine and delivers bootstrap only through startup", async () => { + const { control, machines, guest, provisioner } = fixture(); + const started = await provisioner.start(input); + expect(started).toMatchObject({ proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, machine }); + expect(machines.createMachine).toHaveBeenCalledWith(input.app, expect.objectContaining({ + image: input.image, + region: input.region, + cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", epoch], + restart: { policy: "no" }, + auto_start: false, + metadata: expect.objectContaining({ factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId }), + }), input.operationId); + expect(JSON.stringify(machines.createMachine.mock.calls[0])).not.toContain("fzc_"); + expect(machines.startMachine).toHaveBeenCalledWith(input.app, machine.id); + expect(control.provisionCredential).toHaveBeenCalledWith(input.tenantId, "fly:factorize-proxies:machine-1", epoch); + expect(guest).toHaveBeenCalledWith(machine); + const guestAdapter = guest.mock.results[0]!.value; + expect(guestAdapter.startProxy).toHaveBeenCalledWith(expect.objectContaining({ + tenantId: input.tenantId, + proxyIdentity: "fly:factorize-proxies:machine-1", + bootEpoch: epoch, + controlOrigin: input.controlOrigin, + credential: `fzc_${"b".repeat(43)}`, + })); + }); + + it("revokes bootstrap authority and removes a Machine when startup fails", async () => { + const { control, machines, guest } = fixture(); + guest.mockReturnValue({ startProxy: vi.fn().mockRejectedValue(new Error("private startup channel failed")) }); + const provisioner = createFlyCredentialProxyProvisioner({ control, machines, guest, newBootEpoch: () => epoch }); + await expect(provisioner.start(input)).rejects.toThrow("private startup channel failed"); + expect(control.revokeCredential).toHaveBeenCalledWith(input.tenantId, "fly:factorize-proxies:machine-1"); + expect(machines.destroyMachine).toHaveBeenCalledWith(input.app, machine.id); + }); + + it("rejects mutable images and unsafe control origins before provisioning", async () => { + const { machines, provisioner } = fixture(); + await expect(provisioner.start({ ...input, image: "ghcr.io/example/proxy:latest" })).rejects.toThrow("pinned by sha256 digest"); + await expect(provisioner.start({ ...input, controlOrigin: "http://factorize.example" })).rejects.toThrow("HTTPS origin"); + expect(machines.createMachine).not.toHaveBeenCalled(); + }); +}); + +describe("Fly credential proxy guest adapter", () => { + it("sends the bootstrap credential as stdin to the reviewed startup command", async () => { + const calls: unknown[][] = []; + const spawnProcess = ((command: string, args: string[], options: { stdio: string[] }) => { + calls.push([command, args, options]); + const stdoutHandlers: Array<(chunk: Buffer) => void> = []; + const child: any = { pid: undefined, stdin: { end: (value: string) => { calls.push(["stdin", value]); } }, + stdout: { on: (_event: string, callback: (chunk: Buffer) => void) => stdoutHandlers.push(callback) }, + stderr: { on: () => undefined }, on: (event: string, callback: (code: number) => void) => { + if (event === "close") queueMicrotask(() => { stdoutHandlers.forEach(handler => handler(Buffer.from("Credential proxy started\n"))); callback(0); }); + }, kill: () => undefined }; + return child; + }) as any; + const guest = createFlyCredentialProxyGuest({ app: input.app, machine: machine.id, token: "server-token-fixture", spawnProcess }); + await guest.startProxy({ tenantId: input.tenantId, proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, credential: `fzc_${"c".repeat(43)}` }); + expect(calls[0]?.[1]).toContain("/usr/local/bin/factorize-proxy-startup"); + expect(String(calls[1]?.[1])).toContain(`"credential":"fzc_${"c".repeat(43)}"`); + }); +}); diff --git a/packages/credential-proxy/src/bootstrap-control.mjs b/packages/credential-proxy/src/bootstrap-control.mjs new file mode 100644 index 00000000..80e02fec --- /dev/null +++ b/packages/credential-proxy/src/bootstrap-control.mjs @@ -0,0 +1,67 @@ +import { createServer } from "node:net"; +import { chmod, mkdir, rm } from "node:fs/promises"; +import { dirname } from "node:path"; +import { registerProxyBoot } from "./control-client.mjs"; + +const MAX_BOOTSTRAP_BYTES = 4096; + +/** Local-only receiver for a provider exec channel; credentials never enter env or files. */ +export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot } = {}) { + if (!runtime || typeof register !== "function") throw new TypeError("proxy bootstrap control requires a runtime and registrar"); + await mkdir(dirname(socketPath), { recursive: true, mode: 0o700 }); + await chmod(dirname(socketPath), 0o700); + await rm(socketPath, { force: true }); + const server = createServer((socket) => { + let data = Buffer.alloc(0); + let settled = false; + let processing = false; + const finish = (value) => { + if (settled) return; + settled = true; + socket.end(`${JSON.stringify(value)}\n`); + }; + socket.setTimeout(5000, () => finish({ error: "invalid bootstrap request" })); + socket.on("data", async (chunk) => { + if (settled || processing) return; + if (data.byteLength + chunk.byteLength > MAX_BOOTSTRAP_BYTES) return finish({ error: "invalid bootstrap request" }); + data = Buffer.concat([data, chunk]); + const end = data.indexOf(10); + if (end < 0) return; + if (end !== data.byteLength - 1) return finish({ error: "invalid bootstrap request" }); + let request; + try { request = JSON.parse(data.subarray(0, end).toString("utf8")); } + catch { return finish({ error: "invalid bootstrap request" }); } + if (!request || typeof request !== "object" || Array.isArray(request) + || Object.keys(request).sort().join(",") !== "bootEpoch,controlOrigin,credential" + || request.bootEpoch !== runtime.bootEpoch || typeof request.credential !== "string" + || !/^fzc_[A-Za-z0-9_-]{43}$/.test(request.credential)) { + return finish({ error: "invalid bootstrap request" }); + } + processing = true; + socket.pause(); + socket.removeAllListeners("data"); + let credential = request.credential; + try { + const registration = await register({ controlOrigin: request.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + finish({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); + } catch { + finish({ error: "proxy boot registration failed" }); + } finally { + credential = ""; + request.credential = ""; + } + }); + socket.on("error", () => {}); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, () => { server.removeListener("error", reject); resolve(); }); + }); + await chmod(socketPath, 0o600); + return Object.freeze({ + close: async () => { + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + await rm(socketPath, { force: true }); + }, + }); +} diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index f5409b16..01fce4c6 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -1,8 +1,13 @@ import { createInterface } from "node:readline"; +import { randomUUID } from "node:crypto"; import { createProxyRuntime } from "./runtime.mjs"; import { registerProxyBoot } from "./control-client.mjs"; +import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; -const runtime = createProxyRuntime(); +const epochIndex = process.argv.indexOf("--boot-epoch"); +const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; +const runtime = createProxyRuntime({ bootEpoch }); +const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET }); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); let closing = false; let shutdownDeadline; @@ -15,6 +20,7 @@ async function stop() { if (closing) return; closing = true; await runtime.shutdown(); + await bootstrapControl.close(); lines.close(); clearTimeout(shutdownDeadline); process.exit(0); diff --git a/packages/credential-proxy/src/startup.mjs b/packages/credential-proxy/src/startup.mjs new file mode 100644 index 00000000..1eb7d163 --- /dev/null +++ b/packages/credential-proxy/src/startup.mjs @@ -0,0 +1,46 @@ +import { connect } from "node:net"; +import { once } from "node:events"; + +const SOCKET_PATH = "/run/factorize-proxy/bootstrap.sock"; +const MAX_BYTES = 4096; + +async function readRequest() { + let bytes = 0; + const chunks = []; + for await (const chunk of process.stdin) { + bytes += chunk.byteLength; + if (bytes > MAX_BYTES) throw new Error("invalid bootstrap request"); + chunks.push(chunk); + } + const data = Buffer.concat(chunks); + if (!data.byteLength || data[data.byteLength - 1] !== 10) throw new Error("invalid bootstrap request"); + return data; +} + +async function main() { + const request = await readRequest(); + const socket = connect(SOCKET_PATH); + const result = new Promise((resolve, reject) => { + let data = Buffer.alloc(0); + socket.on("data", chunk => { + data = Buffer.concat([data, chunk]); + const end = data.indexOf(10); + if (end >= 0) { + try { resolve(JSON.parse(data.subarray(0, end).toString("utf8"))); } + catch { reject(new Error("invalid bootstrap response")); } + socket.end(); + } + }); + socket.once("error", () => reject(new Error("proxy startup control unavailable"))); + socket.setTimeout(7000, () => reject(new Error("proxy startup control timed out"))); + }); + socket.write(request); + const response = await result; + if (response.registered !== true) throw new Error("proxy boot registration failed"); + process.stdout.write("Credential proxy started\n"); +} + +main().catch(() => { + process.stderr.write("Credential proxy startup failed\n"); + process.exitCode = 1; +}); diff --git a/packages/credential-proxy/test/bootstrap-control.test.mjs b/packages/credential-proxy/test/bootstrap-control.test.mjs new file mode 100644 index 00000000..2cae7dfb --- /dev/null +++ b/packages/credential-proxy/test/bootstrap-control.test.mjs @@ -0,0 +1,56 @@ +import { strict as assert } from "node:assert"; +import { afterEach, test } from "node:test"; +import { connect } from "node:net"; +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { once } from "node:events"; +import { startProxyBootstrapControl } from "../src/bootstrap-control.mjs"; + +const epoch = "11111111-1111-4111-8111-111111111111"; +const token = `fzc_${"a".repeat(43)}`; +let directory; +let service; +afterEach(async () => { + await service?.close(); service = undefined; + if (directory) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +async function fixture(register) { + directory = await mkdtemp(join(tmpdir(), "factorize-proxy-startup-")); + const socketPath = join(directory, "control.sock"); + const runtime = { bootEpoch: epoch }; + service = await startProxyBootstrapControl({ runtime, socketPath, register }); + return socketPath; +} + +async function request(socketPath, payload) { + const socket = connect(socketPath); + let response = ""; + socket.on("data", chunk => { response += chunk.toString(); }); + await once(socket, "connect"); + socket.end(`${JSON.stringify(payload)}\n`); + await once(socket, "close"); + return JSON.parse(response); +} + +test("local startup control redeems the one-use credential and leaves a private socket", async () => { + const calls = []; + const socketPath = await fixture(async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch }; }); + const result = await request(socketPath, { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + assert.deepEqual(result, { registered: true, proxyIdentity: "fly:app:machine", bootEpoch: epoch }); + assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch }]); + assert.equal((await stat(socketPath)).mode & 0o777, 0o600); +}); + +test("local startup control rejects run credentials, epoch substitution, and extra fields", async () => { + const calls = []; + const socketPath = await fixture(async value => { calls.push(value); return {}; }); + for (const payload of [ + { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, + { controlOrigin: "https://factorize.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, + { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, + ]) assert.deepEqual(await request(socketPath, payload), { error: "invalid bootstrap request" }); + assert.deepEqual(calls, []); +}); diff --git a/packages/credential-proxy/test/runtime-process.test.mjs b/packages/credential-proxy/test/runtime-process.test.mjs index 03944bf9..87ebeb61 100644 --- a/packages/credential-proxy/test/runtime-process.test.mjs +++ b/packages/credential-proxy/test/runtime-process.test.mjs @@ -19,8 +19,8 @@ async function waitFor(predicate, label) { } } -async function startProxy() { - const child = spawn(process.execPath, [entry], { stdio: ["pipe", "pipe", "pipe"] }); +async function startProxy(directory) { + const child = spawn(process.execPath, [entry], { stdio: ["pipe", "pipe", "pipe"], env: { ...process.env, FACTORIZE_PROXY_BOOTSTRAP_SOCKET: join(directory, "bootstrap.sock") } }); let pending = ""; let log = ""; let stderr = ""; @@ -59,7 +59,7 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts try { await writeFile(join(directory, "identity.json"), JSON.stringify({ identity: "public-fixture", generation: 1 })); await writeFile(join(directory, "public-metadata.json"), JSON.stringify({ displayName: "fixture" })); - const first = await startProxy(); + const first = await startProxy(directory); processes.push(first); assert.deepEqual(await first.command({ type: "status" }), { type: "status", state: "locked" }); assert.deepEqual(await first.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked" }); @@ -67,7 +67,7 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts await waitFor(() => first.child.exitCode !== null || first.child.signalCode !== null, "first process stop"); assert.equal(first.child.exitCode, 0); - const restarted = await startProxy(); + const restarted = await startProxy(directory); processes.push(restarted); assert.notEqual(restarted.bootEpoch, first.bootEpoch, "every process restart gets a fresh enrollment epoch"); assert.deepEqual(await restarted.command({ type: "status" }), { type: "status", state: "locked" }); @@ -76,14 +76,14 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts await waitFor(() => restarted.child.exitCode !== null || restarted.child.signalCode !== null, "command shutdown"); assert.equal(restarted.child.exitCode, 0); - const crashed = await startProxy(); + const crashed = await startProxy(directory); processes.push(crashed); assert.deepEqual(await crashed.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked" }); crashed.child.kill("SIGKILL"); await waitFor(() => crashed.child.exitCode !== null || crashed.child.signalCode !== null, "crash stop"); assert.equal(crashed.child.signalCode, "SIGKILL"); - const afterCrash = await startProxy(); + const afterCrash = await startProxy(directory); processes.push(afterCrash); assert.deepEqual(await afterCrash.command({ type: "status" }), { type: "status", state: "locked" }); afterCrash.child.kill("SIGTERM"); From e6315532621f7204d3e17eb5bc7d618ae9dbac7f Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 19:35:26 +0000 Subject: [PATCH 009/102] Retry proxy tunnel enrollment before forwarding --- .../credential-proxy/src/tunnel-transport.mjs | 93 +++++++++++++++---- .../test/proxy-listener.e2e.test.mjs | 30 +++--- 2 files changed, 91 insertions(+), 32 deletions(-) diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index 2ad21a67..8fe4bfa2 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -4,6 +4,9 @@ import { once } from "node:events"; const HANDSHAKE_LIMIT = 4096; const HANDSHAKE_TIMEOUT_MS = 10_000; +const CONTROL_RETRY_WINDOW_MS = 5 * 60_000; +const CONTROL_RETRY_MAX_MS = 30_000; +const CONTROL_RETRY_SLOW_MS = 60_000; function peerIdentity(socket) { const certificate = socket.getPeerCertificate(); @@ -62,6 +65,7 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro target = connectTcp(address.port, address.address); await once(target, "connect"); if (rest.length) target.write(rest); + socket.write("FACTORIZE-TUNNEL/1 ACCEPTED\n"); socket.pipe(target); target.pipe(socket); socket.resume(); @@ -94,32 +98,50 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro } /** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ -export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPort, host = "127.0.0.1", port = 0 } = {}) { +export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPort, host = "127.0.0.1", port = 0, + now = Date.now, random = Math.random, wait = delay } = {}) { if (!tls || typeof enrollment !== "function" || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { throw new TypeError("VM tunnel helper requires TLS, enrollment and proxy address"); } const sockets = new Set(); const server = createTcpServer((local) => { sockets.add(local); + local.pause(); void (async () => { - let remote; - try { - const grant = await enrollment(); - if (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token)) throw new Error("fresh enrollment unavailable"); - remote = connectTls({ ...tls, host: proxyHost, port: proxyPort, servername: tls.servername ?? proxyHost, rejectUnauthorized: true }); - sockets.add(remote); - const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); - try { await once(remote, "secureConnect"); } - finally { clearTimeout(connectTimer); } - remote.write(`FACTORIZE-TUNNEL/1 ${grant.token}\n`); - local.pipe(remote); - remote.pipe(local); - local.resume(); - remote.once("close", () => { sockets.delete(remote); local.destroy(); }); - local.once("close", () => remote.destroy()); - } catch { - local.destroy(); - remote?.destroy(); + const startedAt = now(); + let attempt = 0; + while (!local.destroyed) { + let remote; + try { + const grant = await enrollment(); + if (local.destroyed) return; + if (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token)) throw new Error("fresh enrollment unavailable"); + remote = connectTls({ ...tls, host: proxyHost, port: proxyPort, servername: tls.servername ?? proxyHost, rejectUnauthorized: true }); + sockets.add(remote); + const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); + try { await once(remote, "secureConnect"); } + finally { clearTimeout(connectTimer); } + remote.write(`FACTORIZE-TUNNEL/1 ${grant.token}\n`); + await waitForTunnelAcceptance(remote); + local.pipe(remote); + remote.pipe(local); + local.resume(); + remote.once("close", () => { sockets.delete(remote); local.destroy(); }); + local.once("close", () => remote.destroy()); + return; + } catch { + remote?.destroy(); + if (remote) sockets.delete(remote); + if (local.destroyed) return; + const elapsed = now() - startedAt; + const base = elapsed < CONTROL_RETRY_WINDOW_MS + ? Math.min(1000 * (2 ** Math.min(attempt, 5)), CONTROL_RETRY_MAX_MS) + : CONTROL_RETRY_SLOW_MS; + const jittered = Math.round(base * (0.5 + random())); + const delayMs = Math.max(1, elapsed < CONTROL_RETRY_WINDOW_MS ? Math.min(CONTROL_RETRY_MAX_MS, jittered) : jittered); + attempt++; + try { await wait(delayMs); } catch { local.destroy(); return; } + } } })(); const remove = () => { sockets.delete(local); }; @@ -137,3 +159,36 @@ export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPor }, }); } + +function delay(milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)); +} + +function waitForTunnelAcceptance(socket) { + return new Promise((resolve, reject) => { + let response = Buffer.alloc(0); + const cleanup = () => { + clearTimeout(timer); + socket.removeListener("data", onData); + socket.removeListener("error", onError); + socket.removeListener("close", onClose); + }; + const onError = error => { cleanup(); reject(error); }; + const onClose = () => { cleanup(); reject(new Error("proxy closed before accepting the tunnel")); }; + const onData = chunk => { + response = Buffer.concat([response, chunk]); + if (response.length > HANDSHAKE_LIMIT) { cleanup(); reject(new Error("invalid proxy tunnel response")); return; } + const end = response.indexOf(10); + if (end < 0) return; + const accepted = response.subarray(0, end).toString("utf8") === "FACTORIZE-TUNNEL/1 ACCEPTED"; + const trailingData = response.subarray(end + 1); + cleanup(); + if (!accepted || trailingData.length) reject(new Error("proxy tunnel was rejected")); + else resolve(); + }; + const timer = setTimeout(() => { cleanup(); socket.destroy(); reject(new Error("proxy tunnel acceptance timed out")); }, HANDSHAKE_TIMEOUT_MS); + socket.on("data", onData); + socket.once("error", onError); + socket.once("close", onClose); + }); +} diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index c8c6a940..70a862df 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -140,6 +140,8 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante let observedAuthorization; let enrollmentCount = 0; let authenticationCount = 0; + let mismatchReachedResolve; + const mismatchReached = new Promise(resolve => { mismatchReachedResolve = resolve; }); const upstream = createHttpsServer({ key, cert }, (request, response) => { observedAuthorization = request.headers.authorization; response.end("transport-ok"); @@ -162,13 +164,15 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante runtime, authenticateEnrollment: async ({ token, vmIdentity, bootEpoch }) => { authenticationCount++; - assert.equal(token, (authenticationCount === 1 ? "A" : authenticationCount === 2 ? "B" : "C").repeat(43)); + assert.ok(["A", "B", "C", "X"].some(value => token === value.repeat(43))); assert.equal(vmIdentity, "api.example.test"); assert.equal(bootEpoch, runtime.bootEpoch); + if (token === "A".repeat(43)) throw new Error("temporary control-plane outage"); + if (token === "X".repeat(43)) mismatchReachedResolve(); return { authenticated: true, enrollmentConsumed: true, - vmIdentity: token === "C".repeat(43) ? "other-vm" : vmIdentity, + vmIdentity: token === "X".repeat(43) ? "other-vm" : vmIdentity, bootEpoch, connectionId: "transport-enrollment-1", runId: "run-transport-1", @@ -181,9 +185,10 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante runtime.unlock("synthetic-unlock-key"); const local = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, - enrollment: async () => ({ token: (++enrollmentCount === 1 ? "A" : enrollmentCount === 2 ? "B" : "C").repeat(43) }), + enrollment: async () => ({ token: ["A", "B", "C"][enrollmentCount++ % 3].repeat(43) }), proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, + wait: async () => {}, }); let mismatchLocal; try { @@ -195,21 +200,20 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante assert.equal(observedAuthorization, "Bearer synthetic-bearer"); const denied = await fetch(`http://127.0.0.1:${local.address.port}/integrations/other-tool/echo`); assert.equal(denied.status, 403); - assert.equal(enrollmentCount, 2, "each local connection gets fresh one-use enrollment"); - assert.equal(authenticationCount, 2, "each enrollment is independently consumed by the proxy transport"); + assert.ok(enrollmentCount >= 3, "failed setup and local connections get fresh one-use enrollment"); + assert.equal(enrollmentCount, authenticationCount, "each enrollment is independently consumed by the proxy transport"); mismatchLocal = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, - enrollment: async () => ({ token: (++enrollmentCount === 3 ? "C" : "D").repeat(43) }), + enrollment: async () => ({ token: (enrollmentCount++, "X".repeat(43)) }), proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, }); - await assert.rejects( - fetch(`http://127.0.0.1:${mismatchLocal.address.port}/integrations/tool-1/echo`), - /fetch failed/, - "mismatched VM binding is rejected", - ); - assert.equal(enrollmentCount, 3); - assert.equal(authenticationCount, 3); + const rejectedRequest = fetch(`http://127.0.0.1:${mismatchLocal.address.port}/integrations/tool-1/echo`); + await mismatchReached; + await mismatchLocal.close(); + mismatchLocal = undefined; + await assert.rejects(rejectedRequest, /fetch failed/, "mismatched VM binding is rejected"); + assert.equal(authenticationCount, enrollmentCount); } finally { await mismatchLocal?.close(); await local.close(); From 545d9e7d0fbc677a12e203fc0518a7b1449fd02e Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 20:04:51 +0000 Subject: [PATCH 010/102] Add authenticated proxy enrollment control API --- .../0037_proxy_control_credentials.sql | 2 + .../src/postgres/proxy-control-repository.ts | 22 +++++++--- packages/api/src/postgres/schema.ts | 2 +- packages/api/src/proxy-control.ts | 30 ++++++++++--- .../api/test/proxy-control-repository.test.ts | 4 +- packages/api/test/proxy-control.test.ts | 43 +++++++++++++++++-- .../credential-proxy/src/control-client.mjs | 34 +++++++++++++-- packages/credential-proxy/src/main.mjs | 4 +- .../test/control-client.test.mjs | 27 ++++++++++-- 9 files changed, 143 insertions(+), 25 deletions(-) create mode 100644 packages/api/migrations/0037_proxy_control_credentials.sql diff --git a/packages/api/migrations/0037_proxy_control_credentials.sql b/packages/api/migrations/0037_proxy_control_credentials.sql new file mode 100644 index 00000000..b3c69e79 --- /dev/null +++ b/packages/api/migrations/0037_proxy_control_credentials.sql @@ -0,0 +1,2 @@ +ALTER TABLE app.credential_proxy_control_credentials + ADD COLUMN proxy_token_digest bytea UNIQUE; diff --git a/packages/api/src/postgres/proxy-control-repository.ts b/packages/api/src/postgres/proxy-control-repository.ts index 2dee38ae..09fd28c9 100644 --- a/packages/api/src/postgres/proxy-control-repository.ts +++ b/packages/api/src/postgres/proxy-control-repository.ts @@ -1,6 +1,6 @@ import type { Database } from "./database"; -export type RegisteredProxy = { tenantId: string; proxyIdentity: string; bootEpoch: string }; +export type RegisteredProxy = { tenantId: string; proxyIdentity: string; bootEpoch: string; controlCredential: string }; const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; const digest = async (token: string) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(token))); @@ -13,7 +13,7 @@ export class ProxyControlRepository { if (!uuid.test(tenantId) || !proxyIdentity.trim() || proxyIdentity.length > 255 || !uuid.test(bootEpoch)) throw new TypeError("Invalid proxy identity or boot epoch"); const token = `fzc_${Buffer.from(crypto.getRandomValues(new Uint8Array(32))).toString("base64url")}`; await this.database.pool.query(`INSERT INTO app.credential_proxy_control_credentials(tenant_id,proxy_identity,boot_epoch,token_digest,revoked_at,consumed_at) - VALUES ($1,$2,$3,$4,NULL,NULL) ON CONFLICT (tenant_id,proxy_identity) DO UPDATE SET boot_epoch=excluded.boot_epoch,token_digest=excluded.token_digest,revoked_at=NULL,consumed_at=NULL,created_at=now()`, + VALUES ($1,$2,$3,$4,NULL,NULL) ON CONFLICT (tenant_id,proxy_identity) DO UPDATE SET boot_epoch=excluded.boot_epoch,token_digest=excluded.token_digest,proxy_token_digest=NULL,revoked_at=NULL,consumed_at=NULL,created_at=now()`, [tenantId, proxyIdentity, bootEpoch, Buffer.from(await digest(token))]); return token; } @@ -40,14 +40,26 @@ export class ProxyControlRepository { AND app.credential_proxy_states.disabled_at IS NULL AND app.credential_proxy_states.revoked_at IS NULL RETURNING tenant_id`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); if (!saved.rowCount) return null; - await client.query(`UPDATE app.credential_proxy_control_credentials SET consumed_at=now() - WHERE token_digest=$1 AND boot_epoch=$2 AND consumed_at IS NULL`, [tokenDigest, bootEpoch]); + const proxyToken = `fzc_${Buffer.from(crypto.getRandomValues(new Uint8Array(32))).toString("base64url")}`; + await client.query(`UPDATE app.credential_proxy_control_credentials SET consumed_at=now(),proxy_token_digest=$3 + WHERE token_digest=$1 AND boot_epoch=$2 AND consumed_at IS NULL`, [tokenDigest, bootEpoch, Buffer.from(await digest(proxyToken))]); await client.query(`UPDATE app.credential_run_grants SET revoked_at=now() WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch<>$3 AND revoked_at IS NULL`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); - return { tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch }; + return { tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch, controlCredential: proxyToken }; }); } + /** Authenticates the proxy's memory-held control credential against its current registered boot. */ + async authenticateProxy(token: string): Promise | null> { + if (typeof token !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return null; + const result = await this.database.pool.query<{ tenant_id: string; proxy_identity: string; boot_epoch: string }>(`SELECT c.tenant_id,c.proxy_identity,c.boot_epoch + FROM app.credential_proxy_control_credentials c JOIN app.credential_proxy_states p + ON p.tenant_id=c.tenant_id AND p.proxy_identity=c.proxy_identity AND p.boot_epoch=c.boot_epoch + WHERE c.proxy_token_digest=$1 AND c.revoked_at IS NULL AND p.disabled_at IS NULL AND p.revoked_at IS NULL`, [Buffer.from(await digest(token))]); + const row = result.rows[0]; + return row ? { tenantId: row.tenant_id, proxyIdentity: row.proxy_identity, bootEpoch: row.boot_epoch } : null; + } + async revokeCredential(tenantId: string, proxyIdentity: string): Promise { const result = await this.database.pool.query(`UPDATE app.credential_proxy_control_credentials SET revoked_at=now() WHERE tenant_id=$1 AND proxy_identity=$2 AND revoked_at IS NULL`, [tenantId, proxyIdentity]); diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 4d5e9d50..2fbfed29 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -53,7 +53,7 @@ export const integrationChanges = app.table("integration_changes", { export const credentialProxyStates = app.table("credential_proxy_states", { tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), disabledAt: timestamp("disabled_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), updatedAt: updatedAt(), }); -export const credentialProxyControlCredentials = app.table("credential_proxy_control_credentials", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), tokenDigest: bytea("token_digest").notNull().unique(), revokedAt: timestamp("revoked_at", { withTimezone: true }), consumedAt: timestamp("consumed_at", { withTimezone: true }), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.proxyIdentity] })]); +export const credentialProxyControlCredentials = app.table("credential_proxy_control_credentials", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), tokenDigest: bytea("token_digest").notNull().unique(), proxyTokenDigest: bytea("proxy_token_digest").unique(), revokedAt: timestamp("revoked_at", { withTimezone: true }), consumedAt: timestamp("consumed_at", { withTimezone: true }), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.proxyIdentity] })]); export const credentialRunGrants = app.table("credential_run_grants", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), vmIdentity: text("vm_identity").notNull(), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), grantRevision: bigint("grant_revision", { mode: "number" }).notNull(), revokedAt: timestamp("revoked_at", { withTimezone: true }), createdAt: createdAt(), }, t => [primaryKey({ columns: [t.tenantId, t.runId] }), foreignKey({ columns: [t.tenantId, t.runId], foreignColumns: [jobRuns.tenantId, jobRuns.id] }).onDelete("cascade"), check("credential_run_grants_revision_check", sql`${t.grantRevision} > 0`)]); diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index 3cd1632b..695779cc 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -1,12 +1,13 @@ import { databaseFor } from "./postgres/database"; import { ProxyControlRepository } from "./postgres/proxy-control-repository"; +import { RunEnrollmentRepository } from "./postgres/run-enrollment-repository"; const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "Cache-Control": "no-store" } }); /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (url.pathname !== "/internal/credential-proxy/register-boot") return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; @@ -28,11 +29,26 @@ export async function proxyControlFetch(request: Request, env: Parameters key !== "bootEpoch") - || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); const repository = new ProxyControlRepository(databaseFor(env)); - const registration = await repository.registerBoot(token, (body as any).bootEpoch); - if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); - return json({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); + if (url.pathname === "/internal/credential-proxy/register-boot") { + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => key !== "bootEpoch") || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); + const registration = await repository.registerBoot(token, (body as any).bootEpoch); + if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); + return json({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch, controlCredential: registration.controlCredential }); + } + + const proxy = await repository.authenticateProxy(token); + if (!proxy) return json({ error: "unauthorized" }, 401); + const enrollments = new RunEnrollmentRepository(databaseFor(env)); + if (url.pathname === "/internal/credential-proxy/enroll") { + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity"].includes(key)) + || typeof (body as any).runId !== "string" || typeof (body as any).vmIdentity !== "string") return json({ error: "invalid_request" }, 400); + const enrollment = await enrollments.issue({ tenantId: proxy.tenantId, runId: (body as any).runId, vmIdentity: (body as any).vmIdentity, proxyIdentity: proxy.proxyIdentity, bootEpoch: proxy.bootEpoch }, new Date(Date.now() + 30_000)); + return enrollment ? json(enrollment) : json({ error: "enrollment_unavailable" }, 403); + } + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity", "enrollmentToken"].includes(key)) + || typeof (body as any).runId !== "string" || typeof (body as any).vmIdentity !== "string" || typeof (body as any).enrollmentToken !== "string") return json({ error: "invalid_request" }, 400); + const binding = { tenantId: proxy.tenantId, runId: (body as any).runId, vmIdentity: (body as any).vmIdentity, proxyIdentity: proxy.proxyIdentity, bootEpoch: proxy.bootEpoch }; + const consumed = await enrollments.consume((body as any).enrollmentToken, binding); + return consumed ? json({ consumed: true, binding: consumed }) : json({ error: "enrollment_unavailable" }, 403); } diff --git a/packages/api/test/proxy-control-repository.test.ts b/packages/api/test/proxy-control-repository.test.ts index 20f1934c..9508486a 100644 --- a/packages/api/test/proxy-control-repository.test.ts +++ b/packages/api/test/proxy-control-repository.test.ts @@ -33,7 +33,9 @@ describe("dedicated credential proxy control identity", () => { const tx = { query: vi.fn().mockImplementation((sql: string) => sql.includes("credential_proxy_control_credentials") ? credentialQuery() : sql.includes("INSERT INTO app.credential_proxy_states") ? stateQuery() : Promise.resolve({ rows: [], rowCount: 1 })) }; (repository as any).database = { transaction: (work: (client: any) => Promise) => work(tx) }; const epoch = "22222222-2222-4222-8222-222222222222"; - expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch)).toEqual({ tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch: epoch }); + const registration = await repository.registerBoot("fzc_" + "a".repeat(43), epoch); + expect(registration).toMatchObject({ tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch: epoch }); + expect(registration?.controlCredential).toMatch(/^fzc_[A-Za-z0-9_-]{43}$/); expect(tx.query.mock.calls.map(call => call[0])).toHaveLength(4); expect(tx.query.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(tx.query.mock.calls[2][0]).toContain("consumed_at=now()"); diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 734cfff4..c0a33b1e 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -10,11 +10,14 @@ function envFor({ credentialFound = true, stateSaved = true } = {}) { if (sql.includes("INSERT INTO app.credential_proxy_states")) return { rows: stateSaved ? [{ tenant_id: "11111111-1111-4111-8111-111111111111" }] : [], rowCount: stateSaved ? 1 : 0 }; return { rows: [], rowCount: 1 }; }); - return { DATABASE: { pool: { query: vi.fn() }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery } as any; + const query = vi.fn(async (sql: string) => sql.includes("proxy_token_digest") + ? { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }] : [], rowCount: credentialFound ? 1 : 0 } + : { rows: [], rowCount: 0 }); + return { DATABASE: { pool: { query }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery, query } as any; } -function request({ authorization, body = { bootEpoch }, method = "POST" }: { authorization?: string; body?: unknown; method?: string } = {}) { - return new Request("https://factorize.example/internal/credential-proxy/register-boot", { method, headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}) }, body: method === "POST" ? JSON.stringify(body) : undefined }); +function request({ authorization, body = { bootEpoch }, method = "POST", path = "/internal/credential-proxy/register-boot" }: { authorization?: string; body?: unknown; method?: string; path?: string } = {}) { + return new Request(`https://factorize.example${path}`, { method, headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}) }, body: method === "POST" ? JSON.stringify(body) : undefined }); } describe("dedicated proxy boot registration endpoint", () => { @@ -22,7 +25,7 @@ describe("dedicated proxy boot registration endpoint", () => { const env = envFor(); const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); expect(response.status).toBe(200); - expect(await response.json()).toEqual({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch }); + expect(await response.json()).toMatchObject({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: expect.stringMatching(/^fzc_[A-Za-z0-9_-]{43}$/) }); expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(4); expect(env.transactionQuery.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(env.transactionQuery.mock.calls[2][0]).toContain("consumed_at=now()"); @@ -55,4 +58,36 @@ describe("dedicated proxy boot registration endpoint", () => { expect(response.status).toBe(413); expect(env.transactionQuery).not.toHaveBeenCalled(); }); + + it("issues a short-lived enrollment only through the registered proxy identity", async () => { + const env = envFor(); + env.transactionQuery.mockImplementation(async (sql: string) => sql.includes("SELECT g.grant_revision") + ? { rows: [{ grant_revision: 3 }], rowCount: 1 } + : { rows: [], rowCount: 1 }); + const response = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/enroll", + body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123" }, + }), env); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ grantRevision: 3, enrollmentId: expect.any(String), token: expect.any(String) }); + expect(env.query).toHaveBeenCalled(); + expect(env.transactionQuery.mock.calls[0][0]).toContain("JOIN app.credential_proxy_states"); + }); + + it("consumes enrollment against the authenticated proxy boot and rejects untrusted extra bindings", async () => { + const env = envFor(); + env.transactionQuery.mockImplementation(async (sql: string) => sql.includes("UPDATE app.credential_run_enrollments") + ? { rows: [{ id: "44444444-4444-4444-8444-444444444444", grant_revision: 3 }], rowCount: 1 } + : { rows: [], rowCount: 1 }); + const accepted = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/consume-enrollment", + body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123", enrollmentToken: "e".repeat(43) }, + }), env); + expect(accepted.status).toBe(200); + expect(await accepted.json()).toMatchObject({ consumed: true, binding: { bootEpoch, proxyIdentity: "proxy-fixture", vmIdentity: "fly:vm-123" } }); + const rejected = await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123", tenantId: "other" } }), env); + expect(rejected.status).toBe(400); + }); }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 77a430e2..08b0358f 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -1,9 +1,14 @@ const bootEpochPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; /** Register one proxy process boot using a provisioning-scoped credential. The token is only held in this call's memory. */ -export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetchImpl = fetch }) { +function validateOrigin(controlOrigin) { const origin = new URL(controlOrigin); if (origin.protocol !== "https:" || origin.pathname !== "/" || origin.username || origin.password || origin.search || origin.hash) throw new TypeError("proxy control origin must be an HTTPS origin"); + return origin; +} + +export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetchImpl = fetch }) { + const origin = validateOrigin(controlOrigin); if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token ?? "")) throw new TypeError("invalid proxy control credential"); if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); let credential = token; @@ -17,9 +22,32 @@ export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetch }); if (!response.ok) throw new Error("proxy boot registration rejected"); const result = await response.json(); - if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string") throw new Error("invalid proxy boot registration response"); - return { proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch }; + if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(result.controlCredential ?? "")) throw new Error("invalid proxy boot registration response"); + return { proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch, controlCredential: result.controlCredential }; } finally { credential = ""; } } + +async function proxyControlRequest({ controlOrigin, token, path, body, fetchImpl }) { + const origin = validateOrigin(controlOrigin); + if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token ?? "")) throw new TypeError("invalid proxy control credential"); + const response = await fetchImpl(new URL(path, origin), { + method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, + body: JSON.stringify(body), signal: AbortSignal.timeout(10_000), redirect: "error", + }); + if (!response.ok) throw new Error("proxy control request rejected"); + return response.json(); +} + +export async function issueProxyEnrollment(options) { + const result = await proxyControlRequest({ ...options, path: "/internal/credential-proxy/enroll" }); + if (typeof result?.token !== "string" || typeof result?.enrollmentId !== "string" || !Number.isSafeInteger(result?.grantRevision)) throw new Error("invalid proxy enrollment response"); + return result; +} + +export async function consumeProxyEnrollment(options) { + const result = await proxyControlRequest({ ...options, path: "/internal/credential-proxy/consume-enrollment" }); + if (result?.consumed !== true || typeof result?.binding?.enrollmentId !== "string") throw new Error("invalid consumed enrollment response"); + return result.binding; +} diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 01fce4c6..3a7750ae 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -11,6 +11,7 @@ const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); let closing = false; let shutdownDeadline; +let controlCredential; function respond(value) { process.stdout.write(`${JSON.stringify(value)}\n`); @@ -49,7 +50,8 @@ lines.on("line", async (line) => { command.credential = undefined; try { const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); - respond({ type: "boot-registered", ...registration }); + controlCredential = registration.controlCredential; + respond({ type: "boot-registered", proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { respond({ type: "error", error: "proxy boot registration failed" }); } finally { credential = ""; } diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 7f59aba7..56cc2b26 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { registerProxyBoot } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, issueProxyEnrollment, registerProxyBoot } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -9,9 +9,9 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an let request; const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async (url, init) => { request = { url: String(url), init }; - return Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch }); + return Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); } }); - assert.deepEqual(result, { proxyIdentity: "proxy-fixture", bootEpoch }); + assert.deepEqual(result, { proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); assert.equal(request.url, "https://factorize.example/internal/credential-proxy/register-boot"); assert.equal(request.init.headers.Authorization, `Bearer ${token}`); assert.equal(request.init.redirect, "error"); @@ -23,3 +23,24 @@ test("boot registration rejects unsafe origins, invalid credentials and mismatch await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async () => Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch: "33333333-3333-4333-8333-333333333333" }) }), /invalid proxy boot registration response/); }); + +test("proxy control client requests and consumes a one-use enrollment over authenticated HTTPS", async () => { + const controlToken = `fzc_${"c".repeat(43)}`; + const requests = []; + const fetchImpl = async (url, init) => { + requests.push({ url: String(url), init }); + return requests.length === 1 + ? Response.json({ enrollmentId: "enrollment-1", token: "e".repeat(43), grantRevision: 2 }) + : Response.json({ consumed: true, binding: { enrollmentId: "enrollment-1" } }); + }; + const common = { controlOrigin: "https://factorize.example", token: controlToken, fetchImpl }; + const enrollment = await issueProxyEnrollment({ ...common, body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1" } }); + assert.equal(enrollment.token, "e".repeat(43)); + const consumed = await consumeProxyEnrollment({ ...common, body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1", enrollmentToken: enrollment.token } }); + assert.equal(consumed.enrollmentId, "enrollment-1"); + assert.deepEqual(requests.map(request => request.url), [ + "https://factorize.example/internal/credential-proxy/enroll", + "https://factorize.example/internal/credential-proxy/consume-enrollment", + ]); + assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${controlToken}` && request.init.redirect === "error")); +}); From f1ff6ad6b1f3ba43103aaf2de1924642dfcc49bd Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 20:26:19 +0000 Subject: [PATCH 011/102] Wire proxy-side run enrollment into tunnel auth --- .../credential-proxy/src/control-client.mjs | 45 ++++++++++++++ .../credential-proxy/src/tunnel-transport.mjs | 23 +++++--- .../test/control-client.test.mjs | 58 ++++++++++++++++++- .../test/proxy-listener.e2e.test.mjs | 44 ++++++++++++++ 4 files changed, 160 insertions(+), 10 deletions(-) diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 08b0358f..b5216f10 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -51,3 +51,48 @@ export async function consumeProxyEnrollment(options) { if (result?.consumed !== true || typeof result?.binding?.enrollmentId !== "string") throw new Error("invalid consumed enrollment response"); return result.binding; } + +const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * Create the proxy-side half of enrollment. The job VM supplies only its run + * ID over the already authenticated TLS connection; the proxy derives VM and + * boot identity locally, then issues and consumes a fresh one-use enrollment + * using its private control credential. Integration secrets stay in the + * proxy's local grant resolver and never cross this control API. + */ +export function createProxyEnrollmentAuthenticator({ controlOrigin, controlCredential, bootEpoch, resolveGrants, fetchImpl = fetch } = {}) { + if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(controlCredential ?? "")) throw new TypeError("invalid proxy control credential"); + if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); + if (typeof resolveGrants !== "function") throw new TypeError("proxy enrollment requires a local grant resolver"); + validateOrigin(controlOrigin); + let credential = controlCredential; + return async function authenticateRun({ runId, vmIdentity, bootEpoch: requestedBootEpoch }) { + if (!uuidPattern.test(runId ?? "") || typeof vmIdentity !== "string" || !vmIdentity.trim() || vmIdentity.length > 255 + || requestedBootEpoch !== bootEpoch) throw new Error("invalid proxy enrollment identity"); + const options = { controlOrigin, token: credential, fetchImpl }; + const requested = { runId, vmIdentity }; + const enrollment = await issueProxyEnrollment({ ...options, body: requested }); + const binding = await consumeProxyEnrollment({ ...options, body: { ...requested, enrollmentToken: enrollment.token } }); + if (binding.runId !== runId || binding.vmIdentity !== vmIdentity || binding.bootEpoch !== bootEpoch + || typeof binding.tenantId !== "string" || typeof binding.proxyIdentity !== "string" + || binding.enrollmentId !== enrollment.enrollmentId || binding.grantRevision !== enrollment.grantRevision) { + throw new Error("consumed enrollment binding did not match the authenticated connection"); + } + const grants = await resolveGrants(binding); + if (!Array.isArray(grants)) throw new Error("proxy grants are unavailable"); + return { + authenticated: true, + enrollmentConsumed: true, + connectionId: binding.enrollmentId, + runId: binding.runId, + vmId: binding.vmIdentity, + vmIdentity: binding.vmIdentity, + bootEpoch: binding.bootEpoch, + tenantId: binding.tenantId, + proxyIdentity: binding.proxyIdentity, + grantRevision: binding.grantRevision, + grants, + }; + }; +} diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index 8fe4bfa2..c68f5857 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -19,8 +19,8 @@ function peerIdentity(socket) { * enrollment, and bridge that stream to the connection-scoped HTTP listener. * The bridge does not expose a shell or arbitrary destination forwarding. */ -export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnrollment, host = "127.0.0.1", port = 0 } = {}) { - if (!tls || !runtime || typeof authenticateEnrollment !== "function") throw new TypeError("tunnel server requires TLS, runtime and enrollment authentication"); +export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnrollment, authenticateRun, host = "127.0.0.1", port = 0 } = {}) { + if (!tls || !runtime || (typeof authenticateEnrollment !== "function" && typeof authenticateRun !== "function")) throw new TypeError("tunnel server requires TLS, runtime and enrollment authentication"); const connections = new Set(); const server = createTlsServer({ ...tls, requestCert: true, rejectUnauthorized: true }, (socket) => { connections.add(socket); @@ -46,11 +46,15 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro const line = handshake.subarray(0, end).toString("utf8").replace(/\r$/, ""); const rest = handshake.subarray(end + 1); const match = /^FACTORIZE-TUNNEL\/1 ([A-Za-z0-9_-]{32,128})$/.exec(line); - if (!match) return reject(); + const runMatch = /^FACTORIZE-TUNNEL\/2 ([0-9a-f-]{36})$/i.exec(line); + if ((!match || typeof authenticateEnrollment !== "function") && (!runMatch || typeof authenticateRun !== "function")) return reject(); let listener; let target; try { - const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket), bootEpoch: runtime.bootEpoch }); + const vmIdentity = peerIdentity(socket); + const connection = runMatch + ? await authenticateRun({ runId: runMatch[1], vmIdentity, bootEpoch: runtime.bootEpoch }) + : await authenticateEnrollment({ token: match[1], vmIdentity, bootEpoch: runtime.bootEpoch }); // The production control plane returns the consumed enrollment binding. // Compare its VM and boot identities with the authenticated TLS peer and // this process so an adapter cannot accidentally attach another run's @@ -98,9 +102,10 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro } /** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ -export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPort, host = "127.0.0.1", port = 0, +export async function startVmTunnelHelper({ tls, enrollment, runId, proxyHost, proxyPort, host = "127.0.0.1", port = 0, now = Date.now, random = Math.random, wait = delay } = {}) { - if (!tls || typeof enrollment !== "function" || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { + const runBasedEnrollment = typeof runId === "string" && /^[0-9a-f-]{36}$/i.test(runId); + if (!tls || (!runBasedEnrollment && typeof enrollment !== "function") || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { throw new TypeError("VM tunnel helper requires TLS, enrollment and proxy address"); } const sockets = new Set(); @@ -113,15 +118,15 @@ export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPor while (!local.destroyed) { let remote; try { - const grant = await enrollment(); + const grant = runBasedEnrollment ? undefined : await enrollment(); if (local.destroyed) return; - if (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token)) throw new Error("fresh enrollment unavailable"); + if (!runBasedEnrollment && (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token))) throw new Error("fresh enrollment unavailable"); remote = connectTls({ ...tls, host: proxyHost, port: proxyPort, servername: tls.servername ?? proxyHost, rejectUnauthorized: true }); sockets.add(remote); const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); try { await once(remote, "secureConnect"); } finally { clearTimeout(connectTimer); } - remote.write(`FACTORIZE-TUNNEL/1 ${grant.token}\n`); + remote.write(runBasedEnrollment ? `FACTORIZE-TUNNEL/2 ${runId}\n` : `FACTORIZE-TUNNEL/1 ${grant.token}\n`); await waitForTunnelAcceptance(remote); local.pipe(remote); remote.pipe(local); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 56cc2b26..16e0ab93 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, issueProxyEnrollment, registerProxyBoot } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, registerProxyBoot } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -44,3 +44,59 @@ test("proxy control client requests and consumes a one-use enrollment over authe ]); assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${controlToken}` && request.init.redirect === "error")); }); + +test("proxy enrollment authenticator binds issued and consumed state to the mTLS VM and current boot", async () => { + const controlToken = `fzc_${"d".repeat(43)}`; + const requests = []; + const grants = [{ id: "tool-1", origin: "https://api.example.test", credential: "synthetic-secret" }]; + const fetchImpl = async (url, init) => { + const request = { url: String(url), init, body: JSON.parse(init.body) }; + requests.push(request); + if (requests.length === 1) return Response.json({ enrollmentId: "enrollment-1", token: "e".repeat(43), grantRevision: 7 }); + return Response.json({ consumed: true, binding: { + tenantId: "11111111-1111-4111-8111-111111111111", + runId: request.body.runId, + vmIdentity: request.body.vmIdentity, + proxyIdentity: "fly:proxy-1", + bootEpoch, + enrollmentId: "enrollment-1", + grantRevision: 7, + } }); + }; + const authenticateRun = createProxyEnrollmentAuthenticator({ + controlOrigin: "https://factorize.example", + controlCredential: controlToken, + bootEpoch, + fetchImpl, + resolveGrants: async binding => { + assert.equal(binding.runId, "33333333-3333-4333-8333-333333333333"); + return grants; + }, + }); + const connection = await authenticateRun({ + runId: "33333333-3333-4333-8333-333333333333", + vmIdentity: "fly:vm-1", + bootEpoch, + }); + assert.deepEqual(requests.map(request => request.url), [ + "https://factorize.example/internal/credential-proxy/enroll", + "https://factorize.example/internal/credential-proxy/consume-enrollment", + ]); + assert.deepEqual(requests.map(request => request.body), [ + { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1" }, + { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1", enrollmentToken: "e".repeat(43) }, + ]); + assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${controlToken}`)); + assert.equal(connection.authenticated, true); + assert.equal(connection.enrollmentConsumed, true); + assert.equal(connection.vmId, "fly:vm-1"); + assert.equal(connection.bootEpoch, bootEpoch); + assert.equal(connection.connectionId, "enrollment-1"); + assert.deepEqual(connection.grants, grants); + await assert.rejects(authenticateRun({ + runId: "33333333-3333-4333-8333-333333333333", + vmIdentity: "fly:vm-1", + bootEpoch: "44444444-4444-4444-8444-444444444444", + }), /invalid proxy enrollment identity/); + assert.equal(requests.length, 2, "a stale boot cannot issue another enrollment"); +}); diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 70a862df..0236b0b7 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -222,3 +222,47 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); } }); + +test("VM helper sends only run identity while the proxy performs authenticated enrollment", { timeout: 5_000 }, async () => { + const runtime = createProxyRuntime(); + const authenticatedRuns = []; + const tunnel = createAuthenticatedTunnelServer({ + tls: { key, cert, ca: cert }, + runtime, + authenticateRun: async ({ runId, vmIdentity, bootEpoch }) => { + authenticatedRuns.push({ runId, vmIdentity, bootEpoch }); + return { + authenticated: true, + enrollmentConsumed: true, + connectionId: "enrollment-from-control-api", + runId, + vmId: vmIdentity, + vmIdentity, + bootEpoch, + grants: [], + }; + }, + }); + await tunnel.listening; + runtime.unlock("synthetic-unlock-key"); + const local = await startVmTunnelHelper({ + tls: { key, cert, ca: cert, servername: "api.example.test" }, + runId: "33333333-3333-4333-8333-333333333333", + proxyHost: "127.0.0.1", + proxyPort: tunnel.address().port, + }); + try { + const response = await fetch(`http://127.0.0.1:${local.address.port}/unrecognized`); + assert.equal(response.status, 404); + assert.equal(authenticatedRuns.length, 1); + assert.deepEqual(authenticatedRuns[0], { + runId: "33333333-3333-4333-8333-333333333333", + vmIdentity: "api.example.test", + bootEpoch: runtime.bootEpoch, + }); + } finally { + await local.close(); + await tunnel.close(); + await runtime.shutdown(); + } +}); From b4462a2d7e98c5724cf5c57ba3429c72260c8464 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 20:42:29 +0000 Subject: [PATCH 012/102] Wire proxy control enrollment into tunnel startup --- .../credential-proxy/src/tunnel-transport.mjs | 19 +++++++ .../test/proxy-listener.e2e.test.mjs | 55 +++++++++++-------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index c68f5857..e670f554 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -1,6 +1,7 @@ import { connect as connectTls, createServer as createTlsServer } from "node:tls"; import { connect as connectTcp, createServer as createTcpServer } from "node:net"; import { once } from "node:events"; +import { createProxyEnrollmentAuthenticator } from "./control-client.mjs"; const HANDSHAKE_LIMIT = 4096; const HANDSHAKE_TIMEOUT_MS = 10_000; @@ -101,6 +102,24 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro }; } +/** Start the proxy's production-shaped mTLS listener using its boot-registered + * control credential and process-local decrypted integration grants. */ +export async function startAuthenticatedProxyTunnel({ + tls, runtime, controlOrigin, controlCredential, resolveGrants, fetchImpl = fetch, host = "127.0.0.1", port = 0, +} = {}) { + const authenticateRun = createProxyEnrollmentAuthenticator({ + controlOrigin, controlCredential, bootEpoch: runtime?.bootEpoch, resolveGrants, fetchImpl, + }); + const tunnel = createAuthenticatedTunnelServer({ tls, runtime, authenticateRun, host, port }); + try { + await tunnel.listening; + return tunnel; + } catch (error) { + await tunnel.close().catch(() => {}); + throw error; + } +} + /** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ export async function startVmTunnelHelper({ tls, enrollment, runId, proxyHost, proxyPort, host = "127.0.0.1", port = 0, now = Date.now, random = Math.random, wait = delay } = {}) { diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 0236b0b7..59785b0c 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -7,7 +7,7 @@ import { test } from "node:test"; import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; import { createAuthenticatedProxyListener } from "../src/proxy-listener.mjs"; import { createProxyRuntime } from "../src/runtime.mjs"; -import { createAuthenticatedTunnelServer, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; +import { createAuthenticatedTunnelServer, startAuthenticatedProxyTunnel, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; const fixtures = new URL("./fixtures/", import.meta.url); const key = await readFile(new URL("upstream-key.pem", fixtures)); @@ -225,41 +225,52 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante test("VM helper sends only run identity while the proxy performs authenticated enrollment", { timeout: 5_000 }, async () => { const runtime = createProxyRuntime(); - const authenticatedRuns = []; - const tunnel = createAuthenticatedTunnelServer({ + const requests = []; + const enrollmentId = "11111111-1111-4111-8111-111111111111"; + const runId = "33333333-3333-4333-8333-333333333333"; + const tunnel = await startAuthenticatedProxyTunnel({ tls: { key, cert, ca: cert }, runtime, - authenticateRun: async ({ runId, vmIdentity, bootEpoch }) => { - authenticatedRuns.push({ runId, vmIdentity, bootEpoch }); - return { - authenticated: true, - enrollmentConsumed: true, - connectionId: "enrollment-from-control-api", + controlOrigin: "https://factorize.example", + controlCredential: `fzc_${"c".repeat(43)}`, + fetchImpl: async (url, init) => { + const body = JSON.parse(init.body); + requests.push({ url: String(url), authorization: init.headers.Authorization, body }); + if (String(url).endsWith("/enroll")) { + return Response.json({ enrollmentId, token: "e".repeat(43), grantRevision: 7 }); + } + return Response.json({ consumed: true, binding: { + tenantId: "22222222-2222-4222-8222-222222222222", runId, - vmId: vmIdentity, - vmIdentity, - bootEpoch, - grants: [], - }; + vmIdentity: "api.example.test", + proxyIdentity: "proxy-fixture", + bootEpoch: runtime.bootEpoch, + enrollmentId, + grantRevision: 7, + } }); + }, + resolveGrants: async binding => { + assert.equal(binding.runId, runId); + assert.equal(binding.bootEpoch, runtime.bootEpoch); + return []; }, }); - await tunnel.listening; runtime.unlock("synthetic-unlock-key"); const local = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, - runId: "33333333-3333-4333-8333-333333333333", + runId, proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, }); try { const response = await fetch(`http://127.0.0.1:${local.address.port}/unrecognized`); assert.equal(response.status, 404); - assert.equal(authenticatedRuns.length, 1); - assert.deepEqual(authenticatedRuns[0], { - runId: "33333333-3333-4333-8333-333333333333", - vmIdentity: "api.example.test", - bootEpoch: runtime.bootEpoch, - }); + assert.deepEqual(requests.map(({ url, authorization, body }) => ({ + path: new URL(url).pathname, authorization, body, + })), [ + { path: "/internal/credential-proxy/enroll", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test" } }, + { path: "/internal/credential-proxy/consume-enrollment", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test", enrollmentToken: "e".repeat(43) } }, + ]); } finally { await local.close(); await tunnel.close(); From ef4b95262ef0d4e8d785a7cadc415f23d2fd8ff8 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:06:25 +0000 Subject: [PATCH 013/102] Retain proxy control credential after boot registration --- .../src/bootstrap-control.mjs | 7 ++++-- packages/credential-proxy/src/main.mjs | 25 +++++++++++++++---- .../test/bootstrap-control.test.mjs | 15 ++++++++--- .../test/runtime-process.test.mjs | 12 ++++----- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/packages/credential-proxy/src/bootstrap-control.mjs b/packages/credential-proxy/src/bootstrap-control.mjs index 80e02fec..332bdcc0 100644 --- a/packages/credential-proxy/src/bootstrap-control.mjs +++ b/packages/credential-proxy/src/bootstrap-control.mjs @@ -6,8 +6,8 @@ import { registerProxyBoot } from "./control-client.mjs"; const MAX_BOOTSTRAP_BYTES = 4096; /** Local-only receiver for a provider exec channel; credentials never enter env or files. */ -export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot } = {}) { - if (!runtime || typeof register !== "function") throw new TypeError("proxy bootstrap control requires a runtime and registrar"); +export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot, onRegistered = () => {} } = {}) { + if (!runtime || typeof register !== "function" || typeof onRegistered !== "function") throw new TypeError("proxy bootstrap control requires a runtime, registrar, and registration callback"); await mkdir(dirname(socketPath), { recursive: true, mode: 0o700 }); await chmod(dirname(socketPath), 0o700); await rm(socketPath, { force: true }); @@ -43,6 +43,9 @@ export async function startProxyBootstrapControl({ runtime, socketPath = "/run/f let credential = request.credential; try { const registration = await register({ controlOrigin: request.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + // Keep the exchanged proxy credential inside the proxy process. The + // one-shot startup command receives only the public registration result. + await onRegistered(registration, { controlOrigin: request.controlOrigin, bootEpoch: runtime.bootEpoch }); finish({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { finish({ error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 3a7750ae..d63c3454 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -7,19 +7,33 @@ import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; const epochIndex = process.argv.indexOf("--boot-epoch"); const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; const runtime = createProxyRuntime({ bootEpoch }); -const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET }); +let controlCredential; +let controlOrigin; +const bootstrapControl = await startProxyBootstrapControl({ + runtime, + socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET, + onRegistered(registration, request) { + controlCredential = registration.controlCredential; + controlOrigin = request.controlOrigin; + }, +}); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); let closing = false; let shutdownDeadline; -let controlCredential; function respond(value) { process.stdout.write(`${JSON.stringify(value)}\n`); } +function respondStatus() { + respond({ type: "status", state: runtime.state, proxyControlRegistered: Boolean(controlCredential && controlOrigin) }); +} + async function stop() { if (closing) return; closing = true; + controlCredential = undefined; + controlOrigin = undefined; await runtime.shutdown(); await bootstrapControl.close(); lines.close(); @@ -38,19 +52,20 @@ lines.on("line", async (line) => { let command; try { command = JSON.parse(line); - if (command?.type === "status") respond({ type: "status", state: runtime.state }); + if (command?.type === "status") respondStatus(); else if (command?.type === "unlock") { runtime.unlock(command.key); - respond({ type: "status", state: runtime.state }); + respondStatus(); } else if (command?.type === "lock") { await runtime.lock(); - respond({ type: "status", state: runtime.state }); + respondStatus(); } else if (command?.type === "register-boot") { let credential = command.credential; command.credential = undefined; try { const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); controlCredential = registration.controlCredential; + controlOrigin = command.controlOrigin; respond({ type: "boot-registered", proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { respond({ type: "error", error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/test/bootstrap-control.test.mjs b/packages/credential-proxy/test/bootstrap-control.test.mjs index 2cae7dfb..ece8a05e 100644 --- a/packages/credential-proxy/test/bootstrap-control.test.mjs +++ b/packages/credential-proxy/test/bootstrap-control.test.mjs @@ -17,11 +17,11 @@ afterEach(async () => { directory = undefined; }); -async function fixture(register) { +async function fixture(register, onRegistered) { directory = await mkdtemp(join(tmpdir(), "factorize-proxy-startup-")); const socketPath = join(directory, "control.sock"); const runtime = { bootEpoch: epoch }; - service = await startProxyBootstrapControl({ runtime, socketPath, register }); + service = await startProxyBootstrapControl({ runtime, socketPath, register, onRegistered }); return socketPath; } @@ -37,10 +37,19 @@ async function request(socketPath, payload) { test("local startup control redeems the one-use credential and leaves a private socket", async () => { const calls = []; - const socketPath = await fixture(async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch }; }); + const registered = []; + const socketPath = await fixture( + async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }; }, + (value, request) => registered.push({ value, request }), + ); const result = await request(socketPath, { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); assert.deepEqual(result, { registered: true, proxyIdentity: "fly:app:machine", bootEpoch: epoch }); assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch }]); + assert.deepEqual(registered, [{ + value: { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }, + request: { controlOrigin: "https://factorize.example", bootEpoch: epoch }, + }]); + assert.equal(JSON.stringify(result).includes(token), false, "startup response never returns the proxy control credential"); assert.equal((await stat(socketPath)).mode & 0o777, 0o600); }); diff --git a/packages/credential-proxy/test/runtime-process.test.mjs b/packages/credential-proxy/test/runtime-process.test.mjs index 87ebeb61..1e4d5d90 100644 --- a/packages/credential-proxy/test/runtime-process.test.mjs +++ b/packages/credential-proxy/test/runtime-process.test.mjs @@ -61,8 +61,8 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts await writeFile(join(directory, "public-metadata.json"), JSON.stringify({ displayName: "fixture" })); const first = await startProxy(directory); processes.push(first); - assert.deepEqual(await first.command({ type: "status" }), { type: "status", state: "locked" }); - assert.deepEqual(await first.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked" }); + assert.deepEqual(await first.command({ type: "status" }), { type: "status", state: "locked", proxyControlRegistered: false }); + assert.deepEqual(await first.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked", proxyControlRegistered: false }); first.child.kill("SIGTERM"); await waitFor(() => first.child.exitCode !== null || first.child.signalCode !== null, "first process stop"); assert.equal(first.child.exitCode, 0); @@ -70,22 +70,22 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts const restarted = await startProxy(directory); processes.push(restarted); assert.notEqual(restarted.bootEpoch, first.bootEpoch, "every process restart gets a fresh enrollment epoch"); - assert.deepEqual(await restarted.command({ type: "status" }), { type: "status", state: "locked" }); - assert.deepEqual(await restarted.command({ type: "unlock", key: tokenMarker }), { type: "status", state: "unlocked" }); + assert.deepEqual(await restarted.command({ type: "status" }), { type: "status", state: "locked", proxyControlRegistered: false }); + assert.deepEqual(await restarted.command({ type: "unlock", key: tokenMarker }), { type: "status", state: "unlocked", proxyControlRegistered: false }); restarted.child.stdin.write('{"type":"stop"}\n'); await waitFor(() => restarted.child.exitCode !== null || restarted.child.signalCode !== null, "command shutdown"); assert.equal(restarted.child.exitCode, 0); const crashed = await startProxy(directory); processes.push(crashed); - assert.deepEqual(await crashed.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked" }); + assert.deepEqual(await crashed.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked", proxyControlRegistered: false }); crashed.child.kill("SIGKILL"); await waitFor(() => crashed.child.exitCode !== null || crashed.child.signalCode !== null, "crash stop"); assert.equal(crashed.child.signalCode, "SIGKILL"); const afterCrash = await startProxy(directory); processes.push(afterCrash); - assert.deepEqual(await afterCrash.command({ type: "status" }), { type: "status", state: "locked" }); + assert.deepEqual(await afterCrash.command({ type: "status" }), { type: "status", state: "locked", proxyControlRegistered: false }); afterCrash.child.kill("SIGTERM"); await waitFor(() => afterCrash.child.exitCode !== null || afterCrash.child.signalCode !== null, "post-crash shutdown"); assert.equal(afterCrash.child.exitCode, 0); From 59e6beaa7e60129ffd410849033e223bb646350a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:29:12 +0000 Subject: [PATCH 014/102] Authenticate tunnel enrollment with run credential --- packages/api/src/proxy-control.ts | 16 +++++++-- packages/api/test/proxy-control.test.ts | 34 +++++++++++++++++-- .../credential-proxy/src/control-client.mjs | 6 ++-- .../credential-proxy/src/tunnel-transport.mjs | 11 +++--- .../test/control-client.test.mjs | 14 +++++--- .../test/proxy-listener.e2e.test.mjs | 6 ++-- 6 files changed, 68 insertions(+), 19 deletions(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index 695779cc..fbc1d125 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -1,6 +1,7 @@ import { databaseFor } from "./postgres/database"; import { ProxyControlRepository } from "./postgres/proxy-control-repository"; import { RunEnrollmentRepository } from "./postgres/run-enrollment-repository"; +import { authenticateRunCredential } from "./run-credentials"; const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "Cache-Control": "no-store" } }); @@ -41,9 +42,18 @@ export async function proxyControlFetch(request: Request, env: Parameters !["runId", "vmIdentity"].includes(key)) - || typeof (body as any).runId !== "string" || typeof (body as any).vmIdentity !== "string") return json({ error: "invalid_request" }, 400); - const enrollment = await enrollments.issue({ tenantId: proxy.tenantId, runId: (body as any).runId, vmIdentity: (body as any).vmIdentity, proxyIdentity: proxy.proxyIdentity, bootEpoch: proxy.bootEpoch }, new Date(Date.now() + 30_000)); + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity", "runCredential"].includes(key)) + || typeof (body as any).runId !== "string" || typeof (body as any).vmIdentity !== "string" || typeof (body as any).runCredential !== "string" + || (body as any).vmIdentity.length === 0 || (body as any).vmIdentity.length > 255) return json({ error: "invalid_request" }, 400); + // A proxy-authenticated request alone must not let a peer that can reach + // the proxy claim another active run. The VM's distinct run credential is + // validated against the current run and tenant before issuing enrollment. + const runAuth = await authenticateRunCredential(new Request(request.url, { + headers: { Authorization: `Bearer ${(body as any).runCredential}` }, + }), env); + if (!runAuth || runAuth.runId !== (body as any).runId || runAuth.tenantId !== proxy.tenantId + || !runAuth.scopes.includes("integrations:read")) return json({ error: "enrollment_unavailable" }, 403); + const enrollment = await enrollments.issue({ tenantId: proxy.tenantId, runId: runAuth.runId, vmIdentity: (body as any).vmIdentity, proxyIdentity: proxy.proxyIdentity, bootEpoch: proxy.bootEpoch }, new Date(Date.now() + 30_000)); return enrollment ? json(enrollment) : json({ error: "enrollment_unavailable" }, 403); } if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity", "enrollmentToken"].includes(key)) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index c0a33b1e..2c8d91dc 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -3,6 +3,8 @@ import { proxyControlFetch } from "../src/proxy-control"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; +const runId = "33333333-3333-4333-8333-333333333333"; +const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R".repeat(43)}`; function envFor({ credentialFound = true, stateSaved = true } = {}) { const transactionQuery = vi.fn(async (sql: string) => { @@ -12,7 +14,9 @@ function envFor({ credentialFound = true, stateSaved = true } = {}) { }); const query = vi.fn(async (sql: string) => sql.includes("proxy_token_digest") ? { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }] : [], rowCount: credentialFound ? 1 : 0 } - : { rows: [], rowCount: 0 }); + : sql.includes("FROM app.run_credentials") + ? { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", user_id: "owner-fixture", session_version: 1, job_id: "job-fixture", scopes: ["integrations:read"] }] : [], rowCount: credentialFound ? 1 : 0 } + : { rows: [], rowCount: 0 }); return { DATABASE: { pool: { query }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery, query } as any; } @@ -67,14 +71,40 @@ describe("dedicated proxy boot registration endpoint", () => { const response = await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", - body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123" }, + body: { runId, vmIdentity: "fly:vm-123", runCredential }, }), env); expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ grantRevision: 3, enrollmentId: expect.any(String), token: expect.any(String) }); expect(env.query).toHaveBeenCalled(); + expect(env.query.mock.calls.some((call: any[]) => call[0].includes("FROM app.run_credentials"))).toBe(true); expect(env.transactionQuery.mock.calls[0][0]).toContain("JOIN app.credential_proxy_states"); }); + it("refuses enrollment when the run credential is absent, invalid, cross-tenant, mismatched, or lacks integration scope", async () => { + for (const body of [ + { runId, vmIdentity: "fly:vm-123" }, + { runId, vmIdentity: "fly:vm-123", runCredential: "fzt_run_invalid" }, + { runId: "44444444-4444-4444-8444-444444444444", vmIdentity: "fly:vm-123", runCredential }, + ]) { + const env = envFor(); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body }), env)).status).toBe(403); + expect(env.transactionQuery).not.toHaveBeenCalled(); + } + const crossTenant = envFor(); + crossTenant.query.mockImplementation(async (sql: string) => sql.includes("proxy_token_digest") + ? { rows: [{ tenant_id: "other-tenant", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 } + : { rows: [], rowCount: 0 }); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body: { runId, vmIdentity: "fly:vm-123", runCredential } }), crossTenant)).status).toBe(403); + const noScope = envFor(); + noScope.query.mockImplementation(async (sql: string) => sql.includes("proxy_token_digest") + ? { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 } + : sql.includes("FROM app.run_credentials") + ? { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", user_id: "owner-fixture", session_version: 1, job_id: "job-fixture", scopes: ["runs:read"] }], rowCount: 1 } + : { rows: [], rowCount: 0 }); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body: { runId, vmIdentity: "fly:vm-123", runCredential } }), noScope)).status).toBe(403); + expect(noScope.transactionQuery).not.toHaveBeenCalled(); + }); + it("consumes enrollment against the authenticated proxy boot and rejects untrusted extra bindings", async () => { const env = envFor(); env.transactionQuery.mockImplementation(async (sql: string) => sql.includes("UPDATE app.credential_run_enrollments") diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index b5216f10..faf9ad3e 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -67,11 +67,11 @@ export function createProxyEnrollmentAuthenticator({ controlOrigin, controlCrede if (typeof resolveGrants !== "function") throw new TypeError("proxy enrollment requires a local grant resolver"); validateOrigin(controlOrigin); let credential = controlCredential; - return async function authenticateRun({ runId, vmIdentity, bootEpoch: requestedBootEpoch }) { + return async function authenticateRun({ runId, vmIdentity, runCredential, bootEpoch: requestedBootEpoch }) { if (!uuidPattern.test(runId ?? "") || typeof vmIdentity !== "string" || !vmIdentity.trim() || vmIdentity.length > 255 - || requestedBootEpoch !== bootEpoch) throw new Error("invalid proxy enrollment identity"); + || !/^fzt_run_[A-Za-z0-9_-]{43,200}$/.test(runCredential ?? "") || requestedBootEpoch !== bootEpoch) throw new Error("invalid proxy enrollment identity"); const options = { controlOrigin, token: credential, fetchImpl }; - const requested = { runId, vmIdentity }; + const requested = { runId, vmIdentity, runCredential }; const enrollment = await issueProxyEnrollment({ ...options, body: requested }); const binding = await consumeProxyEnrollment({ ...options, body: { ...requested, enrollmentToken: enrollment.token } }); if (binding.runId !== runId || binding.vmIdentity !== vmIdentity || binding.bootEpoch !== bootEpoch diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index e670f554..28d9d9a9 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -47,14 +47,14 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro const line = handshake.subarray(0, end).toString("utf8").replace(/\r$/, ""); const rest = handshake.subarray(end + 1); const match = /^FACTORIZE-TUNNEL\/1 ([A-Za-z0-9_-]{32,128})$/.exec(line); - const runMatch = /^FACTORIZE-TUNNEL\/2 ([0-9a-f-]{36})$/i.exec(line); + const runMatch = /^FACTORIZE-TUNNEL\/2 ([0-9a-f-]{36}) (fzt_run_[A-Za-z0-9_-]{43,200})$/i.exec(line); if ((!match || typeof authenticateEnrollment !== "function") && (!runMatch || typeof authenticateRun !== "function")) return reject(); let listener; let target; try { const vmIdentity = peerIdentity(socket); const connection = runMatch - ? await authenticateRun({ runId: runMatch[1], vmIdentity, bootEpoch: runtime.bootEpoch }) + ? await authenticateRun({ runId: runMatch[1], runCredential: runMatch[2], vmIdentity, bootEpoch: runtime.bootEpoch }) : await authenticateEnrollment({ token: match[1], vmIdentity, bootEpoch: runtime.bootEpoch }); // The production control plane returns the consumed enrollment binding. // Compare its VM and boot identities with the authenticated TLS peer and @@ -121,10 +121,11 @@ export async function startAuthenticatedProxyTunnel({ } /** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ -export async function startVmTunnelHelper({ tls, enrollment, runId, proxyHost, proxyPort, host = "127.0.0.1", port = 0, +export async function startVmTunnelHelper({ tls, enrollment, runId, runCredential, proxyHost, proxyPort, host = "127.0.0.1", port = 0, now = Date.now, random = Math.random, wait = delay } = {}) { const runBasedEnrollment = typeof runId === "string" && /^[0-9a-f-]{36}$/i.test(runId); - if (!tls || (!runBasedEnrollment && typeof enrollment !== "function") || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { + if (!tls || (!runBasedEnrollment && typeof enrollment !== "function") || (runBasedEnrollment && !/^fzt_run_[A-Za-z0-9_-]{43,200}$/.test(runCredential ?? "")) + || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { throw new TypeError("VM tunnel helper requires TLS, enrollment and proxy address"); } const sockets = new Set(); @@ -145,7 +146,7 @@ export async function startVmTunnelHelper({ tls, enrollment, runId, proxyHost, p const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); try { await once(remote, "secureConnect"); } finally { clearTimeout(connectTimer); } - remote.write(runBasedEnrollment ? `FACTORIZE-TUNNEL/2 ${runId}\n` : `FACTORIZE-TUNNEL/1 ${grant.token}\n`); + remote.write(runBasedEnrollment ? `FACTORIZE-TUNNEL/2 ${runId} ${runCredential}\n` : `FACTORIZE-TUNNEL/1 ${grant.token}\n`); await waitForTunnelAcceptance(remote); local.pipe(remote); remote.pipe(local); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 16e0ab93..1501dc9f 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -4,6 +4,8 @@ import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyE const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; +const runId = "33333333-3333-4333-8333-333333333333"; +const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R".repeat(43)}`; test("boot registration uses a one-call control credential, HTTPS, exact path and fresh process epoch", async () => { let request; @@ -74,7 +76,8 @@ test("proxy enrollment authenticator binds issued and consumed state to the mTLS }, }); const connection = await authenticateRun({ - runId: "33333333-3333-4333-8333-333333333333", + runId, + runCredential, vmIdentity: "fly:vm-1", bootEpoch, }); @@ -83,8 +86,8 @@ test("proxy enrollment authenticator binds issued and consumed state to the mTLS "https://factorize.example/internal/credential-proxy/consume-enrollment", ]); assert.deepEqual(requests.map(request => request.body), [ - { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1" }, - { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1", enrollmentToken: "e".repeat(43) }, + { runId, vmIdentity: "fly:vm-1", runCredential }, + { runId, vmIdentity: "fly:vm-1", enrollmentToken: "e".repeat(43) }, ]); assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${controlToken}`)); assert.equal(connection.authenticated, true); @@ -94,9 +97,12 @@ test("proxy enrollment authenticator binds issued and consumed state to the mTLS assert.equal(connection.connectionId, "enrollment-1"); assert.deepEqual(connection.grants, grants); await assert.rejects(authenticateRun({ - runId: "33333333-3333-4333-8333-333333333333", + runId, + runCredential, vmIdentity: "fly:vm-1", bootEpoch: "44444444-4444-4444-8444-444444444444", }), /invalid proxy enrollment identity/); assert.equal(requests.length, 2, "a stale boot cannot issue another enrollment"); + await assert.rejects(authenticateRun({ runId, vmIdentity: "fly:vm-1", bootEpoch }), /invalid proxy enrollment identity/); + assert.equal(requests.length, 2, "a missing job run credential cannot issue enrollment"); }); diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 59785b0c..5098aed7 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -223,11 +223,12 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante } }); -test("VM helper sends only run identity while the proxy performs authenticated enrollment", { timeout: 5_000 }, async () => { +test("VM helper authenticates the active run while the proxy performs authenticated enrollment", { timeout: 5_000 }, async () => { const runtime = createProxyRuntime(); const requests = []; const enrollmentId = "11111111-1111-4111-8111-111111111111"; const runId = "33333333-3333-4333-8333-333333333333"; + const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R".repeat(43)}`; const tunnel = await startAuthenticatedProxyTunnel({ tls: { key, cert, ca: cert }, runtime, @@ -259,6 +260,7 @@ test("VM helper sends only run identity while the proxy performs authenticated e const local = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, runId, + runCredential, proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, }); @@ -268,7 +270,7 @@ test("VM helper sends only run identity while the proxy performs authenticated e assert.deepEqual(requests.map(({ url, authorization, body }) => ({ path: new URL(url).pathname, authorization, body, })), [ - { path: "/internal/credential-proxy/enroll", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test" } }, + { path: "/internal/credential-proxy/enroll", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test", runCredential } }, { path: "/internal/credential-proxy/consume-enrollment", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test", enrollmentToken: "e".repeat(43) } }, ]); } finally { From 2188c7db369aee53d96aac3b818d6569d1485255 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:30:26 +0000 Subject: [PATCH 015/102] Keep run credential out of consume request --- packages/credential-proxy/src/control-client.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index faf9ad3e..192acaa4 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -73,7 +73,7 @@ export function createProxyEnrollmentAuthenticator({ controlOrigin, controlCrede const options = { controlOrigin, token: credential, fetchImpl }; const requested = { runId, vmIdentity, runCredential }; const enrollment = await issueProxyEnrollment({ ...options, body: requested }); - const binding = await consumeProxyEnrollment({ ...options, body: { ...requested, enrollmentToken: enrollment.token } }); + const binding = await consumeProxyEnrollment({ ...options, body: { runId, vmIdentity, enrollmentToken: enrollment.token } }); if (binding.runId !== runId || binding.vmIdentity !== vmIdentity || binding.bootEpoch !== bootEpoch || typeof binding.tenantId !== "string" || typeof binding.proxyIdentity !== "string" || binding.enrollmentId !== enrollment.enrollmentId || binding.grantRevision !== enrollment.grantRevision) { From 78b86ee0904018419dd176ddf3ca5cf50ec11ee9 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:30:39 +0000 Subject: [PATCH 016/102] Cover invalid run enrollment bindings --- packages/api/test/proxy-control.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 2c8d91dc..f36674b8 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -81,13 +81,13 @@ describe("dedicated proxy boot registration endpoint", () => { }); it("refuses enrollment when the run credential is absent, invalid, cross-tenant, mismatched, or lacks integration scope", async () => { - for (const body of [ - { runId, vmIdentity: "fly:vm-123" }, - { runId, vmIdentity: "fly:vm-123", runCredential: "fzt_run_invalid" }, - { runId: "44444444-4444-4444-8444-444444444444", vmIdentity: "fly:vm-123", runCredential }, - ]) { + for (const [body, expectedStatus] of [ + [{ runId, vmIdentity: "fly:vm-123" }, 400], + [{ runId, vmIdentity: "fly:vm-123", runCredential: "fzt_run_invalid" }, 403], + [{ runId: "44444444-4444-4444-8444-444444444444", vmIdentity: "fly:vm-123", runCredential }, 403], + ] as const) { const env = envFor(); - expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body }), env)).status).toBe(403); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body }), env)).status).toBe(expectedStatus); expect(env.transactionQuery).not.toHaveBeenCalled(); } const crossTenant = envFor(); From c46c3a2b7397ed9bf4e66002232e32c0dee05ced Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:32:47 +0000 Subject: [PATCH 017/102] Narrow active run identity before enrollment --- packages/api/src/proxy-control.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index fbc1d125..e38a6c68 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -51,7 +51,7 @@ export async function proxyControlFetch(request: Request, env: Parameters Date: Wed, 11 Nov 2026 21:54:30 +0000 Subject: [PATCH 018/102] Close proxy tunnels when a run is revoked --- packages/credential-proxy/src/runtime.mjs | 22 ++++++- .../test/runtime-revocation.test.mjs | 57 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 packages/credential-proxy/test/runtime-revocation.test.mjs diff --git a/packages/credential-proxy/src/runtime.mjs b/packages/credential-proxy/src/runtime.mjs index 06cf340c..d7f76cf9 100644 --- a/packages/credential-proxy/src/runtime.mjs +++ b/packages/credential-proxy/src/runtime.mjs @@ -16,10 +16,13 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL let shutdown = false; let locking = false; const listeners = new Set(); + const listenerRuns = new Map(); + const revokedRuns = new Set(); async function closeListeners() { const current = [...listeners]; listeners.clear(); + listenerRuns.clear(); await Promise.allSettled(current.map((listener) => listener.close())); } @@ -49,20 +52,33 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL if (shutdown) throw new Error("runtime is shutting down"); if (state !== "unlocked") throw new Error("proxy is locked"); if (connection?.bootEpoch !== bootEpoch) throw new Error("enrollment belongs to a different proxy boot"); + if (typeof connection.runId !== "string" || !connection.runId) throw new TypeError("authenticated connection requires a run identity"); + if (revokedRuns.has(connection.runId)) throw new Error("run grants have been revoked"); // The transport adapter calls this only after atomically consuming and // authenticating enrollment. Grants are the consumed, current run state. const listener = await listenerFactory({ connection }); - if (state !== "unlocked" || shutdown) { + if (state !== "unlocked" || shutdown || revokedRuns.has(connection.runId)) { await listener.close(); - throw new Error("proxy locked while accepting the connection"); + throw new Error(revokedRuns.has(connection.runId) ? "run grants have been revoked" : "proxy locked while accepting the connection"); } listeners.add(listener); + // Keep the run identity beside the listener so a control-plane revocation + // can tear down every established connection without retaining grants. + listenerRuns.set(listener, connection.runId); return listener; }, async disconnect(connectionId) { const matching = [...listeners].filter((listener) => listener.connectionId === connectionId); await Promise.allSettled(matching.map((listener) => listener.close())); - for (const listener of matching) listeners.delete(listener); + for (const listener of matching) { listeners.delete(listener); listenerRuns.delete(listener); } + }, + async revokeRun(runId) { + if (typeof runId !== "string" || !runId) throw new TypeError("run identity is required for revocation"); + revokedRuns.add(runId); + const matching = [...listeners].filter((listener) => listenerRuns.get(listener) === runId); + await Promise.allSettled(matching.map((listener) => listener.close())); + for (const listener of matching) { listeners.delete(listener); listenerRuns.delete(listener); } + return matching.length; }, async shutdown() { shutdown = true; diff --git a/packages/credential-proxy/test/runtime-revocation.test.mjs b/packages/credential-proxy/test/runtime-revocation.test.mjs new file mode 100644 index 00000000..76bd6f13 --- /dev/null +++ b/packages/credential-proxy/test/runtime-revocation.test.mjs @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createProxyRuntime } from "../src/runtime.mjs"; + +const bootEpoch = "11111111-1111-4111-8111-111111111111"; + +function connection(runId, connectionId) { + return { authenticated: true, enrollmentConsumed: true, runId, connectionId, bootEpoch }; +} + +test("run revocation closes every established connection and rejects later enrollment", async () => { + const closed = []; + const runtime = createProxyRuntime({ + bootEpoch, + listenerFactory: async ({ connection: value }) => ({ + connectionId: value.connectionId, + close: async () => { closed.push(value.connectionId); }, + }), + }); + runtime.unlock("synthetic-unlock-key"); + try { + await runtime.acceptAuthenticatedConnection(connection("run-a", "connection-a1")); + await runtime.acceptAuthenticatedConnection(connection("run-a", "connection-a2")); + await runtime.acceptAuthenticatedConnection(connection("run-b", "connection-b1")); + + assert.equal(await runtime.revokeRun("run-a"), 2); + assert.deepEqual(closed.sort(), ["connection-a1", "connection-a2"]); + await assert.rejects(runtime.acceptAuthenticatedConnection(connection("run-a", "connection-a3")), /revoked/); + + const stillActive = await runtime.acceptAuthenticatedConnection(connection("run-b", "connection-b2")); + assert.equal(stillActive.connectionId, "connection-b2"); + assert.equal(await runtime.revokeRun("run-a"), 0); + assert.deepEqual(closed.sort(), ["connection-a1", "connection-a2"]); + } finally { + await runtime.shutdown(); + } +}); + +test("revocation racing listener creation closes the new listener before accepting it", async () => { + let resolveListener; + let closed = false; + const runtime = createProxyRuntime({ + bootEpoch, + listenerFactory: async () => new Promise(resolve => { resolveListener = resolve; }), + }); + runtime.unlock("synthetic-unlock-key"); + try { + const accepting = runtime.acceptAuthenticatedConnection(connection("run-race", "connection-race")); + await new Promise(resolve => setImmediate(resolve)); + await runtime.revokeRun("run-race"); + resolveListener({ connectionId: "connection-race", close: async () => { closed = true; } }); + await assert.rejects(accepting, /revoked/); + assert.equal(closed, true); + } finally { + await runtime.shutdown(); + } +}); From 68d348eceec14eb70fe4be3f7d4bc9733c4145d9 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:17:25 +0000 Subject: [PATCH 019/102] Add decrypted run integration grant resolver --- .../credential-proxy/src/decrypted-grants.mjs | 98 +++++++++++++++++++ .../test/decrypted-grants.test.mjs | 69 +++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 packages/credential-proxy/src/decrypted-grants.mjs create mode 100644 packages/credential-proxy/test/decrypted-grants.test.mjs diff --git a/packages/credential-proxy/src/decrypted-grants.mjs b/packages/credential-proxy/src/decrypted-grants.mjs new file mode 100644 index 00000000..683ff547 --- /dev/null +++ b/packages/credential-proxy/src/decrypted-grants.mjs @@ -0,0 +1,98 @@ +import { webcrypto } from "node:crypto"; + +const KEY_PREFIX = "fzk1."; +const KEY_BYTES = 32; +const NONCE_BYTES = 12; +const SALT = new TextEncoder().encode("factorize.unlock-key.v1"); +const INFO = new TextEncoder().encode("factorize.record.aes-256-gcm.v1"); +const encoder = new TextEncoder(); +const decoder = new TextDecoder("utf-8", { fatal: true }); + +function decodeBase64Url(value, label) { + if (typeof value !== "string" || !/^[A-Za-z0-9_-]+$/.test(value)) throw new TypeError(`malformed ${label}`); + const bytes = Buffer.from(value, "base64url"); + if (bytes.toString("base64url") !== value) throw new TypeError(`malformed ${label}`); + return new Uint8Array(bytes); +} + +function decodeStoredCiphertext(value) { + if (typeof value !== "string" || value.length > 65_536) throw new TypeError("malformed integration ciphertext"); + const parts = value.split("."); + if (parts.length !== 2 || !parts.every(part => /^[A-Za-z0-9+/]+={0,2}$/.test(part))) throw new TypeError("malformed integration ciphertext"); + const nonce = Buffer.from(parts[0], "base64"); + const ciphertext = Buffer.from(parts[1], "base64"); + if (nonce.toString("base64") !== parts[0] || ciphertext.toString("base64") !== parts[1] + || nonce.byteLength !== NONCE_BYTES || ciphertext.byteLength < 16) throw new TypeError("malformed integration ciphertext"); + return { nonce: new Uint8Array(nonce), ciphertext: new Uint8Array(ciphertext) }; +} + +function recordAad(ownerId, record) { + if (typeof ownerId !== "string" || !ownerId || typeof record?.id !== "string" || !record.id + || typeof record.origin !== "string" || !record.origin || !Number.isSafeInteger(record.revision) || record.revision < 1) { + throw new TypeError("malformed integration record binding"); + } + return encoder.encode(JSON.stringify(["factorize-record-v1", ownerId, record.id, "bearer", record.origin, record.revision])); +} + +async function deriveKey(key) { + if (typeof key !== "string" || !key.startsWith(KEY_PREFIX)) throw new TypeError("malformed unlock key"); + const raw = decodeBase64Url(key.slice(KEY_PREFIX.length), "unlock key"); + if (raw.byteLength !== KEY_BYTES) throw new TypeError("malformed unlock key"); + try { + const material = await webcrypto.subtle.importKey("raw", raw, "HKDF", false, ["deriveKey"]); + return await webcrypto.subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt: SALT, info: INFO }, material, + { name: "AES-GCM", length: 256 }, false, ["decrypt"]); + } finally { + raw.fill(0); + } +} + +function validateOrigin(origin) { + let parsed; + try { parsed = new URL(origin); } catch { throw new TypeError("invalid integration origin"); } + if (parsed.protocol !== "https:" || parsed.origin !== origin || parsed.username || parsed.password) throw new TypeError("invalid integration origin"); + return parsed.origin; +} + +/** + * Resolve only the selected bearer records from a current, server-decrypted + * snapshot. The API supplies ciphertext after removing its own encryption + * layer; this helper never sends the user's unlock key or plaintext upstream. + */ +export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resolveIntegrationIds } = {}) { + if (typeof readSnapshot !== "function" || typeof getUnlockKey !== "function" + || (resolveIntegrationIds !== undefined && typeof resolveIntegrationIds !== "function")) throw new TypeError("grant resolver requires snapshot and unlock-key readers"); + return async function resolveGrants(binding) { + if (!binding || typeof binding.tenantId !== "string" || !binding.tenantId + || (binding.integrationIds !== undefined && !Array.isArray(binding.integrationIds))) throw new TypeError("invalid run integration selection"); + const integrationIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding); + if (!Array.isArray(integrationIds) || integrationIds.some(id => typeof id !== "string" || !id) + || new Set(integrationIds).size !== integrationIds.length) throw new TypeError("invalid run integration selection"); + if (integrationIds.length === 0) return []; + const snapshot = await readSnapshot(binding); + if (!snapshot || !Array.isArray(snapshot.records) || typeof snapshot.ownerId !== "string" || !snapshot.ownerId) throw new Error("integration snapshot is unavailable"); + const selected = new Set(integrationIds); + const records = snapshot.records.filter(record => selected.has(record?.id) && record.kind === "bearer" && record.deletedAt == null); + if (records.length !== selected.size) throw new Error("selected integration is unavailable"); + const unlockKey = await getUnlockKey(); + if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); + const aesKey = await deriveKey(unlockKey); + const grants = []; + for (const record of records) { + const origin = validateOrigin(record.origin); + const { nonce, ciphertext } = decodeStoredCiphertext(record.encryptedPayload); + const plaintext = new Uint8Array(await webcrypto.subtle.decrypt({ name: "AES-GCM", iv: nonce, + additionalData: recordAad(snapshot.ownerId, record), tagLength: 128 }, aesKey, ciphertext)); + try { + const credential = decoder.decode(plaintext); + if (!credential || credential.length > 16_384 || /[\u0000-\u001f\u007f]/.test(credential)) throw new TypeError("invalid bearer credential"); + grants.push(Object.freeze({ id: record.id, origin, credential })); + } finally { + plaintext.fill(0); + nonce.fill(0); + ciphertext.fill(0); + } + } + return grants; + }; +} diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs new file mode 100644 index 00000000..d0f873aa --- /dev/null +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { webcrypto } from "node:crypto"; +import { test } from "node:test"; +import { createDecryptedGrantResolver } from "../src/decrypted-grants.mjs"; + +const encoder = new TextEncoder(); +const salt = encoder.encode("factorize.unlock-key.v1"); +const info = encoder.encode("factorize.record.aes-256-gcm.v1"); +const ownerId = "owner-fixture"; +const unlockKey = `fzk1.${Buffer.from(webcrypto.getRandomValues(new Uint8Array(32))).toString("base64url")}`; + +async function encryptCredential({ id, origin, revision, credential }) { + const raw = Buffer.from(unlockKey.slice("fzk1.".length), "base64url"); + const material = await webcrypto.subtle.importKey("raw", raw, "HKDF", false, ["deriveKey"]); + const aesKey = await webcrypto.subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt, info }, material, + { name: "AES-GCM", length: 256 }, false, ["encrypt"]); + const nonce = webcrypto.getRandomValues(new Uint8Array(12)); + const additionalData = encoder.encode(JSON.stringify(["factorize-record-v1", ownerId, id, "bearer", origin, revision])); + const ciphertext = new Uint8Array(await webcrypto.subtle.encrypt({ name: "AES-GCM", iv: nonce, additionalData, tagLength: 128 }, aesKey, encoder.encode(credential))); + return `${Buffer.from(nonce).toString("base64")}.${Buffer.from(ciphertext).toString("base64")}`; +} + +test("resolver decrypts only selected active bearer records using the shared record binding", async () => { + const first = { id: "integration-a", kind: "bearer", origin: "https://api.example.test", revision: 3, deletedAt: null }; + const second = { id: "integration-b", kind: "bearer", origin: "https://other.example.test", revision: 4, deletedAt: null }; + const records = [ + { ...first, encryptedPayload: await encryptCredential({ ...first, credential: "first-token" }) }, + { ...second, encryptedPayload: await encryptCredential({ ...second, credential: "second-token" }) }, + { id: "deleted", kind: "bearer", origin: "https://api.example.test", revision: 5, deletedAt: new Date(), encryptedPayload: "unused" }, + ]; + const resolver = createDecryptedGrantResolver({ + readSnapshot: async () => ({ ownerId, records }), + getUnlockKey: async () => unlockKey, + }); + assert.deepEqual(await resolver({ tenantId: "tenant-fixture", integrationIds: ["integration-b"] }), [ + { id: "integration-b", origin: "https://other.example.test", credential: "second-token" }, + ]); + assert.deepEqual(await resolver({ tenantId: "tenant-fixture", integrationIds: [] }), []); + const runResolver = createDecryptedGrantResolver({ + readSnapshot: async () => ({ ownerId, records }), + resolveIntegrationIds: async binding => binding.runId === "run-fixture" ? ["integration-a"] : [], + getUnlockKey: async () => unlockKey, + }); + assert.deepEqual(await runResolver({ tenantId: "tenant-fixture", runId: "run-fixture" }), [ + { id: "integration-a", origin: "https://api.example.test", credential: "first-token" }, + ]); +}); + +test("resolver rejects missing, deleted, ambiguous, malformed, and misbound grants", async () => { + const record = { id: "integration-a", kind: "bearer", origin: "https://api.example.test", revision: 3, deletedAt: null }; + const encryptedPayload = await encryptCredential({ ...record, credential: "synthetic-token" }); + const resolver = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [{ ...record, encryptedPayload }] }), getUnlockKey: async () => unlockKey }); + await assert.rejects(resolver({ tenantId: "tenant-fixture", integrationIds: ["missing"] }), /unavailable/); + await assert.rejects(resolver({ tenantId: "tenant-fixture", integrationIds: ["integration-a", "integration-a"] }), /selection/); + const tampered = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId: "other-owner", records: [{ ...record, encryptedPayload }] }), getUnlockKey: async () => unlockKey }); + await assert.rejects(tampered({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] })); + const locked = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [{ ...record, encryptedPayload }] }), getUnlockKey: async () => undefined }); + await assert.rejects(locked({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] }), /locked/); +}); + +test("resolver rejects non-HTTPS destinations and invalid credential bytes", async () => { + const badOrigin = { id: "integration-a", kind: "bearer", origin: "http://api.example.test", revision: 3, deletedAt: null, + encryptedPayload: await encryptCredential({ id: "integration-a", origin: "http://api.example.test", revision: 3, credential: "token" }) }; + const badOriginResolver = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [badOrigin] }), getUnlockKey: async () => unlockKey }); + await assert.rejects(badOriginResolver({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] }), /integration origin/); + const badCredential = { ...badOrigin, origin: "https://api.example.test", encryptedPayload: await encryptCredential({ id: "integration-a", origin: "https://api.example.test", revision: 3, credential: "bad\ncredential" }) }; + const badCredentialResolver = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [badCredential] }), getUnlockKey: async () => unlockKey }); + await assert.rejects(badCredentialResolver({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] }), /credential/); +}); From eb90329e7f73099d282ef9d676c29276812452fc Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:31:01 +0000 Subject: [PATCH 020/102] Expose selected integrations to authenticated proxy --- packages/api/src/proxy-control.ts | 24 ++++++++++++++++- packages/api/test/proxy-control.test.ts | 27 +++++++++++++++++++ .../credential-proxy/src/control-client.mjs | 21 +++++++++++++++ .../test/control-client.test.mjs | 16 ++++++++++- 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index e38a6c68..d3bb08b5 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -1,6 +1,7 @@ import { databaseFor } from "./postgres/database"; import { ProxyControlRepository } from "./postgres/proxy-control-repository"; import { RunEnrollmentRepository } from "./postgres/run-enrollment-repository"; +import { EncryptedIntegrationRepository } from "./postgres/encrypted-integration-repository"; import { authenticateRunCredential } from "./run-credentials"; const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "Cache-Control": "no-store" } }); @@ -8,7 +9,7 @@ const json = (body: unknown, status = 200) => Response.json(body, { status, head /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment"].includes(url.pathname)) return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; @@ -40,6 +41,27 @@ export async function proxyControlFetch(request: Request, env: Parameters !["runId", "grantRevision"].includes(key)) + || typeof (body as any).runId !== "string" || !Number.isSafeInteger((body as any).grantRevision) || (body as any).grantRevision < 0) return json({ error: "invalid_request" }, 400); + const database = databaseFor(env); + const active = await database.pool.query<{ execution_config: Record; grant_revision: string | number }>(`SELECT r.execution_config,g.grant_revision + FROM app.runs r JOIN app.credential_run_grants g ON g.tenant_id=r.tenant_id AND g.run_id=r.id + JOIN app.credential_proxy_states p ON p.tenant_id=g.tenant_id AND p.proxy_identity=g.proxy_identity AND p.boot_epoch=g.boot_epoch + WHERE r.tenant_id=$1 AND r.id=$2 AND r.state IN ('starting','running','blocked') + AND g.proxy_identity=$3 AND g.boot_epoch=$4 AND g.grant_revision=$5 AND g.revoked_at IS NULL + AND p.disabled_at IS NULL AND p.revoked_at IS NULL`, [proxy.tenantId, (body as any).runId, proxy.proxyIdentity, proxy.bootEpoch, (body as any).grantRevision]); + const run = active.rows[0]; + if (!run) return json({ error: "integration_snapshot_unavailable" }, 403); + const selected = run.execution_config?.portableIntegrationIds; + if (!Array.isArray(selected) || selected.some(id => typeof id !== "string") || new Set(selected).size !== selected.length) return json({ error: "integration_snapshot_unavailable" }, 403); + const records = selected.length + ? (await new EncryptedIntegrationRepository(database, proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("bearer")) + .filter(record => selected.includes(record.id)) + : []; + if (records.length !== selected.length) return json({ error: "integration_snapshot_unavailable" }, 403); + return json({ ownerId: proxy.tenantId, records }); + } const enrollments = new RunEnrollmentRepository(databaseFor(env)); if (url.pathname === "/internal/credential-proxy/enroll") { if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity", "runCredential"].includes(key)) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index f36674b8..e1c64539 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { proxyControlFetch } from "../src/proxy-control"; +import { encrypt } from "../src/crypto"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -120,4 +121,30 @@ describe("dedicated proxy boot registration endpoint", () => { const rejected = await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123", tenantId: "other" } }), env); expect(rejected.status).toBe(400); }); + + it("returns only the current run's selected user-encrypted bearer records to its registered proxy", async () => { + const integrationId = "55555555-5555-4555-8555-555555555555"; + const key = "test-server-encryption-key"; + const env = envFor(); + env.CREDENTIAL_ENCRYPTION_KEY = key; + env.query.mockImplementation(async (sql: string) => { + if (sql.includes("proxy_token_digest")) return { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 }; + if (sql.includes("FROM app.runs r")) return { rows: [{ execution_config: { portableIntegrationIds: [integrationId] }, grant_revision: 3 }], rowCount: 1 }; + if (sql.includes("FROM app.integration_records")) return { rows: [ + { id: integrationId, kind: "bearer", name: "selected", origin: "https://api.example", description: null, revision: 4, encrypted_payload: await encrypt("user-ciphertext", key) }, + { id: "66666666-6666-4666-8666-666666666666", kind: "bearer", name: "not-selected", origin: "https://other.example", description: null, revision: 5, encrypted_payload: await encrypt("other-ciphertext", key) }, + ], rowCount: 2 }; + return { rows: [], rowCount: 0 }; + }); + const response = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/integration-snapshot", + body: { runId, grantRevision: 3 }, + }), env); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ownerId: "11111111-1111-4111-8111-111111111111", records: [ + { id: integrationId, kind: "bearer", name: "selected", origin: "https://api.example", description: null, revision: 4, encryptedPayload: "user-ciphertext" }, + ] }); + expect(env.query.mock.calls.map((call: any[]) => call[0])).toEqual(expect.arrayContaining([expect.stringContaining("grant_revision=$5"), expect.stringContaining("FROM app.integration_records")])); + }); }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 192acaa4..0912b108 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -52,6 +52,27 @@ export async function consumeProxyEnrollment(options) { return result.binding; } +/** Read the current user-encrypted integration records selected by an active run. + * The control plane decrypts only Factorize's outer encryption layer. */ +export async function readProxyIntegrationSnapshot({ controlOrigin, token, binding, fetchImpl = fetch } = {}) { + if (!binding || !uuidPattern.test(binding.runId ?? "") || !Number.isSafeInteger(binding.grantRevision) || binding.grantRevision < 0) { + throw new TypeError("invalid proxy integration snapshot binding"); + } + const result = await proxyControlRequest({ + controlOrigin, + token, + path: "/internal/credential-proxy/integration-snapshot", + body: { runId: binding.runId, grantRevision: binding.grantRevision }, + fetchImpl, + }); + if (typeof result?.ownerId !== "string" || !Array.isArray(result.records) + || result.records.some(record => !record || typeof record.id !== "string" || record.kind !== "bearer" + || typeof record.origin !== "string" || typeof record.encryptedPayload !== "string" || !Number.isSafeInteger(record.revision))) { + throw new Error("invalid proxy integration snapshot response"); + } + return result; +} + const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; /** diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 1501dc9f..c8386843 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, registerProxyBoot } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, registerProxyBoot } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -20,6 +20,20 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an assert.deepEqual(JSON.parse(request.init.body), { bootEpoch }); }); +test("proxy snapshot reader requests only the bound run revision and validates encrypted records", async () => { + let request; + const snapshot = { ownerId: "owner-1", records: [{ id: "integration-1", kind: "bearer", origin: "https://api.example", revision: 4, encryptedPayload: "ciphertext" }] }; + const result = await readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async (url, init) => { + request = { url: String(url), init, body: JSON.parse(init.body) }; + return Response.json(snapshot); + } }); + assert.deepEqual(result, snapshot); + assert.equal(request.url, "https://factorize.example/internal/credential-proxy/integration-snapshot"); + assert.deepEqual(request.body, { runId, grantRevision: 7 }); + assert.equal(request.init.headers.Authorization, `Bearer ${token}`); + await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ownerId: "owner-1", records: [{ ...snapshot.records[0], kind: "model" }] }) }), /invalid proxy integration snapshot response/); +}); + test("boot registration rejects unsafe origins, invalid credentials and mismatched response bindings", async () => { await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch }), /HTTPS origin/); await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); From 6f13d7bd6bc9b53281b7a30ef0a20dd7fda28d60 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:34:21 +0000 Subject: [PATCH 021/102] Fix encrypted snapshot proxy test fixture --- packages/api/test/proxy-control.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index e1c64539..649e16e9 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -124,7 +124,7 @@ describe("dedicated proxy boot registration endpoint", () => { it("returns only the current run's selected user-encrypted bearer records to its registered proxy", async () => { const integrationId = "55555555-5555-4555-8555-555555555555"; - const key = "test-server-encryption-key"; + const key = Buffer.alloc(32, 7).toString("base64"); const env = envFor(); env.CREDENTIAL_ENCRYPTION_KEY = key; env.query.mockImplementation(async (sql: string) => { From fea379cd2535c1c09599054929bc0a0bee98ec51 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:45:26 +0000 Subject: [PATCH 022/102] Wire proxy snapshots into local grant resolution --- .../credential-proxy/src/decrypted-grants.mjs | 28 ++++++++++++++--- packages/credential-proxy/src/runtime.mjs | 5 +++ .../test/decrypted-grants.test.mjs | 31 ++++++++++++++++++- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/packages/credential-proxy/src/decrypted-grants.mjs b/packages/credential-proxy/src/decrypted-grants.mjs index 683ff547..5fde128f 100644 --- a/packages/credential-proxy/src/decrypted-grants.mjs +++ b/packages/credential-proxy/src/decrypted-grants.mjs @@ -1,4 +1,5 @@ import { webcrypto } from "node:crypto"; +import { readProxyIntegrationSnapshot } from "./control-client.mjs"; const KEY_PREFIX = "fzk1."; const KEY_BYTES = 32; @@ -65,15 +66,21 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol return async function resolveGrants(binding) { if (!binding || typeof binding.tenantId !== "string" || !binding.tenantId || (binding.integrationIds !== undefined && !Array.isArray(binding.integrationIds))) throw new TypeError("invalid run integration selection"); - const integrationIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding); - if (!Array.isArray(integrationIds) || integrationIds.some(id => typeof id !== "string" || !id) - || new Set(integrationIds).size !== integrationIds.length) throw new TypeError("invalid run integration selection"); - if (integrationIds.length === 0) return []; const snapshot = await readSnapshot(binding); if (!snapshot || !Array.isArray(snapshot.records) || typeof snapshot.ownerId !== "string" || !snapshot.ownerId) throw new Error("integration snapshot is unavailable"); + // Production snapshots are already filtered by the control plane using the + // active run's immutable selection and grant revision. Local callers may + // still provide IDs explicitly for additional narrowing. + const integrationIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding) ?? snapshot.records.map(record => record?.id); + if (!Array.isArray(integrationIds) || integrationIds.some(id => typeof id !== "string" || !id) + || new Set(integrationIds).size !== integrationIds.length) throw new TypeError("invalid run integration selection"); + if (integrationIds.length === 0) { + if (snapshot.records.length) throw new Error("integration snapshot does not match the run selection"); + return []; + } const selected = new Set(integrationIds); const records = snapshot.records.filter(record => selected.has(record?.id) && record.kind === "bearer" && record.deletedAt == null); - if (records.length !== selected.size) throw new Error("selected integration is unavailable"); + if (records.length !== selected.size || snapshot.records.length !== records.length) throw new Error("selected integration is unavailable"); const unlockKey = await getUnlockKey(); if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const aesKey = await deriveKey(unlockKey); @@ -96,3 +103,14 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol return grants; }; } + +/** Compose the boot-registered control client, current snapshot reader, and + * process-local unlock key into the resolver used by authenticated tunnels. */ +export function createProxyIntegrationGrantResolver({ runtime, controlOrigin, controlCredential, fetchImpl = fetch } = {}) { + if (!runtime || typeof runtime.getUnlockKey !== "function") throw new TypeError("proxy runtime does not expose its unlocked key to the grant resolver"); + if (typeof controlOrigin !== "string" || typeof controlCredential !== "string") throw new TypeError("proxy control registration is required"); + return createDecryptedGrantResolver({ + readSnapshot: binding => readProxyIntegrationSnapshot({ controlOrigin, token: controlCredential, binding, fetchImpl }), + getUnlockKey: () => runtime.getUnlockKey(), + }); +} diff --git a/packages/credential-proxy/src/runtime.mjs b/packages/credential-proxy/src/runtime.mjs index d7f76cf9..ac5db602 100644 --- a/packages/credential-proxy/src/runtime.mjs +++ b/packages/credential-proxy/src/runtime.mjs @@ -40,6 +40,11 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL unlockMaterial = Buffer.from(key, "utf8"); state = "unlocked"; }, + /** Return a transient copy only to the local grant resolver. */ + getUnlockKey() { + if (shutdown || locking || state !== "unlocked" || !unlockMaterial) throw new Error("proxy is locked"); + return unlockMaterial.toString("utf8"); + }, async lock() { locking = true; state = "locked"; diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs index d0f873aa..3dd01866 100644 --- a/packages/credential-proxy/test/decrypted-grants.test.mjs +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -1,7 +1,8 @@ import assert from "node:assert/strict"; import { webcrypto } from "node:crypto"; import { test } from "node:test"; -import { createDecryptedGrantResolver } from "../src/decrypted-grants.mjs"; +import { createDecryptedGrantResolver, createProxyIntegrationGrantResolver } from "../src/decrypted-grants.mjs"; +import { createProxyRuntime } from "../src/runtime.mjs"; const encoder = new TextEncoder(); const salt = encoder.encode("factorize.unlock-key.v1"); @@ -67,3 +68,31 @@ test("resolver rejects non-HTTPS destinations and invalid credential bytes", asy const badCredentialResolver = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [badCredential] }), getUnlockKey: async () => unlockKey }); await assert.rejects(badCredentialResolver({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] }), /credential/); }); + +test("production grant resolver reads the run-bound snapshot and only decrypts while the runtime is unlocked", async () => { + const record = { id: "integration-a", kind: "bearer", origin: "https://api.example.test", revision: 3, deletedAt: null }; + const encryptedPayload = await encryptCredential({ ...record, credential: "snapshot-token" }); + const runtime = createProxyRuntime({ bootEpoch: "11111111-1111-4111-8111-111111111111" }); + const requests = []; + const resolve = createProxyIntegrationGrantResolver({ + runtime, + controlOrigin: "https://factorize.example", + controlCredential: `fzc_${"c".repeat(43)}`, + fetchImpl: async (url, init) => { + requests.push({ url: String(url), authorization: new Headers(init.headers).get("authorization"), body: JSON.parse(init.body) }); + return Response.json({ ownerId, records: [{ ...record, encryptedPayload }] }); + }, + }); + const binding = { runId: "22222222-2222-4222-8222-222222222222", grantRevision: 7 }; + await assert.rejects(resolve(binding), /locked/); + runtime.unlock(unlockKey); + assert.deepEqual(await resolve(binding), [{ id: record.id, origin: record.origin, credential: "snapshot-token" }]); + assert.deepEqual(requests, [{ + url: "https://factorize.example/internal/credential-proxy/integration-snapshot", + authorization: `Bearer fzc_${"c".repeat(43)}`, + body: binding, + }]); + await runtime.lock(); + await assert.rejects(resolve(binding), /locked/); + await runtime.shutdown(); +}); From b7656918894f12119f43b912471850b0008e39e4 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:48:34 +0000 Subject: [PATCH 023/102] Preserve grant resolver selection boundaries --- packages/credential-proxy/src/decrypted-grants.mjs | 3 ++- packages/credential-proxy/test/decrypted-grants.test.mjs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/credential-proxy/src/decrypted-grants.mjs b/packages/credential-proxy/src/decrypted-grants.mjs index 5fde128f..4be1d19b 100644 --- a/packages/credential-proxy/src/decrypted-grants.mjs +++ b/packages/credential-proxy/src/decrypted-grants.mjs @@ -80,7 +80,8 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol } const selected = new Set(integrationIds); const records = snapshot.records.filter(record => selected.has(record?.id) && record.kind === "bearer" && record.deletedAt == null); - if (records.length !== selected.size || snapshot.records.length !== records.length) throw new Error("selected integration is unavailable"); + const selectionWasNarrowedLocally = binding.integrationIds !== undefined || resolveIntegrationIds !== undefined; + if (records.length !== selected.size || (!selectionWasNarrowedLocally && snapshot.records.length !== records.length)) throw new Error("selected integration is unavailable"); const unlockKey = await getUnlockKey(); if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const aesKey = await deriveKey(unlockKey); diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs index 3dd01866..a3467672 100644 --- a/packages/credential-proxy/test/decrypted-grants.test.mjs +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -83,7 +83,7 @@ test("production grant resolver reads the run-bound snapshot and only decrypts w return Response.json({ ownerId, records: [{ ...record, encryptedPayload }] }); }, }); - const binding = { runId: "22222222-2222-4222-8222-222222222222", grantRevision: 7 }; + const binding = { tenantId: ownerId, runId: "22222222-2222-4222-8222-222222222222", grantRevision: 7 }; await assert.rejects(resolve(binding), /locked/); runtime.unlock(unlockKey); assert.deepEqual(await resolve(binding), [{ id: record.id, origin: record.origin, credential: "snapshot-token" }]); From 15335fadaf0400f01cbe2fdb1b749857a58874da Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:49:39 +0000 Subject: [PATCH 024/102] Resolve empty and locked grants safely --- packages/credential-proxy/src/decrypted-grants.mjs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/credential-proxy/src/decrypted-grants.mjs b/packages/credential-proxy/src/decrypted-grants.mjs index 4be1d19b..247a1bf3 100644 --- a/packages/credential-proxy/src/decrypted-grants.mjs +++ b/packages/credential-proxy/src/decrypted-grants.mjs @@ -66,12 +66,18 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol return async function resolveGrants(binding) { if (!binding || typeof binding.tenantId !== "string" || !binding.tenantId || (binding.integrationIds !== undefined && !Array.isArray(binding.integrationIds))) throw new TypeError("invalid run integration selection"); + const requestedIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding); + if (requestedIds !== undefined && (!Array.isArray(requestedIds) || requestedIds.some(id => typeof id !== "string" || !id) + || new Set(requestedIds).size !== requestedIds.length)) throw new TypeError("invalid run integration selection"); + if (requestedIds?.length === 0) return []; + const unlockKey = await getUnlockKey(); + if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const snapshot = await readSnapshot(binding); if (!snapshot || !Array.isArray(snapshot.records) || typeof snapshot.ownerId !== "string" || !snapshot.ownerId) throw new Error("integration snapshot is unavailable"); // Production snapshots are already filtered by the control plane using the // active run's immutable selection and grant revision. Local callers may // still provide IDs explicitly for additional narrowing. - const integrationIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding) ?? snapshot.records.map(record => record?.id); + const integrationIds = requestedIds ?? snapshot.records.map(record => record?.id); if (!Array.isArray(integrationIds) || integrationIds.some(id => typeof id !== "string" || !id) || new Set(integrationIds).size !== integrationIds.length) throw new TypeError("invalid run integration selection"); if (integrationIds.length === 0) { @@ -82,8 +88,6 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol const records = snapshot.records.filter(record => selected.has(record?.id) && record.kind === "bearer" && record.deletedAt == null); const selectionWasNarrowedLocally = binding.integrationIds !== undefined || resolveIntegrationIds !== undefined; if (records.length !== selected.size || (!selectionWasNarrowedLocally && snapshot.records.length !== records.length)) throw new Error("selected integration is unavailable"); - const unlockKey = await getUnlockKey(); - if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const aesKey = await deriveKey(unlockKey); const grants = []; for (const record of records) { From 32414a2e8a227415d1624e2d4c602310cabad856 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:49:56 +0000 Subject: [PATCH 025/102] Match snapshot control request contract --- packages/credential-proxy/test/decrypted-grants.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs index a3467672..18ad4d62 100644 --- a/packages/credential-proxy/test/decrypted-grants.test.mjs +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -90,7 +90,7 @@ test("production grant resolver reads the run-bound snapshot and only decrypts w assert.deepEqual(requests, [{ url: "https://factorize.example/internal/credential-proxy/integration-snapshot", authorization: `Bearer fzc_${"c".repeat(43)}`, - body: binding, + body: { runId: binding.runId, grantRevision: binding.grantRevision }, }]); await runtime.lock(); await assert.rejects(resolve(binding), /locked/); From c22a4126e930923d880fe25e5f6f01a8cfb61df0 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 23:08:07 +0000 Subject: [PATCH 026/102] Bind tunnel VM identity to certificate SAN --- .../credential-proxy/src/tunnel-transport.mjs | 19 ++++++++++++--- .../test/fixtures/upstream-cert.pem | 24 +++++++++---------- .../test/proxy-listener.e2e.test.mjs | 17 ++++++++++++- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index 28d9d9a9..d61f6b99 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -9,10 +9,23 @@ const CONTROL_RETRY_WINDOW_MS = 5 * 60_000; const CONTROL_RETRY_MAX_MS = 30_000; const CONTROL_RETRY_SLOW_MS = 60_000; -function peerIdentity(socket) { +export function peerIdentity(socket) { const certificate = socket.getPeerCertificate(); - if (!socket.authorized || !certificate?.subject?.CN) throw new Error("tunnel client certificate is not authorized"); - return certificate.subject.CN; + if (!socket.authorized || !certificate?.subject?.CN || typeof certificate.subjectaltname !== "string") { + throw new Error("tunnel client certificate is not authorized"); + } + const identities = certificate.subjectaltname.split(/,\s*/).filter(value => value.startsWith("URI:")); + if (identities.length !== 1) throw new Error("tunnel client certificate must contain one VM identity URI"); + const match = /^URI:urn:factorize:vm:([A-Za-z0-9_-]+)$/.exec(identities[0]); + if (!match) throw new Error("tunnel client certificate has an invalid VM identity URI"); + const identityBytes = Buffer.from(match[1], "base64url"); + const identity = identityBytes.toString("utf8"); + if (!identity || identityBytes.toString("base64url") !== match[1] + || identity.length > 255 || /[\u0000-\u001f\u007f]/.test(identity) + || certificate.subject.CN !== identity) { + throw new Error("tunnel client certificate identity is invalid or inconsistent"); + } + return identity; } /** diff --git a/packages/credential-proxy/test/fixtures/upstream-cert.pem b/packages/credential-proxy/test/fixtures/upstream-cert.pem index 685d5670..afec81f5 100644 --- a/packages/credential-proxy/test/fixtures/upstream-cert.pem +++ b/packages/credential-proxy/test/fixtures/upstream-cert.pem @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNDCCAhygAwIBAgIUWvCDWltyenSy441ZnfAvqkjgIwUwDQYJKoZIhvcNAQEL -BQAwGzEZMBcGA1UEAwwQYXBpLmV4YW1wbGUudGVzdDAeFw0yNjExMTExNTQwNTha -Fw0zNjExMDgxNTQwNThaMBsxGTAXBgNVBAMMEGFwaS5leGFtcGxlLnRlc3QwggEi +MIIDTjCCAjagAwIBAgIUPy+fWru0gF7QNal9Xg4vXdIuUCgwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQYXBpLmV4YW1wbGUudGVzdDAeFw0yNjExMTEyMzA3Mjla +Fw0zNjExMDgyMzA3MjlaMBsxGTAXBgNVBAMMEGFwaS5leGFtcGxlLnRlc3QwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDClgGTbXnob9GHbVxMt1lSmsfW 6lSfuOEBEuE5n96bKcfSsVKMRKvw3j8PyGasF+DTQTwqFlcavHzmmVkOBpQYLSz3 bvD0RnKJ8CXhZwcRSA77BENRJQJ3TRr9WIAVrz3dgkAEl5Tv4Y+0KILRqZMx+vg1 R0CuAzt6JbhGTvFugRAgeSJootV/O9Uu+z4SqtEbOkpj7isRUJa1uirAmKoFucIU sa/4wTBkWIL74Z2iEMdmNM0HQ0xTsoVmFVJb+yxCoCr5rzIdPuGDyQeqWaWtju7Z kVJjUskyYgSrycUivxAElY9aedgmyZxoBIHiwdFsSt/Nu5Gj9mOahqAfAPdjAgMB -AAGjcDBuMB0GA1UdDgQWBBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAfBgNVHSMEGDAW -gBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAPBgNVHRMBAf8EBTADAQH/MBsGA1UdEQQU -MBKCEGFwaS5leGFtcGxlLnRlc3QwDQYJKoZIhvcNAQELBQADggEBAFOW5O4OKDe8 -6MWztml7Yt9Rvn6zcDTM4xsMKhMDITAlorXuAfu51JGgFOsqiqNP9L9cpN57k5JS -5wYA9Fy+cFiiXtkAqImSvjSf1gvlzKpHCh1s0dGjIYnoStoWbE0PzVweSQ2zCQy/ -aQC11E3/CyS2+toZR+i1zffaBLjbA6T4GToOKCrXdGl8Wv7vCyTbEMA6ouEspkJO -U/glcrkekGgIZVC10di70LI+LkgqsnuDpFIALb7qDMtZQuqnIEU/wyapORpQn6Lm -WjHPEcWSHe5dlHEQWO0P3XXEA2mbr+fboVWWqQ/5gt7aBTLOVfV9rSkQqGpvriPQ -RmX+kW6AbK4= +AAGjgYkwgYYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAqQwRAYDVR0R +BD0wO4IQYXBpLmV4YW1wbGUudGVzdIYndXJuOmZhY3Rvcml6ZTp2bTpZWEJwTG1W +NFlXMXdiR1V1ZEdWemRBMB0GA1UdDgQWBBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAN +BgkqhkiG9w0BAQsFAAOCAQEASBvpzgJtGPUxB8ao20lUU2J95DRPFH6BEPoJq38+ +62D7YTNIesVPykdnZQy/8uVWczIAKURAU/MRv10N8K6nfo/PAx3Hi2m8qbnoTcRa +2JLoE++bQltKI7GAg//wviMpb3CjgObjoIYVa8pdHvR2PLEIroJuGitOyZTWDBFh +bu5VcH5FDkCLD8DzatgGyEVDDR9vELzS5m1j9WgVXAHnMhnKegmvhhJhsdT+/LEE +gtNFdHYNRPGbxhJ6Kc8CbmSZN7T8zUjCTHfcAB5iz6lrQRbbbxzIekXeqzv+Ya0C +vnY+CxOlQab0mac1Y72EnoDMSkvAywIbdwe9qcpITlrKmQ== -----END CERTIFICATE----- diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 5098aed7..a470bf0a 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -7,7 +7,7 @@ import { test } from "node:test"; import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; import { createAuthenticatedProxyListener } from "../src/proxy-listener.mjs"; import { createProxyRuntime } from "../src/runtime.mjs"; -import { createAuthenticatedTunnelServer, startAuthenticatedProxyTunnel, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; +import { createAuthenticatedTunnelServer, peerIdentity, startAuthenticatedProxyTunnel, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; const fixtures = new URL("./fixtures/", import.meta.url); const key = await readFile(new URL("upstream-key.pem", fixtures)); @@ -18,6 +18,21 @@ async function waitForEvent(emitter, event) { return pending; } +test("mTLS VM identity comes from one canonical URI SAN bound to the certificate CN", () => { + const identity = "api.example.test"; + const encoded = Buffer.from(identity).toString("base64url"); + const socket = (subjectaltname, commonName = identity, authorized = true) => ({ + authorized, + getPeerCertificate: () => ({ subject: { CN: commonName }, subjectaltname }), + }); + assert.equal(peerIdentity(socket(`DNS:api.example.test, URI:urn:factorize:vm:${encoded}`)), identity); + assert.throws(() => peerIdentity(socket("DNS:api.example.test")), /one VM identity URI/); + assert.throws(() => peerIdentity(socket(`URI:urn:factorize:vm:${encoded}, URI:urn:factorize:vm:${encoded}`)), /one VM identity URI/); + assert.throws(() => peerIdentity(socket("URI:urn:factorize:vm:not_canonical=")), /invalid VM identity URI/); + assert.throws(() => peerIdentity(socket(`URI:urn:factorize:vm:${encoded}`, "other-vm")), /inconsistent/); + assert.throws(() => peerIdentity(socket(`URI:urn:factorize:vm:${encoded}`, identity, false)), /not authorized/); +}); + test("authenticated tunnel listener streams over TLS, confines grants and cancels upstream work", { timeout: 5_000 }, async () => { let observedAuthorization; let upstreamEnded = false; From 3ed3f07a7452fda1df52e93c169a2cdd70cfcecf Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 23:47:48 +0000 Subject: [PATCH 027/102] GEN-2824 provision mTLS identities and revoke tunnels --- .../src/fly-credential-proxy-provisioner.ts | 25 ++++++ packages/api/src/proxy-control.ts | 10 ++- .../fly-credential-proxy-provisioner.test.ts | 29 ++++++- packages/api/test/proxy-control.test.ts | 22 +++++ .../src/bootstrap-control.mjs | 28 +++++- .../credential-proxy/src/control-client.mjs | 52 +++++++++++ packages/credential-proxy/src/main.mjs | 52 +++++++++-- .../credential-proxy/src/tls-identity.mjs | 87 +++++++++++++++++++ .../credential-proxy/src/vm-certificate.mjs | 61 +++++++++++++ .../test/bootstrap-control.test.mjs | 32 +++++-- .../test/control-client.test.mjs | 29 ++++++- .../test/tls-identity.test.mjs | 50 +++++++++++ 12 files changed, 458 insertions(+), 19 deletions(-) create mode 100644 packages/credential-proxy/src/tls-identity.mjs create mode 100644 packages/credential-proxy/src/vm-certificate.mjs create mode 100644 packages/credential-proxy/test/tls-identity.test.mjs diff --git a/packages/api/src/fly-credential-proxy-provisioner.ts b/packages/api/src/fly-credential-proxy-provisioner.ts index ad8751d4..101b0ba5 100644 --- a/packages/api/src/fly-credential-proxy-provisioner.ts +++ b/packages/api/src/fly-credential-proxy-provisioner.ts @@ -28,6 +28,9 @@ export type FlyCredentialProxyMachineClient = { export type FlyCredentialProxyGuest = { /** The reviewed image startup command consumes this payload from stdin only. */ startProxy(input: ProxyBootstrapProvisioning & { credential: string }): Promise; + /** Signs a VM-generated CSR over the provider's private guest channel. Only + * the public certificate and CA trust bundle return to the caller. */ + issueVmCertificate(input: { bootEpoch: string; vmIdentity: string; csr: string }): Promise<{ certificate: string; trustBundle: string }>; }; /** Provider exec adapter: the bootstrap record is the command's stdin payload. */ @@ -42,6 +45,7 @@ export function createFlyCredentialProxyGuest(options: { return { async startProxy(input) { const result = await transport.run(["/usr/local/bin/factorize-proxy-startup"], `${JSON.stringify({ + type: "register-boot", controlOrigin: input.controlOrigin, bootEpoch: input.bootEpoch, credential: input.credential, @@ -49,6 +53,22 @@ export function createFlyCredentialProxyGuest(options: { if (!result.ok) throw new Error("Credential proxy startup command failed"); if (result.stdout.trim() !== "Credential proxy started") throw new Error("Credential proxy startup was not confirmed"); }, + async issueVmCertificate(input) { + if (!input || !UUID_V4.test(input.bootEpoch) || !/^[A-Za-z0-9:._-]{1,255}$/.test(input.vmIdentity) + || typeof input.csr !== "string" || input.csr.length < 100 || input.csr.length > 16_384) throw new TypeError("Invalid VM certificate provisioning request"); + const result = await transport.run(["/usr/local/bin/factorize-proxy-certify-vm"], `${JSON.stringify(input)}\n`); + if (!result.ok) throw new Error("Credential proxy VM certificate provisioning failed"); + let response: unknown; + try { response = JSON.parse(result.stdout); } catch { throw new Error("Credential proxy returned an invalid VM certificate response"); } + if (!response || typeof response !== "object" || Array.isArray(response) + || Object.keys(response).sort().join(",") !== "bootEpoch,certificate,trustBundle,vmIdentity" + || (response as any).bootEpoch !== input.bootEpoch || (response as any).vmIdentity !== input.vmIdentity + || typeof (response as any).certificate !== "string" || !(response as any).certificate.startsWith("-----BEGIN CERTIFICATE-----") + || typeof (response as any).trustBundle !== "string" || !(response as any).trustBundle.startsWith("-----BEGIN CERTIFICATE-----")) { + throw new Error("Credential proxy returned an invalid VM certificate response"); + } + return { certificate: (response as any).certificate, trustBundle: (response as any).trustBundle }; + }, }; } @@ -62,6 +82,7 @@ export type FlyCredentialProxyProvisionerOptions = { function validate(input: FlyCredentialProxyConfig): void { if (!input.tenantId || !input.app || !input.operationId || input.operationId.length > 200) throw new TypeError("Invalid credential proxy provisioning identity"); + if (!/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(input.app)) throw new TypeError("Invalid credential proxy Fly app name"); if (!IMAGE_DIGEST.test(input.image)) throw new TypeError("Credential proxy image must be pinned by sha256 digest"); if (!/^[a-z0-9]{3}$/.test(input.region)) throw new TypeError("Invalid credential proxy region"); if (!/^[a-z0-9-]{1,64}$/.test(input.size)) throw new TypeError("Invalid credential proxy size"); @@ -89,6 +110,10 @@ export function createFlyCredentialProxyProvisioner(options: FlyCredentialProxyP region: input.region, cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", candidateBootEpoch], guest: { cpu_kind: "shared", cpus: 1, memory_mb: input.memoryMb ?? 512 }, + env: { + FACTORIZE_PROXY_TUNNEL_SERVER_NAME: `${input.app}.internal`, + FACTORIZE_PROXY_TUNNEL_PORT: "9090", + }, metadata: { factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId, diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index d3bb08b5..fa91e53c 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -9,7 +9,7 @@ const json = (body: unknown, status = 200) => Response.json(body, { status, head /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot"].includes(url.pathname)) return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot", "/internal/credential-proxy/revocations"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; @@ -41,6 +41,14 @@ export async function proxyControlFetch(request: Request, env: Parameters key !== "bootEpoch") + || typeof (body as any).bootEpoch !== "string" || (body as any).bootEpoch !== proxy.bootEpoch) return json({ error: "invalid_request" }, 400); + const revoked = await databaseFor(env).pool.query<{ run_id: string }>(`SELECT run_id FROM app.credential_run_grants + WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch=$3 AND revoked_at IS NOT NULL ORDER BY revoked_at,run_id`, + [proxy.tenantId, proxy.proxyIdentity, proxy.bootEpoch]); + return json({ revokedRunIds: revoked.rows.map(row => row.run_id) }); + } if (url.pathname === "/internal/credential-proxy/integration-snapshot") { if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "grantRevision"].includes(key)) || typeof (body as any).runId !== "string" || !Number.isSafeInteger((body as any).grantRevision) || (body as any).grantRevision < 0) return json({ error: "invalid_request" }, 400); diff --git a/packages/api/test/fly-credential-proxy-provisioner.test.ts b/packages/api/test/fly-credential-proxy-provisioner.test.ts index aeecf1ca..d2b1a976 100644 --- a/packages/api/test/fly-credential-proxy-provisioner.test.ts +++ b/packages/api/test/fly-credential-proxy-provisioner.test.ts @@ -23,7 +23,7 @@ function fixture() { startMachine: vi.fn(async () => undefined), destroyMachine: vi.fn(async () => undefined), }; - const guest = vi.fn(() => ({ startProxy: vi.fn(async () => undefined) })); + const guest = vi.fn(() => ({ startProxy: vi.fn(async () => undefined), issueVmCertificate: vi.fn(async () => ({ certificate: "", trustBundle: "" })) })); const provisioner = createFlyCredentialProxyProvisioner({ control, machines, guest, newBootEpoch: () => epoch }); return { control, machines, guest, provisioner }; } @@ -37,6 +37,7 @@ describe("Fly credential proxy startup", () => { image: input.image, region: input.region, cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", epoch], + env: { FACTORIZE_PROXY_TUNNEL_SERVER_NAME: "factorize-proxies.internal", FACTORIZE_PROXY_TUNNEL_PORT: "9090" }, restart: { policy: "no" }, auto_start: false, metadata: expect.objectContaining({ factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId }), @@ -57,7 +58,7 @@ describe("Fly credential proxy startup", () => { it("revokes bootstrap authority and removes a Machine when startup fails", async () => { const { control, machines, guest } = fixture(); - guest.mockReturnValue({ startProxy: vi.fn().mockRejectedValue(new Error("private startup channel failed")) }); + guest.mockReturnValue({ startProxy: vi.fn().mockRejectedValue(new Error("private startup channel failed")), issueVmCertificate: vi.fn() }); const provisioner = createFlyCredentialProxyProvisioner({ control, machines, guest, newBootEpoch: () => epoch }); await expect(provisioner.start(input)).rejects.toThrow("private startup channel failed"); expect(control.revokeCredential).toHaveBeenCalledWith(input.tenantId, "fly:factorize-proxies:machine-1"); @@ -68,6 +69,7 @@ describe("Fly credential proxy startup", () => { const { machines, provisioner } = fixture(); await expect(provisioner.start({ ...input, image: "ghcr.io/example/proxy:latest" })).rejects.toThrow("pinned by sha256 digest"); await expect(provisioner.start({ ...input, controlOrigin: "http://factorize.example" })).rejects.toThrow("HTTPS origin"); + await expect(provisioner.start({ ...input, app: "invalid/app" })).rejects.toThrow("Fly app name"); expect(machines.createMachine).not.toHaveBeenCalled(); }); }); @@ -88,6 +90,29 @@ describe("Fly credential proxy guest adapter", () => { const guest = createFlyCredentialProxyGuest({ app: input.app, machine: machine.id, token: "server-token-fixture", spawnProcess }); await guest.startProxy({ tenantId: input.tenantId, proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, credential: `fzc_${"c".repeat(43)}` }); expect(calls[0]?.[1]).toContain("/usr/local/bin/factorize-proxy-startup"); + expect(String(calls[1]?.[1])).toContain(`"type":"register-boot"`); expect(String(calls[1]?.[1])).toContain(`"credential":"fzc_${"c".repeat(43)}"`); }); + + it("requests a VM CSR signature through the private guest channel and verifies both bindings", async () => { + const calls: unknown[][] = []; + const spawnProcess = ((command: string, args: string[], options: { stdio: string[] }) => { + calls.push([command, args, options]); + const stdoutHandlers: Array<(chunk: Buffer) => void> = []; + const child: any = { pid: undefined, stdin: { end: (value: string) => { calls.push(["stdin", value]); } }, + stdout: { on: (_event: string, callback: (chunk: Buffer) => void) => stdoutHandlers.push(callback) }, + stderr: { on: () => undefined }, on: (event: string, callback: (code: number) => void) => { + if (event === "close") queueMicrotask(() => { stdoutHandlers.forEach(handler => handler(Buffer.from(JSON.stringify({ + bootEpoch: epoch, vmIdentity: "fly:job-1", certificate: "-----BEGIN CERTIFICATE-----cert", trustBundle: "-----BEGIN CERTIFICATE-----ca", + }) + "\n"))); callback(0); }); + }, kill: () => undefined }; + return child; + }) as any; + const guest = createFlyCredentialProxyGuest({ app: input.app, machine: machine.id, token: "server-token-fixture", spawnProcess }); + await expect(guest.issueVmCertificate({ bootEpoch: epoch, vmIdentity: "fly:job-1", csr: `-----BEGIN CERTIFICATE REQUEST-----\n${"A".repeat(120)}\n-----END CERTIFICATE REQUEST-----` })) + .resolves.toEqual({ certificate: "-----BEGIN CERTIFICATE-----cert", trustBundle: "-----BEGIN CERTIFICATE-----ca" }); + expect(calls[0]?.[1]).toContain("/usr/local/bin/factorize-proxy-certify-vm"); + expect(String(calls[1]?.[1])).toContain("\"vmIdentity\":\"fly:job-1\""); + await expect(guest.issueVmCertificate({ bootEpoch: epoch, vmIdentity: "fly:job-1", csr: "bad" })).rejects.toThrow("Invalid VM certificate provisioning request"); + }); }); diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 649e16e9..7a8cd90d 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -147,4 +147,26 @@ describe("dedicated proxy boot registration endpoint", () => { ] }); expect(env.query.mock.calls.map((call: any[]) => call[0])).toEqual(expect.arrayContaining([expect.stringContaining("grant_revision=$5"), expect.stringContaining("FROM app.integration_records")])); }); + + it("returns durable revoked runs only for the authenticated current proxy boot", async () => { + const env = envFor(); + env.query.mockImplementation(async (sql: string) => sql.includes("proxy_token_digest") + ? { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 } + : { rows: [{ run_id: runId }], rowCount: 1 }); + const response = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/revocations", + body: { bootEpoch }, + }), env); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ revokedRunIds: [runId] }); + expect(env.query.mock.calls[1][0]).toContain("revoked_at IS NOT NULL"); + expect(env.query.mock.calls[1][1]).toEqual(["11111111-1111-4111-8111-111111111111", "proxy-fixture", bootEpoch]); + const stale = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/revocations", + body: { bootEpoch: "44444444-4444-4444-8444-444444444444" }, + }), env); + expect(stale.status).toBe(400); + }); }); diff --git a/packages/credential-proxy/src/bootstrap-control.mjs b/packages/credential-proxy/src/bootstrap-control.mjs index 332bdcc0..8d17234c 100644 --- a/packages/credential-proxy/src/bootstrap-control.mjs +++ b/packages/credential-proxy/src/bootstrap-control.mjs @@ -6,8 +6,9 @@ import { registerProxyBoot } from "./control-client.mjs"; const MAX_BOOTSTRAP_BYTES = 4096; /** Local-only receiver for a provider exec channel; credentials never enter env or files. */ -export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot, onRegistered = () => {} } = {}) { - if (!runtime || typeof register !== "function" || typeof onRegistered !== "function") throw new TypeError("proxy bootstrap control requires a runtime, registrar, and registration callback"); +export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot, onRegistered = () => {}, issueVmCertificate } = {}) { + if (!runtime || typeof register !== "function" || typeof onRegistered !== "function" + || (issueVmCertificate !== undefined && typeof issueVmCertificate !== "function")) throw new TypeError("proxy bootstrap control requires a runtime, registrar, and registration callback"); await mkdir(dirname(socketPath), { recursive: true, mode: 0o700 }); await chmod(dirname(socketPath), 0o700); await rm(socketPath, { force: true }); @@ -31,8 +32,29 @@ export async function startProxyBootstrapControl({ runtime, socketPath = "/run/f let request; try { request = JSON.parse(data.subarray(0, end).toString("utf8")); } catch { return finish({ error: "invalid bootstrap request" }); } + if (request?.type === "issue-vm-certificate") { + if (!issueVmCertificate || request.bootEpoch !== runtime.bootEpoch + || typeof request.vmIdentity !== "string" || typeof request.csr !== "string" + || Object.keys(request).sort().join(",") !== "bootEpoch,csr,type,vmIdentity") { + return finish({ error: "invalid certificate request" }); + } + processing = true; + socket.pause(); + socket.removeAllListeners("data"); + try { + const issued = await issueVmCertificate({ vmIdentity: request.vmIdentity, csr: request.csr, bootEpoch: runtime.bootEpoch }); + if (typeof issued?.certificate !== "string" || typeof issued?.trustBundle !== "string") throw new Error("invalid certificate response"); + finish({ issued: true, bootEpoch: runtime.bootEpoch, vmIdentity: request.vmIdentity, certificate: issued.certificate, trustBundle: issued.trustBundle }); + } catch { + finish({ error: "certificate provisioning failed" }); + } finally { + request.csr = ""; + } + return; + } + if (request?.type !== "register-boot") return finish({ error: "invalid bootstrap request" }); if (!request || typeof request !== "object" || Array.isArray(request) - || Object.keys(request).sort().join(",") !== "bootEpoch,controlOrigin,credential" + || Object.keys(request).sort().join(",") !== "bootEpoch,controlOrigin,credential,type" || request.bootEpoch !== runtime.bootEpoch || typeof request.credential !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(request.credential)) { return finish({ error: "invalid bootstrap request" }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 0912b108..fd1b4c46 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -73,6 +73,58 @@ export async function readProxyIntegrationSnapshot({ controlOrigin, token, bindi return result; } +/** Read durable revocations for the registered proxy boot. Replaying the full + * set is intentional: it closes runs revoked while this proxy was offline and + * runtime.revokeRun is idempotent. */ +export async function readProxyRunRevocations({ controlOrigin, token, bootEpoch, fetchImpl = fetch } = {}) { + if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); + const result = await proxyControlRequest({ + controlOrigin, + token, + path: "/internal/credential-proxy/revocations", + body: { bootEpoch }, + fetchImpl, + }); + if (!Array.isArray(result?.revokedRunIds) || result.revokedRunIds.some(id => !uuidPattern.test(id))) { + throw new Error("invalid proxy revocation response"); + } + return result.revokedRunIds; +} + +/** Poll durable control-plane state. A failed request leaves established + * connections running and the next bounded poll retries when communication + * recovers; it does not add an authorization lease or expiry. */ +export function startProxyRunRevocationMonitor({ controlOrigin, controlCredential, bootEpoch, runtime, intervalMs = 5_000, fetchImpl = fetch } = {}) { + if (!runtime || typeof runtime.revokeRun !== "function" || !Number.isInteger(intervalMs) || intervalMs < 100 || intervalMs > 60_000) { + throw new TypeError("invalid proxy revocation monitor configuration"); + } + let stopped = false; + let timer; + let polling; + const poll = async () => { + if (stopped || polling) return; + polling = (async () => { + try { + const runIds = await readProxyRunRevocations({ controlOrigin, token: controlCredential, bootEpoch, fetchImpl }); + for (const runId of runIds) await runtime.revokeRun(runId); + } catch { + // Keep existing connections alive during an outage and retry on schedule. + } finally { + polling = undefined; + if (!stopped) timer = setTimeout(poll, intervalMs); + } + })(); + }; + void poll(); + return Object.freeze({ + async close() { + stopped = true; + clearTimeout(timer); + await polling; + }, + }); +} + const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; /** diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index d63c3454..60ab7333 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -1,20 +1,56 @@ import { createInterface } from "node:readline"; import { randomUUID } from "node:crypto"; import { createProxyRuntime } from "./runtime.mjs"; -import { registerProxyBoot } from "./control-client.mjs"; +import { registerProxyBoot, startProxyRunRevocationMonitor } from "./control-client.mjs"; import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; +import { createProxyIntegrationGrantResolver } from "./decrypted-grants.mjs"; +import { startAuthenticatedProxyTunnel } from "./tunnel-transport.mjs"; +import { createProxyTlsAuthority } from "./tls-identity.mjs"; const epochIndex = process.argv.indexOf("--boot-epoch"); const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; const runtime = createProxyRuntime({ bootEpoch }); +const tunnelPort = Number(process.env.FACTORIZE_PROXY_TUNNEL_PORT ?? "9090"); +if (!Number.isInteger(tunnelPort) || tunnelPort < 1 || tunnelPort > 65535) throw new Error("invalid proxy tunnel port"); +const tlsAuthority = await createProxyTlsAuthority({ + serverName: process.env.FACTORIZE_PROXY_TUNNEL_SERVER_NAME ?? "localhost", + bootEpoch: runtime.bootEpoch, +}); let controlCredential; let controlOrigin; +let tunnel; +let revocationMonitor; + +async function activateRegistration(registration, origin) { + if (tunnel) throw new Error("proxy boot is already registered"); + const resolveGrants = createProxyIntegrationGrantResolver({ runtime, controlOrigin: origin, controlCredential: registration.controlCredential }); + const started = await startAuthenticatedProxyTunnel({ + tls: tlsAuthority.tls, + runtime, + controlOrigin: origin, + controlCredential: registration.controlCredential, + resolveGrants, + host: process.env.FACTORIZE_PROXY_TUNNEL_HOST ?? "0.0.0.0", + port: tunnelPort, + }); + controlCredential = registration.controlCredential; + controlOrigin = origin; + tunnel = started; + revocationMonitor = startProxyRunRevocationMonitor({ + controlOrigin: origin, + controlCredential: registration.controlCredential, + bootEpoch: runtime.bootEpoch, + runtime, + }); +} + const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET, - onRegistered(registration, request) { - controlCredential = registration.controlCredential; - controlOrigin = request.controlOrigin; + onRegistered: (registration, request) => activateRegistration(registration, request.controlOrigin), + issueVmCertificate: ({ vmIdentity, csr }) => { + if (!tunnel) throw new Error("proxy tunnel is not registered"); + return tlsAuthority.signVmCsr({ vmIdentity, csr }); }, }); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); @@ -34,7 +70,12 @@ async function stop() { closing = true; controlCredential = undefined; controlOrigin = undefined; + await revocationMonitor?.close(); + revocationMonitor = undefined; await runtime.shutdown(); + await tunnel?.close().catch(() => {}); + tunnel = undefined; + await tlsAuthority.close(); await bootstrapControl.close(); lines.close(); clearTimeout(shutdownDeadline); @@ -64,8 +105,7 @@ lines.on("line", async (line) => { command.credential = undefined; try { const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); - controlCredential = registration.controlCredential; - controlOrigin = command.controlOrigin; + await activateRegistration(registration, command.controlOrigin); respond({ type: "boot-registered", proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { respond({ type: "error", error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/src/tls-identity.mjs b/packages/credential-proxy/src/tls-identity.mjs new file mode 100644 index 00000000..933fde50 --- /dev/null +++ b/packages/credential-proxy/src/tls-identity.mjs @@ -0,0 +1,87 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFile = promisify(execFileCallback); +const SAFE_VM_IDENTITY = /^[A-Za-z0-9:._-]{1,255}$/; +const SAFE_DNS_NAME = /^(?=.{1,253}$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(?:\.(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?))*$/; + +async function openssl(args) { + try { + await execFile("openssl", args, { timeout: 10_000, maxBuffer: 64 * 1024, windowsHide: true }); + } catch { + // Never include command output or key material in errors. + throw new Error("proxy TLS identity operation failed"); + } +} + +function pem(bytes) { + return Buffer.from(bytes).toString("utf8"); +} + +/** + * Creates a per-process private CA and proxy server identity on a memory-backed + * filesystem. The CA private key never leaves this process or tmpfs. Job VMs + * generate their own private keys and submit CSRs; only their signed public + * certificate and the public trust root are returned for secure distribution. + */ +export async function createProxyTlsAuthority({ serverName, bootEpoch } = {}) { + if (typeof serverName !== "string" || !SAFE_DNS_NAME.test(serverName)) throw new TypeError("proxy TLS server name is invalid"); + if (typeof bootEpoch !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(bootEpoch)) throw new TypeError("proxy TLS boot epoch is invalid"); + const directory = await mkdtemp(join("/dev/shm", "factorize-proxy-tls-")); + const caKeyPath = join(directory, "ca-key.pem"); + const caCertPath = join(directory, "ca-cert.pem"); + const serverKeyPath = join(directory, "server-key.pem"); + const serverCertPath = join(directory, "server-cert.pem"); + try { + await openssl(["genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", "-out", caKeyPath]); + await chmod(caKeyPath, 0o600); + await openssl(["req", "-new", "-x509", "-key", caKeyPath, "-sha256", "-days", "1", "-subj", `/CN=factorize-proxy-${bootEpoch}`, "-addext", "basicConstraints=critical,CA:TRUE,pathlen:0", "-addext", "keyUsage=critical,keyCertSign,cRLSign", "-out", caCertPath]); + await openssl(["genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", "-out", serverKeyPath]); + await chmod(serverKeyPath, 0o600); + await openssl(["req", "-new", "-key", serverKeyPath, "-subj", `/CN=${serverName}`, "-out", join(directory, "server.csr")]); + await writeFile(join(directory, "server.ext"), `basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyAgreement\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:${serverName}\n`, { mode: 0o600 }); + await openssl(["x509", "-req", "-in", join(directory, "server.csr"), "-CA", caCertPath, "-CAkey", caKeyPath, "-CAcreateserial", "-days", "1", "-sha256", "-extfile", join(directory, "server.ext"), "-out", serverCertPath]); + const ca = await readFile(caCertPath); + const key = await readFile(serverKeyPath); + const cert = await readFile(serverCertPath); + + let closed = false; + return Object.freeze({ + tls: Object.freeze({ key, cert, ca }), + trustBundle: pem(ca), + async signVmCsr({ vmIdentity, csr } = {}) { + if (closed) throw new Error("proxy TLS identity authority is closed"); + if (typeof vmIdentity !== "string" || !SAFE_VM_IDENTITY.test(vmIdentity) + || typeof csr !== "string" || csr.length < 100 || csr.length > 16_384 + || !csr.startsWith("-----BEGIN CERTIFICATE REQUEST-----\n") + || !csr.trimEnd().endsWith("-----END CERTIFICATE REQUEST-----")) { + throw new TypeError("invalid VM certificate request"); + } + const identityUri = `urn:factorize:vm:${Buffer.from(vmIdentity, "utf8").toString("base64url")}`; + const csrPath = join(directory, `vm-${randomUUID()}.csr`); + const certPath = join(directory, `vm-${randomUUID()}.pem`); + const extensionPath = join(directory, `vm-${randomUUID()}.ext`); + await writeFile(csrPath, csr, { mode: 0o600 }); + await writeFile(extensionPath, `basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyAgreement\nextendedKeyUsage=clientAuth\nsubjectAltName=URI:${identityUri}\n`, { mode: 0o600 }); + try { + await openssl(["req", "-in", csrPath, "-verify", "-noout"]); + await openssl(["x509", "-req", "-in", csrPath, "-CA", caCertPath, "-CAkey", caKeyPath, "-CAcreateserial", "-days", "1", "-sha256", "-subj", `/CN=${vmIdentity}`, "-extfile", extensionPath, "-out", certPath]); + return Object.freeze({ certificate: pem(await readFile(certPath)), trustBundle: pem(ca) }); + } finally { + await Promise.all([rm(csrPath, { force: true }), rm(certPath, { force: true }), rm(extensionPath, { force: true })]); + } + }, + async close() { + if (closed) return; + closed = true; + await rm(directory, { recursive: true, force: true }); + }, + }); + } catch (error) { + await rm(directory, { recursive: true, force: true }); + throw error; + } +} diff --git a/packages/credential-proxy/src/vm-certificate.mjs b/packages/credential-proxy/src/vm-certificate.mjs new file mode 100644 index 00000000..a06a743a --- /dev/null +++ b/packages/credential-proxy/src/vm-certificate.mjs @@ -0,0 +1,61 @@ +import { connect } from "node:net"; + +const SOCKET_PATH = "/run/factorize-proxy/bootstrap.sock"; +const MAX_REQUEST_BYTES = 20 * 1024; +const MAX_RESPONSE_BYTES = 40 * 1024; + +async function readRequest() { + let length = 0; + const chunks = []; + for await (const chunk of process.stdin) { + length += chunk.byteLength; + if (length > MAX_REQUEST_BYTES) throw new Error("invalid certificate request"); + chunks.push(chunk); + } + if (!length || chunks.at(-1).at(-1) !== 10) throw new Error("invalid certificate request"); + const value = JSON.parse(Buffer.concat(chunks).toString("utf8")); + if (!value || typeof value !== "object" || Array.isArray(value) + || Object.keys(value).sort().join(",") !== "bootEpoch,csr,vmIdentity" + || typeof value.bootEpoch !== "string" || typeof value.csr !== "string" || typeof value.vmIdentity !== "string") { + throw new Error("invalid certificate request"); + } + return { type: "issue-vm-certificate", ...value }; +} + +async function send(request) { + const socket = connect(SOCKET_PATH); + try { + return await new Promise((resolve, reject) => { + let data = Buffer.alloc(0); + socket.on("data", chunk => { + data = Buffer.concat([data, chunk]); + if (data.length > MAX_RESPONSE_BYTES) { + reject(new Error("invalid certificate response")); + socket.destroy(); + return; + } + const end = data.indexOf(10); + if (end < 0) return; + try { resolve(JSON.parse(data.subarray(0, end).toString("utf8"))); } + catch { reject(new Error("invalid certificate response")); } + socket.end(); + }); + socket.once("error", () => reject(new Error("proxy certificate service unavailable"))); + socket.setTimeout(7000, () => reject(new Error("proxy certificate service timed out"))); + socket.once("connect", () => socket.write(`${JSON.stringify(request)}\n`)); + }); + } finally { + socket.destroy(); + } +} + +try { + const request = await readRequest(); + const result = await send(request); + if (result?.issued !== true || result.bootEpoch !== request.bootEpoch || result.vmIdentity !== request.vmIdentity + || typeof result.certificate !== "string" || typeof result.trustBundle !== "string") throw new Error("VM certificate was not issued"); + process.stdout.write(`${JSON.stringify({ bootEpoch: result.bootEpoch, vmIdentity: result.vmIdentity, certificate: result.certificate, trustBundle: result.trustBundle })}\n`); +} catch { + process.stderr.write("VM certificate provisioning failed\n"); + process.exitCode = 1; +} diff --git a/packages/credential-proxy/test/bootstrap-control.test.mjs b/packages/credential-proxy/test/bootstrap-control.test.mjs index ece8a05e..5e7956ac 100644 --- a/packages/credential-proxy/test/bootstrap-control.test.mjs +++ b/packages/credential-proxy/test/bootstrap-control.test.mjs @@ -17,11 +17,11 @@ afterEach(async () => { directory = undefined; }); -async function fixture(register, onRegistered) { +async function fixture(register, onRegistered, issueVmCertificate) { directory = await mkdtemp(join(tmpdir(), "factorize-proxy-startup-")); const socketPath = join(directory, "control.sock"); const runtime = { bootEpoch: epoch }; - service = await startProxyBootstrapControl({ runtime, socketPath, register, onRegistered }); + service = await startProxyBootstrapControl({ runtime, socketPath, register, onRegistered, issueVmCertificate }); return socketPath; } @@ -42,7 +42,7 @@ test("local startup control redeems the one-use credential and leaves a private async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }; }, (value, request) => registered.push({ value, request }), ); - const result = await request(socketPath, { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + const result = await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); assert.deepEqual(result, { registered: true, proxyIdentity: "fly:app:machine", bootEpoch: epoch }); assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch }]); assert.deepEqual(registered, [{ @@ -57,9 +57,29 @@ test("local startup control rejects run credentials, epoch substitution, and ext const calls = []; const socketPath = await fixture(async value => { calls.push(value); return {}; }); for (const payload of [ - { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, - { controlOrigin: "https://factorize.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, - { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, + { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, + { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, + { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, ]) assert.deepEqual(await request(socketPath, payload), { error: "invalid bootstrap request" }); assert.deepEqual(calls, []); }); + +test("local provider channel returns only a boot-bound VM certificate and public trust bundle", async () => { + const csr = `-----BEGIN CERTIFICATE REQUEST-----\n${"A".repeat(120)}\n-----END CERTIFICATE REQUEST-----`; + let registered = false; + const socketPath = await fixture( + async () => ({ proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }), + async () => { registered = true; }, + async request => { + assert.equal(registered, true); + return { certificate: `cert-for:${request.vmIdentity}`, trustBundle: "public-ca" }; + }, + ); + const payload = { type: "issue-vm-certificate", bootEpoch: epoch, vmIdentity: "fly:job-1", csr }; + assert.deepEqual(await request(socketPath, payload), { error: "certificate provisioning failed" }); + await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + assert.deepEqual(await request(socketPath, payload), { + issued: true, bootEpoch: epoch, vmIdentity: "fly:job-1", certificate: "cert-for:fly:job-1", trustBundle: "public-ca", + }); + assert.equal(JSON.stringify(await request(socketPath, payload)).includes(token), false); +}); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index c8386843..1c275e3b 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, registerProxyBoot } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, readProxyRunRevocations, registerProxyBoot, startProxyRunRevocationMonitor } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -34,6 +34,33 @@ test("proxy snapshot reader requests only the bound run revision and validates e await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ownerId: "owner-1", records: [{ ...snapshot.records[0], kind: "model" }] }) }), /invalid proxy integration snapshot response/); }); +test("proxy revocation monitor applies durable run revocations and tolerates an outage", async () => { + const requests = []; + const revoked = []; + let failFirst = true; + const fetchImpl = async (url, init) => { + requests.push({ url: String(url), init, body: JSON.parse(init.body) }); + if (failFirst) { failFirst = false; throw new Error("offline fixture"); } + return Response.json({ revokedRunIds: [runId] }); + }; + const runtime = { revokeRun: async id => { revoked.push(id); } }; + const monitor = startProxyRunRevocationMonitor({ + controlOrigin: "https://factorize.example", controlCredential: token, bootEpoch, runtime, intervalMs: 100, fetchImpl, + }); + try { + const deadline = Date.now() + 1_000; + while (!revoked.length && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 10)); + assert.deepEqual(revoked, [runId]); + assert.equal(requests.length, 2, "a failed poll leaves the proxy live and retries through the control plane"); + assert.ok(requests.every(request => request.url === "https://factorize.example/internal/credential-proxy/revocations")); + assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${token}`)); + assert.ok(requests.every(request => request.body.bootEpoch === bootEpoch)); + } finally { + await monitor.close(); + } + await assert.rejects(readProxyRunRevocations({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async () => Response.json({ revokedRunIds: ["invalid"] }) }), /invalid proxy revocation response/); +}); + test("boot registration rejects unsafe origins, invalid credentials and mismatched response bindings", async () => { await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch }), /HTTPS origin/); await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); diff --git a/packages/credential-proxy/test/tls-identity.test.mjs b/packages/credential-proxy/test/tls-identity.test.mjs new file mode 100644 index 00000000..dd09554d --- /dev/null +++ b/packages/credential-proxy/test/tls-identity.test.mjs @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { execFile as execFileCallback } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { test } from "node:test"; +import { createProxyTlsAuthority } from "../src/tls-identity.mjs"; + +const execFile = promisify(execFileCallback); +const bootEpoch = "11111111-1111-4111-8111-111111111111"; + +test("proxy TLS authority provisions a boot-bound server cert and signs only canonical VM identities", async () => { + const directory = await mkdtemp(join(tmpdir(), "factorize-tls-test-")); + const authority = await createProxyTlsAuthority({ serverName: "proxy.internal.test", bootEpoch }); + try { + const keyPath = join(directory, "vm-key.pem"); + const csrPath = join(directory, "vm.csr"); + await execFile("openssl", ["genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", "-out", keyPath]); + await execFile("openssl", ["req", "-new", "-key", keyPath, "-subj", "/CN=ignored-by-issuer", "-out", csrPath]); + const csr = await readFile(csrPath, "utf8"); + const issued = await authority.signVmCsr({ vmIdentity: "fly:machine-7", csr }); + assert.equal(issued.trustBundle, authority.trustBundle); + assert.match(issued.certificate, /BEGIN CERTIFICATE/); + await writeFile(join(directory, "root.pem"), issued.trustBundle, { mode: 0o600 }); + await writeFile(join(directory, "client.pem"), issued.certificate, { mode: 0o600 }); + await execFile("openssl", ["verify", "-CAfile", join(directory, "root.pem"), join(directory, "client.pem")]); + const { stdout: clientText } = await execFile("openssl", ["x509", "-in", join(directory, "client.pem"), "-noout", "-subject", "-ext", "subjectAltName"]); + assert.match(clientText, /CN\s*=\s*fly:machine-7/); + assert.match(clientText, /URI:urn:factorize:vm:Zmx5Om1hY2hpbmUtNw/); + const { stdout: usageText } = await execFile("openssl", ["x509", "-in", join(directory, "client.pem"), "-noout", "-ext", "extendedKeyUsage"]); + assert.match(usageText, /TLS Web Client Authentication/); + + await writeFile(join(directory, "server.pem"), authority.tls.cert, { mode: 0o600 }); + await writeFile(join(directory, "server-root.pem"), authority.tls.ca, { mode: 0o600 }); + await execFile("openssl", ["verify", "-CAfile", join(directory, "server-root.pem"), "-verify_hostname", "proxy.internal.test", join(directory, "server.pem")]); + await assert.rejects(authority.signVmCsr({ vmIdentity: "../machine", csr }), /invalid VM certificate request/); + await assert.rejects(authority.signVmCsr({ vmIdentity: "fly:machine-7", csr: "not a CSR" })); + await authority.close(); + await assert.rejects(authority.signVmCsr({ vmIdentity: "fly:machine-7", csr }), /authority is closed/); + } finally { + await authority.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("proxy TLS authority rejects malformed server names and boot identities", async () => { + await assert.rejects(createProxyTlsAuthority({ serverName: "https://proxy.test", bootEpoch })); + await assert.rejects(createProxyTlsAuthority({ serverName: "proxy.test", bootEpoch: "old" })); +}); From a3aad85f70dcd4cf0d7f1145b271c987c17da31c Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 23:58:36 +0000 Subject: [PATCH 028/102] GEN-2824 include selected model account snapshot --- packages/api/src/proxy-control.ts | 15 ++++++++++++++- packages/api/test/proxy-control.test.ts | 14 +++++++++----- packages/credential-proxy/src/control-client.mjs | 5 ++++- .../credential-proxy/test/control-client.test.mjs | 3 ++- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index fa91e53c..18ef39f3 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -68,7 +68,20 @@ export async function proxyControlFetch(request: Request, env: Parameters selected.includes(record.id)) : []; if (records.length !== selected.length) return json({ error: "integration_snapshot_unavailable" }, 403); - return json({ ownerId: proxy.tenantId, records }); + const modelAccess = run.execution_config?.modelAccess ?? "native"; + const modelAccountId = run.execution_config?.modelAccountId; + let modelAccount = null; + if (modelAccess === "portable") { + if (typeof modelAccountId !== "string") return json({ error: "integration_snapshot_unavailable" }, 403); + modelAccount = (await new EncryptedIntegrationRepository(database, proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("model")) + .find(record => record.id === modelAccountId) ?? null; + if (!modelAccount) return json({ error: "integration_snapshot_unavailable" }, 403); + } else if (modelAccess !== "native" || modelAccountId !== undefined) { + return json({ error: "integration_snapshot_unavailable" }, 403); + } + // Only the selected model account's user-encrypted payload crosses the + // control plane. It remains encrypted until the proxy is unlocked. + return json({ ownerId: proxy.tenantId, records, modelAccount }); } const enrollments = new RunEnrollmentRepository(databaseFor(env)); if (url.pathname === "/internal/credential-proxy/enroll") { diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 7a8cd90d..42a02a58 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -122,17 +122,21 @@ describe("dedicated proxy boot registration endpoint", () => { expect(rejected.status).toBe(400); }); - it("returns only the current run's selected user-encrypted bearer records to its registered proxy", async () => { + it("returns only the current run's selected user-encrypted bearer and model records to its registered proxy", async () => { const integrationId = "55555555-5555-4555-8555-555555555555"; + const modelAccountId = "77777777-7777-4777-8777-777777777777"; const key = Buffer.alloc(32, 7).toString("base64"); const env = envFor(); env.CREDENTIAL_ENCRYPTION_KEY = key; - env.query.mockImplementation(async (sql: string) => { + env.query.mockImplementation(async (sql: string, values: unknown[] = []) => { if (sql.includes("proxy_token_digest")) return { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 }; - if (sql.includes("FROM app.runs r")) return { rows: [{ execution_config: { portableIntegrationIds: [integrationId] }, grant_revision: 3 }], rowCount: 1 }; - if (sql.includes("FROM app.integration_records")) return { rows: [ + if (sql.includes("FROM app.runs r")) return { rows: [{ execution_config: { portableIntegrationIds: [integrationId], modelAccess: "portable", modelAccountId }, grant_revision: 3 }], rowCount: 1 }; + if (sql.includes("FROM app.integration_records")) return values[1] === "bearer" ? { rows: [ { id: integrationId, kind: "bearer", name: "selected", origin: "https://api.example", description: null, revision: 4, encrypted_payload: await encrypt("user-ciphertext", key) }, { id: "66666666-6666-4666-8666-666666666666", kind: "bearer", name: "not-selected", origin: "https://other.example", description: null, revision: 5, encrypted_payload: await encrypt("other-ciphertext", key) }, + ], rowCount: 2 } : { rows: [ + { id: modelAccountId, kind: "model", name: "selected-model", origin: null, description: "fixture account", revision: 6, encrypted_payload: await encrypt("model-user-ciphertext", key) }, + { id: "88888888-8888-4888-8888-888888888888", kind: "model", name: "other-model", origin: null, description: null, revision: 7, encrypted_payload: await encrypt("other-model-ciphertext", key) }, ], rowCount: 2 }; return { rows: [], rowCount: 0 }; }); @@ -144,7 +148,7 @@ describe("dedicated proxy boot registration endpoint", () => { expect(response.status).toBe(200); expect(await response.json()).toEqual({ ownerId: "11111111-1111-4111-8111-111111111111", records: [ { id: integrationId, kind: "bearer", name: "selected", origin: "https://api.example", description: null, revision: 4, encryptedPayload: "user-ciphertext" }, - ] }); + ], modelAccount: { id: modelAccountId, kind: "model", name: "selected-model", origin: null, description: "fixture account", revision: 6, encryptedPayload: "model-user-ciphertext" } }); expect(env.query.mock.calls.map((call: any[]) => call[0])).toEqual(expect.arrayContaining([expect.stringContaining("grant_revision=$5"), expect.stringContaining("FROM app.integration_records")])); }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index fd1b4c46..f36c3fa4 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -65,9 +65,12 @@ export async function readProxyIntegrationSnapshot({ controlOrigin, token, bindi body: { runId: binding.runId, grantRevision: binding.grantRevision }, fetchImpl, }); + const modelAccount = result?.modelAccount; if (typeof result?.ownerId !== "string" || !Array.isArray(result.records) || result.records.some(record => !record || typeof record.id !== "string" || record.kind !== "bearer" - || typeof record.origin !== "string" || typeof record.encryptedPayload !== "string" || !Number.isSafeInteger(record.revision))) { + || typeof record.origin !== "string" || typeof record.encryptedPayload !== "string" || !Number.isSafeInteger(record.revision)) + || !(modelAccount === null || (modelAccount && typeof modelAccount.id === "string" && modelAccount.kind === "model" + && modelAccount.origin === null && typeof modelAccount.encryptedPayload === "string" && Number.isSafeInteger(modelAccount.revision)))) { throw new Error("invalid proxy integration snapshot response"); } return result; diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 1c275e3b..b3ab4cbd 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -22,7 +22,7 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an test("proxy snapshot reader requests only the bound run revision and validates encrypted records", async () => { let request; - const snapshot = { ownerId: "owner-1", records: [{ id: "integration-1", kind: "bearer", origin: "https://api.example", revision: 4, encryptedPayload: "ciphertext" }] }; + const snapshot = { ownerId: "owner-1", records: [{ id: "integration-1", kind: "bearer", origin: "https://api.example", revision: 4, encryptedPayload: "ciphertext" }], modelAccount: { id: "model-1", kind: "model", origin: null, revision: 5, encryptedPayload: "model-ciphertext" } }; const result = await readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async (url, init) => { request = { url: String(url), init, body: JSON.parse(init.body) }; return Response.json(snapshot); @@ -32,6 +32,7 @@ test("proxy snapshot reader requests only the bound run revision and validates e assert.deepEqual(request.body, { runId, grantRevision: 7 }); assert.equal(request.init.headers.Authorization, `Bearer ${token}`); await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ownerId: "owner-1", records: [{ ...snapshot.records[0], kind: "model" }] }) }), /invalid proxy integration snapshot response/); + await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ...snapshot, modelAccount: { ...snapshot.modelAccount, kind: "bearer" } }) }), /invalid proxy integration snapshot response/); }); test("proxy revocation monitor applies durable run revocations and tolerates an outage", async () => { From 463ce5333ec72637620d1eed2ab19d36c19dd140 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:02:00 +0000 Subject: [PATCH 029/102] GEN-2824 update snapshot fixture for model selection --- packages/credential-proxy/test/decrypted-grants.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs index 18ad4d62..5b41501e 100644 --- a/packages/credential-proxy/test/decrypted-grants.test.mjs +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -80,7 +80,7 @@ test("production grant resolver reads the run-bound snapshot and only decrypts w controlCredential: `fzc_${"c".repeat(43)}`, fetchImpl: async (url, init) => { requests.push({ url: String(url), authorization: new Headers(init.headers).get("authorization"), body: JSON.parse(init.body) }); - return Response.json({ ownerId, records: [{ ...record, encryptedPayload }] }); + return Response.json({ ownerId, records: [{ ...record, encryptedPayload }], modelAccount: null }); }, }); const binding = { tenantId: ownerId, runId: "22222222-2222-4222-8222-222222222222", grantRevision: 7 }; From 44c5bde8f2b7989429e3a6cfe3f295c60edc95ec Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:25:34 +0000 Subject: [PATCH 030/102] GEN-2824 add isolated Codex auth helper --- .../src/codex-auth-helper.mjs | 158 ++++++++++++++++++ .../test/codex-auth-helper.test.mjs | 84 ++++++++++ 2 files changed, 242 insertions(+) create mode 100644 packages/credential-proxy/src/codex-auth-helper.mjs create mode 100644 packages/credential-proxy/test/codex-auth-helper.test.mjs diff --git a/packages/credential-proxy/src/codex-auth-helper.mjs b/packages/credential-proxy/src/codex-auth-helper.mjs new file mode 100644 index 00000000..199d906f --- /dev/null +++ b/packages/credential-proxy/src/codex-auth-helper.mjs @@ -0,0 +1,158 @@ +import { mkdir, lstat } from "node:fs/promises"; +import { resolve, sep } from "node:path"; +import { spawn as spawnDefault } from "node:child_process"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const PINNED_VERSION = "codex-cli 0.167.1"; +const METHODS = new Set(["device-login", "account-status", "refresh", "cancel", "logout"]); + +function accountHome(root, accountId) { + if (typeof accountId !== "string" || !UUID.test(accountId)) throw new TypeError("invalid model account id"); + const base = resolve(root); + const home = resolve(base, accountId); + if (!home.startsWith(`${base}${sep}`)) throw new TypeError("invalid model account id"); + return { base, home }; +} + +/** + * Narrow in-process adapter for Codex authentication commands. Callers must + * keep this object on the proxy's private control channel; it is not a job API. + * Each account gets its own normal Codex credential home. + */ +export function createCodexAuthHelper({ + codexPath = "/usr/local/bin/codex", + accountHomeRoot = "/var/lib/factorize/codex-accounts", + spawnProcess = spawnDefault, + expectedVersion = PINNED_VERSION, +} = {}) { + const logins = new Map(); + + async function prepareHome(accountId) { + const { base, home } = accountHome(accountHomeRoot, accountId); + await mkdir(base, { recursive: true, mode: 0o700 }); + await mkdir(home, { recursive: true, mode: 0o700 }); + const stat = await lstat(home); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Codex account home is not a private directory"); + return home; + } + + function run(accountId, args, { timeoutMs = 10_000, input, onStdout } = {}) { + return prepareHome(accountId).then(home => new Promise((resolveRun, rejectRun) => { + let stdout = "", stderr = "", settled = false; + const child = spawnProcess(codexPath, args, { + env: { PATH: process.env.PATH ?? "", CODEX_HOME: home, HOME: home }, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + const finish = (error, result) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) rejectRun(error); + else resolveRun(result); + }; + const collect = (current, chunk, limit) => { + const next = current + chunk.toString("utf8"); + if (Buffer.byteLength(next) > limit) { + child.kill("SIGTERM"); + finish(new Error("Codex authentication helper output exceeded its limit")); + return current; + } + return next; + }; + child.stdout.on("data", chunk => { + stdout = collect(stdout, chunk, 16_384); + if (!settled) onStdout?.(chunk.toString("utf8")); + }); + child.stderr.on("data", chunk => { stderr = collect(stderr, chunk, 16_384); }); + child.once("error", () => finish(new Error("Codex authentication helper could not start"))); + child.once("close", (code, signal) => { + if (code === 0) finish(null, { code, stdout, stderr }); + else finish(new Error(signal ? `Codex authentication helper stopped (${signal})` : `Codex authentication helper failed (${code ?? "unknown"})`)); + }); + const timer = setTimeout(() => { + child.kill("SIGTERM"); + finish(new Error("Codex authentication helper timed out")); + }, timeoutMs); + timer.unref?.(); + if (input !== undefined) child.stdin.end(input); + else child.stdin.end(); + return child; + })); + } + + async function checkVersion(accountId) { + // `--version` is the only non-authentication command accepted by this helper. + const result = await run(accountId, ["--version"]); + if (result.stdout.trim() !== expectedVersion) throw new Error("Unsupported Codex version for model account authentication"); + } + + async function invoke(request) { + if (!request || typeof request !== "object" || Array.isArray(request) + || !METHODS.has(request.method) || Object.keys(request).some(key => !["method", "accountId"].includes(key))) { + throw new TypeError("unsupported Codex authentication operation"); + } + const { accountId } = request; + accountHome(accountHomeRoot, accountId); + await checkVersion(accountId); + if (request.method === "device-login") { + if (logins.has(accountId)) throw new Error("device authorization is already running for this account"); + const home = await prepareHome(accountId); + let child; + const completion = new Promise((resolveRun, rejectRun) => { + child = spawnProcess(codexPath, ["login", "--device-auth"], { + env: { PATH: process.env.PATH ?? "", CODEX_HOME: home, HOME: home }, + stdio: ["ignore", "pipe", "pipe"], windowsHide: true, + }); + let stdout = ""; + const timer = setTimeout(() => { child.kill("SIGTERM"); rejectRun(new Error("Codex device authorization timed out")); }, 15 * 60_000); + timer.unref?.(); + child.stdout.on("data", chunk => { + stdout = (stdout + chunk.toString("utf8")).slice(-16_384); + }); + child.stderr.on("data", chunk => { + // Retain no provider error text, which could include device details. + if (chunk.byteLength > 16_384) child.kill("SIGTERM"); + }); + child.once("error", () => { clearTimeout(timer); rejectRun(new Error("Codex authentication helper could not start")); }); + child.once("close", (code, signal) => { + clearTimeout(timer); + if (code === 0) resolveRun({ state: "authorized", output: stdout }); + else rejectRun(new Error(signal ? `Codex device authorization stopped (${signal})` : `Codex device authorization failed (${code ?? "unknown"})`)); + }); + }); + const entry = { child, completion }; + logins.set(accountId, entry); + completion.finally(() => { if (logins.get(accountId) === entry) logins.delete(accountId); }).catch(() => {}); + return { state: "pending", completion }; + } + if (request.method === "cancel") { + const login = logins.get(accountId); + if (!login) return { state: "idle" }; + login.child.kill("SIGTERM"); + logins.delete(accountId); + return { state: "canceled" }; + } + if (request.method === "logout") { + const login = logins.get(accountId); + if (login) { + login.child.kill("SIGTERM"); + logins.delete(accountId); + } + await run(accountId, ["logout"]); + return { state: "logged-out" }; + } + if (request.method === "account-status") { + const result = await run(accountId, ["login", "status"]); + return { state: result.stdout.trim() ? "authenticated" : "unauthenticated" }; + } + // Codex owns refresh semantics in its authentication manager. A status call + // exercises that manager without opening the agent command surface. + const result = await run(accountId, ["login", "status"]); + return { state: result.stdout.trim() ? "refreshed" : "reauthentication-required" }; + } + + return Object.freeze({ invoke }); +} + +export { PINNED_VERSION as CODEX_AUTH_HELPER_VERSION }; diff --git a/packages/credential-proxy/test/codex-auth-helper.test.mjs b/packages/credential-proxy/test/codex-auth-helper.test.mjs new file mode 100644 index 00000000..1b386c1d --- /dev/null +++ b/packages/credential-proxy/test/codex-auth-helper.test.mjs @@ -0,0 +1,84 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createCodexAuthHelper, CODEX_AUTH_HELPER_VERSION } from "../src/codex-auth-helper.mjs"; + +const accountA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const accountB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + +function fakeCodex() { + const calls = [], children = []; + const spawnProcess = (command, args, options) => { + calls.push({ command, args, options }); + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.stdin = new PassThrough(); + child.kill = signal => { + child.killed = signal; + queueMicrotask(() => child.emit("close", null, signal)); + return true; + }; + children.push(child); + if (args[0] !== "login" || args[1] !== "--device-auth") { + queueMicrotask(() => { + child.stdout.end(args[0] === "--version" ? `${CODEX_AUTH_HELPER_VERSION}\n` : "authenticated\n"); + child.stderr.end(); + child.emit("close", 0, null); + }); + } + return child; + }; + return { calls, children, spawnProcess }; +} + +test("Codex auth helper constrains commands and isolates account credential homes", async () => { + const directory = await mkdtemp(join(tmpdir(), "factorize-codex-auth-")); + try { + const fake = fakeCodex(); + const helper = createCodexAuthHelper({ codexPath: "/opt/codex", accountHomeRoot: directory, spawnProcess: fake.spawnProcess }); + assert.deepEqual(await helper.invoke({ method: "account-status", accountId: accountA }), { state: "authenticated" }); + assert.deepEqual(await helper.invoke({ method: "refresh", accountId: accountB }), { state: "refreshed" }); + assert.deepEqual(fake.calls.map(call => call.args), [ + ["--version"], ["login", "status"], ["--version"], ["login", "status"], + ]); + assert.ok(fake.calls.every(call => call.command === "/opt/codex")); + assert.equal(fake.calls[0].options.env.CODEX_HOME, join(directory, accountA)); + assert.equal(fake.calls[3].options.env.CODEX_HOME, join(directory, accountB)); + assert.equal(fake.calls[0].options.env.HOME, join(directory, accountA)); + assert.equal("OPENAI_API_KEY" in fake.calls[0].options.env, false); + await assert.rejects(helper.invoke({ method: "exec", accountId: accountA, args: ["--dangerously-bypass-approvals-and-sandbox"] }), /unsupported Codex authentication operation/); + await assert.rejects(helper.invoke({ method: "logout", accountId: "../../outside" }), /invalid model account id/); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +test("device authorization runs as a separate cancellable login process", async () => { + const directory = await mkdtemp(join(tmpdir(), "factorize-codex-auth-")); + try { + const fake = fakeCodex(); + const helper = createCodexAuthHelper({ accountHomeRoot: directory, spawnProcess: fake.spawnProcess }); + const result = await helper.invoke({ method: "device-login", accountId: accountA }); + assert.equal(result.state, "pending"); + assert.deepEqual(fake.calls.map(call => call.args), [["--version"], ["login", "--device-auth"]]); + assert.equal(fake.calls[1].options.env.CODEX_HOME, join(directory, accountA)); + assert.deepEqual(await helper.invoke({ method: "cancel", accountId: accountA }), { state: "canceled" }); + await assert.rejects(result.completion, /stopped/); + assert.deepEqual(await helper.invoke({ method: "cancel", accountId: accountA }), { state: "idle" }); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +test("logout terminates pending device login and invokes only Codex logout", async () => { + const directory = await mkdtemp(join(tmpdir(), "factorize-codex-auth-")); + try { + const fake = fakeCodex(); + const helper = createCodexAuthHelper({ accountHomeRoot: directory, spawnProcess: fake.spawnProcess }); + const result = await helper.invoke({ method: "device-login", accountId: accountA }); + assert.deepEqual(await helper.invoke({ method: "logout", accountId: accountA }), { state: "logged-out" }); + await assert.rejects(result.completion, /stopped/); + assert.deepEqual(fake.calls.map(call => call.args), [["--version"], ["login", "--device-auth"], ["--version"], ["logout"]]); + } finally { await rm(directory, { recursive: true, force: true }); } +}); From 69f05bc727695014f91a1fb232478b9916a90ed5 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:41:37 +0000 Subject: [PATCH 031/102] GEN-2824 persist encrypted Codex account refreshes --- .../src/model-account-lifecycle.mjs | 154 ++++++++++++++++++ .../test/model-account-lifecycle.test.mjs | 113 +++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 packages/credential-proxy/src/model-account-lifecycle.mjs create mode 100644 packages/credential-proxy/test/model-account-lifecycle.test.mjs diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs new file mode 100644 index 00000000..236d4008 --- /dev/null +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -0,0 +1,154 @@ +import { readFile, rm } from "node:fs/promises"; +import { join, resolve, sep } from "node:path"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function cachePath(root, accountId) { + if (!UUID.test(accountId ?? "")) throw new TypeError("invalid model account id"); + const base = resolve(root), path = resolve(base, accountId, "auth.json"); + if (!path.startsWith(`${base}${sep}`)) throw new TypeError("invalid model account id"); + return path; +} + +/** + * Keep Codex token refresh and encrypted persistence inside the proxy. The + * persistence callback receives only user-encrypted auth-cache bytes; token + * JSON and the unlock key never leave this process in plaintext. + */ +export function createModelAccountLifecycle({ + authHelper, + accountHomeRoot = "/var/lib/factorize/codex-accounts", + getUnlockKey, + getAccount, + encryptCache, + persistEncrypted, + onUnsaved = () => {}, + retryDelaysMs = [100, 500, 2_000], + sleep = ms => new Promise(resolveSleep => setTimeout(resolveSleep, ms)), + removeCache = rm, + readCache = readFile, +} = {}) { + if (!authHelper || typeof authHelper.invoke !== "function" || typeof getUnlockKey !== "function" + || typeof getAccount !== "function" || typeof encryptCache !== "function" || typeof persistEncrypted !== "function" + || !Array.isArray(retryDelaysMs) || retryDelaysMs.length > 5 + || retryDelaysMs.some(ms => !Number.isSafeInteger(ms) || ms < 0 || ms > 30_000)) { + throw new TypeError("model account lifecycle configuration is invalid"); + } + + const generations = new Map(); + const queues = new Map(); + const unsaved = new Set(); + const currentGeneration = id => generations.get(id) ?? 0; + + function serialize(id, operation) { + const previous = queues.get(id) ?? Promise.resolve(); + const next = previous.catch(() => {}).then(operation); + queues.set(id, next); + return next.finally(() => { if (queues.get(id) === next) queues.delete(id); }); + } + + async function persist(id, generation) { + const account = await getAccount(id); + if (!account || account.id !== id || account.kind !== "model" || !Number.isSafeInteger(account.revision) + || typeof account.ownerId !== "string" || !account.ownerId || typeof account.keyGeneration !== "string" || !account.keyGeneration) { + throw new Error("model account persistence binding is unavailable"); + } + const unlockKey = await getUnlockKey(); + if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); + const plain = await readCache(cachePath(accountHomeRoot, id)); + if (!Buffer.isBuffer(plain) || plain.byteLength === 0 || plain.byteLength > 65_536) throw new Error("Codex token cache is missing or exceeds its limit"); + let encrypted; + try { + encrypted = await encryptCache(new Uint8Array(plain), unlockKey, { + ownerId: account.ownerId, + recordId: id, + recordType: "model", + destination: "factorize", + revision: account.revision + 1, + keyGeneration: account.keyGeneration, + }); + } finally { + plain.fill(0); + } + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted }; + for (let attempt = 0; ; attempt++) { + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + try { + await persistEncrypted(request); + unsaved.delete(id); + onUnsaved({ accountId: id, unsaved: false }); + return { state: "saved" }; + } catch (error) { + if (attempt >= retryDelaysMs.length) { + unsaved.add(id); + onUnsaved({ accountId: id, unsaved: true }); + return { state: "unsaved", attempts: attempt + 1 }; + } + await sleep(retryDelaysMs[attempt]); + } + } + } + + async function refresh(id) { + return serialize(id, async () => { + const generation = currentGeneration(id); + const result = await authHelper.invoke({ method: "refresh", accountId: id }); + if (result.state === "reauthentication-required") return result; + return persist(id, generation); + }); + } + + async function deviceLogin(id) { + return serialize(id, async () => { + const generation = currentGeneration(id); + const result = await authHelper.invoke({ method: "device-login", accountId: id }); + if (result.state !== "pending" || !result.completion || typeof result.completion.then !== "function") throw new Error("Codex device authorization did not start"); + const completion = Promise.resolve(result.completion).then(async outcome => { + if (currentGeneration(id) !== generation) return { state: "deleted" }; + if (outcome?.state !== "authorized") return { state: outcome?.state ?? "reauthentication-required" }; + return persist(id, generation); + }); + return { state: "pending", completion }; + }); + } + + async function cancel(id) { + return serialize(id, async () => authHelper.invoke({ method: "cancel", accountId: id })); + } + + async function deleteAccount(id) { + if (!UUID.test(id ?? "")) throw new TypeError("invalid model account id"); + // Invalidate in-flight save retries immediately; cleanup then waits behind + // any already running helper process before deleting its cache. + generations.set(id, currentGeneration(id) + 1); + return serialize(id, async () => { + await authHelper.invoke({ method: "cancel", accountId: id }).catch(() => {}); + await authHelper.invoke({ method: "logout", accountId: id }).catch(() => {}); + await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); + unsaved.delete(id); + onUnsaved({ accountId: id, unsaved: false }); + return { state: "deleted" }; + }); + } + + return Object.freeze({ + deviceLogin, + cancel, + refresh, + deleteAccount, + isUnsaved: id => unsaved.has(id), + /** Invoke after encrypted snapshots are reconciled at unlock. */ + reconcile: async id => serialize(id, async () => { + if (!await getAccount(id)) { + await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); + unsaved.delete(id); + return { state: "deleted" }; + } + const result = await authHelper.invoke({ method: "account-status", accountId: id }); + return result.state === "authenticated" ? persist(id, currentGeneration(id)) : { state: "reauthentication-required" }; + }), + }); +} + +export { cachePath as modelAccountCachePath }; diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs new file mode 100644 index 00000000..449fbdb4 --- /dev/null +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -0,0 +1,113 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createModelAccountLifecycle } from "../src/model-account-lifecycle.mjs"; + +const accountId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const account = { id: accountId, kind: "model", ownerId: "owner", keyGeneration: "generation", revision: 4 }; + +async function fixture(t, overrides = {}) { + const root = await mkdtemp(join(tmpdir(), "factorize-model-account-")); + t.after(() => rm(root, { recursive: true, force: true })); + const home = join(root, accountId); + await mkdir(home, { recursive: true }); + await writeFile(join(home, "auth.json"), JSON.stringify({ tokens: { access_token: "synthetic-access", refresh_token: "synthetic-refresh" } })); + const calls = [], events = [], methods = []; + const helper = { invoke: async request => { + methods.push(request.method); + if (request.method === "device-login") return { state: "pending", completion: overrides.completion ?? Promise.resolve({ state: "authorized" }) }; + if (request.method === "refresh") { calls.push("refresh"); return { state: "refreshed" }; } + if (request.method === "account-status") return { state: "authenticated" }; + return { state: request.method === "cancel" ? "canceled" : "logged-out" }; + } }; + let saves = 0; + const lifecycle = createModelAccountLifecycle({ + authHelper: helper, + accountHomeRoot: root, + getUnlockKey: async () => "synthetic-unlock-key", + getAccount: async () => overrides.getAccount ? overrides.getAccount() : account, + encryptCache: async (plain, key, binding) => { + assert.equal(key, "synthetic-unlock-key"); + events.push({ type: "encrypt", binding, plaintext: new TextDecoder().decode(plain) }); + return `encrypted.${Buffer.from(plain).toString("base64")}`; + }, + persistEncrypted: async request => { + saves++; + events.push({ type: "persist", request }); + if (overrides.persistEncrypted) return overrides.persistEncrypted(request, saves); + }, + onUnsaved: value => events.push({ type: "unsaved", ...value }), + retryDelaysMs: overrides.retryDelaysMs ?? [], + sleep: async () => {}, + ...overrides.options, + }); + return { root, home, lifecycle, events, methods, calls, get saves() { return saves; } }; +} + +test("device authorization persists only encrypted, account-bound cache after success", async t => { + const f = await fixture(t); + const login = await f.lifecycle.deviceLogin(accountId); + assert.equal(login.state, "pending"); + assert.deepEqual(await login.completion, { state: "saved" }); + const [encrypted, saved] = f.events.filter(event => event.type === "encrypt" || event.type === "persist"); + assert.equal(encrypted.binding.ownerId, "owner"); + assert.equal(encrypted.binding.recordType, "model"); + assert.equal(encrypted.binding.revision, 5); + assert.equal(saved.request.expectedRevision, 4); + assert.match(saved.request.encryptedPayload, /^encrypted\./); + assert.equal("unlockKey" in saved.request, false); +}); + +test("denied or expired device authorization does not persist token state", async t => { + const f = await fixture(t, { completion: Promise.resolve({ state: "reauthentication-required" }) }); + const login = await f.lifecycle.deviceLogin(accountId); + assert.deepEqual(await login.completion, { state: "reauthentication-required" }); + assert.equal(f.saves, 0); +}); + +test("refresh keeps the working local cache and reports bounded encrypted-save failures", async t => { + const f = await fixture(t, { retryDelaysMs: [0, 0], persistEncrypted: async () => { throw new Error("fixture outage"); } }); + assert.deepEqual(await f.lifecycle.refresh(accountId), { state: "unsaved", attempts: 3 }); + assert.equal(f.lifecycle.isUnsaved(accountId), true); + assert.equal(f.calls.length, 1); + assert.equal(f.saves, 3); + assert.deepEqual(f.events.filter(event => event.type === "unsaved").map(event => event.unsaved), [true]); +}); + +test("refresh requests for one account serialize and use the latest revision", async t => { + let revision = 0, inFlight = 0, maximumInFlight = 0; + const f = await fixture(t, { + getAccount: () => ({ ...account, revision }), + persistEncrypted: async () => { + inFlight++; + maximumInFlight = Math.max(maximumInFlight, inFlight); + await new Promise(resolve => setImmediate(resolve)); + revision++; + inFlight--; + }, + }); + await Promise.all([f.lifecycle.refresh(accountId), f.lifecycle.refresh(accountId)]); + assert.equal(maximumInFlight, 1); + assert.equal(f.saves, 2); +}); + +test("deletion invalidates a retry and removes the account cache", async t => { + let release; + const waiting = new Promise(resolve => { release = resolve; }); + const f = await fixture(t, { + retryDelaysMs: [0, 0, 0], + persistEncrypted: async (_request, attempt) => { + if (attempt === 1) { await waiting; throw new Error("first save failed"); } + }, + }); + const refreshing = f.lifecycle.refresh(accountId); + while (!f.saves) await new Promise(resolve => setImmediate(resolve)); + const deleting = f.lifecycle.deleteAccount(accountId); + release(); + await assert.rejects(refreshing, /changed before token persistence/); + assert.deepEqual(await deleting, { state: "deleted" }); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); +}); From 454232c4c1b4a0c89280e8f14fe3d60e378e0188 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:58:22 +0000 Subject: [PATCH 032/102] GEN-2824 encrypt Codex account caches in proxy --- .../src/model-account-crypto.mjs | 61 +++++++++++++++++++ .../test/model-account-crypto.test.mjs | 37 +++++++++++ 2 files changed, 98 insertions(+) create mode 100644 packages/credential-proxy/src/model-account-crypto.mjs create mode 100644 packages/credential-proxy/test/model-account-crypto.test.mjs diff --git a/packages/credential-proxy/src/model-account-crypto.mjs b/packages/credential-proxy/src/model-account-crypto.mjs new file mode 100644 index 00000000..4602c480 --- /dev/null +++ b/packages/credential-proxy/src/model-account-crypto.mjs @@ -0,0 +1,61 @@ +import { webcrypto } from "node:crypto"; + +const KEY_PREFIX = "fzk1."; +const KEY_BYTES = 32; +const NONCE_BYTES = 12; +const SALT = new TextEncoder().encode("factorize.unlock-key.v1"); +const INFO = new TextEncoder().encode("factorize.record.aes-256-gcm.v1"); +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function bindingData(binding) { + if (!binding || typeof binding.ownerId !== "string" || !binding.ownerId + || !UUID.test(binding.accountId ?? "") || !UUID.test(binding.keyGeneration ?? "") + || !Number.isSafeInteger(binding.revision) || binding.revision < 1) { + throw new TypeError("invalid model account encryption binding"); + } + return new TextEncoder().encode(JSON.stringify([ + "factorize-record-v1", binding.ownerId, binding.accountId, "model", "factorize", binding.keyGeneration, binding.revision, + ])); +} + +async function encryptionKey(unlockKey, usage) { + if (typeof unlockKey !== "string" || !unlockKey.startsWith(KEY_PREFIX)) throw new TypeError("malformed unlock key"); + const encoded = unlockKey.slice(KEY_PREFIX.length); + if (!/^[A-Za-z0-9_-]{43}$/.test(encoded)) throw new TypeError("malformed unlock key"); + const raw = Buffer.from(encoded, "base64url"); + if (raw.byteLength !== KEY_BYTES || raw.toString("base64url") !== encoded) throw new TypeError("malformed unlock key"); + try { + const material = await webcrypto.subtle.importKey("raw", raw, "HKDF", false, ["deriveKey"]); + return await webcrypto.subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt: SALT, info: INFO }, material, + { name: "AES-GCM", length: 256 }, false, [usage]); + } finally { raw.fill(0); } +} + +function decodeStored(value, label) { + if (typeof value !== "string" || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) throw new TypeError(`malformed ${label}`); + const bytes = Buffer.from(value, "base64"); + if (bytes.toString("base64") !== value) throw new TypeError(`malformed ${label}`); + return new Uint8Array(bytes); +} + +/** Encrypt a Codex auth cache for persistence. The cache and unlock key remain in the proxy process. */ +export async function encryptModelAccountCache(plaintext, unlockKey, binding) { + if (!(plaintext instanceof Uint8Array) || plaintext.byteLength === 0 || plaintext.byteLength > 65_536) throw new TypeError("invalid model account cache"); + const aad = bindingData(binding), nonce = webcrypto.getRandomValues(new Uint8Array(NONCE_BYTES)); + const key = await encryptionKey(unlockKey, "encrypt"); + const ciphertext = new Uint8Array(await webcrypto.subtle.encrypt({ name: "AES-GCM", iv: nonce, additionalData: aad, tagLength: 128 }, key, plaintext)); + return `${Buffer.from(nonce).toString("base64")}.${Buffer.from(ciphertext).toString("base64")}`; +} + +/** Decrypt only after the selected account, owner, key generation, and revision are verified. */ +export async function decryptModelAccountCache(encryptedPayload, unlockKey, binding) { + if (typeof encryptedPayload !== "string" || encryptedPayload.length > 65_536) throw new TypeError("malformed model account ciphertext"); + const parts = encryptedPayload.split("."); + if (parts.length !== 2) throw new TypeError("malformed model account ciphertext"); + const nonce = decodeStored(parts[0], "model account nonce"), ciphertext = decodeStored(parts[1], "model account ciphertext"); + if (nonce.byteLength !== NONCE_BYTES || ciphertext.byteLength < 16) throw new TypeError("malformed model account ciphertext"); + const aad = bindingData(binding), key = await encryptionKey(unlockKey, "decrypt"); + try { + return new Uint8Array(await webcrypto.subtle.decrypt({ name: "AES-GCM", iv: nonce, additionalData: aad, tagLength: 128 }, key, ciphertext)); + } finally { nonce.fill(0); ciphertext.fill(0); } +} diff --git a/packages/credential-proxy/test/model-account-crypto.test.mjs b/packages/credential-proxy/test/model-account-crypto.test.mjs new file mode 100644 index 00000000..2308a6df --- /dev/null +++ b/packages/credential-proxy/test/model-account-crypto.test.mjs @@ -0,0 +1,37 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; +import { decryptModelAccountCache, encryptModelAccountCache } from "../src/model-account-crypto.mjs"; + +const ownerId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const accountId = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; +const otherAccountId = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; +const keyGeneration = "dddddddd-dddd-4ddd-8ddd-dddddddddddd"; +const unlockKey = `fzk1.${randomBytes(32).toString("base64url")}`; +const binding = { ownerId, accountId, keyGeneration, revision: 7 }; + +test("model account cache round-trips only for its complete owner/account/generation/revision binding", async () => { + const plaintext = new TextEncoder().encode('{"access_token":"synthetic-model-token"}'); + const encrypted = await encryptModelAccountCache(plaintext, unlockKey, binding); + assert.equal(encrypted.includes("synthetic-model-token"), false); + const restored = await decryptModelAccountCache(encrypted, unlockKey, binding); + assert.deepEqual(restored, plaintext); + restored.fill(0); + plaintext.fill(0); + + for (const changed of [ + { ...binding, ownerId: "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee" }, + { ...binding, accountId: otherAccountId }, + { ...binding, keyGeneration: "ffffffff-ffff-4fff-8fff-ffffffffffff" }, + { ...binding, revision: 8 }, + ]) await assert.rejects(decryptModelAccountCache(encrypted, unlockKey, changed)); + await assert.rejects(decryptModelAccountCache(encrypted, `fzk1.${randomBytes(32).toString("base64url")}`, binding)); +}); + +test("model account cache rejects malformed keys, ciphertext, and metadata", async () => { + const plaintext = new TextEncoder().encode("synthetic-cache"); + await assert.rejects(encryptModelAccountCache(plaintext, "not-an-unlock-key", binding), /malformed unlock key/); + await assert.rejects(encryptModelAccountCache(plaintext, unlockKey, { ...binding, revision: 0 }), /invalid model account encryption binding/); + await assert.rejects(decryptModelAccountCache("bad.payload.extra", unlockKey, binding), /malformed model account ciphertext/); + await assert.rejects(decryptModelAccountCache("AA==.AA==", unlockKey, binding), /malformed model account ciphertext/); +}); From 28f8f1c45139632bbab85fb1dacd37cf4e0eeaf7 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 01:18:17 +0000 Subject: [PATCH 033/102] GEN-2824 restore encrypted Codex caches on unlock --- .../src/model-account-lifecycle.mjs | 63 ++++++++++++++++++- .../test/model-account-lifecycle.test.mjs | 39 ++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index 236d4008..0ee1bcc3 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -1,4 +1,5 @@ -import { readFile, rm } from "node:fs/promises"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; import { join, resolve, sep } from "node:path"; const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; @@ -27,6 +28,11 @@ export function createModelAccountLifecycle({ sleep = ms => new Promise(resolveSleep => setTimeout(resolveSleep, ms)), removeCache = rm, readCache = readFile, + writeCache = writeFile, + makeDirectory = mkdir, + replaceFile = rename, + getEncryptedCache, + decryptCache, } = {}) { if (!authHelper || typeof authHelper.invoke !== "function" || typeof getUnlockKey !== "function" || typeof getAccount !== "function" || typeof encryptCache !== "function" || typeof persistEncrypted !== "function" @@ -34,6 +40,11 @@ export function createModelAccountLifecycle({ || retryDelaysMs.some(ms => !Number.isSafeInteger(ms) || ms < 0 || ms > 30_000)) { throw new TypeError("model account lifecycle configuration is invalid"); } + if ((getEncryptedCache === undefined) !== (decryptCache === undefined) + || (getEncryptedCache !== undefined && typeof getEncryptedCache !== "function") + || (decryptCache !== undefined && typeof decryptCache !== "function")) { + throw new TypeError("model account cache restoration configuration is invalid"); + } const generations = new Map(); const queues = new Map(); @@ -132,11 +143,57 @@ export function createModelAccountLifecycle({ }); } + async function restore(id) { + if (!getEncryptedCache || !decryptCache) throw new Error("model account cache restoration is unavailable"); + const account = await getAccount(id); + if (!account || account.id !== id || account.kind !== "model" || !Number.isSafeInteger(account.revision) + || typeof account.ownerId !== "string" || !account.ownerId || typeof account.keyGeneration !== "string" || !account.keyGeneration) { + await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); + return { state: "reauthentication-required" }; + } + const encrypted = await getEncryptedCache(id); + if (!encrypted || encrypted.accountId !== id || encrypted.ownerId !== account.ownerId + || encrypted.keyGeneration !== account.keyGeneration || !Number.isSafeInteger(encrypted.revision) + || typeof encrypted.encryptedPayload !== "string") { + await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); + return { state: "reauthentication-required" }; + } + const unlockKey = await getUnlockKey(); + if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); + let plain; + try { + plain = await decryptCache(encrypted.encryptedPayload, unlockKey, { + ownerId: account.ownerId, + recordId: id, + recordType: "model", + destination: "factorize", + revision: encrypted.revision, + keyGeneration: account.keyGeneration, + }); + if (!(plain instanceof Uint8Array) || plain.byteLength === 0 || plain.byteLength > 65_536) { + throw new Error("restored Codex token cache is invalid"); + } + const destination = cachePath(accountHomeRoot, id); + await makeDirectory(resolve(accountHomeRoot, id), { recursive: true, mode: 0o700 }); + const temporary = `${destination}.${randomUUID()}.tmp`; + try { + await writeCache(temporary, plain, { mode: 0o600, flag: "wx" }); + await replaceFile(temporary, destination); + } finally { + await rm(temporary, { force: true }); + } + } finally { + plain?.fill(0); + } + return { state: "restored", revision: encrypted.revision }; + } + return Object.freeze({ deviceLogin, cancel, refresh, deleteAccount, + restore, isUnsaved: id => unsaved.has(id), /** Invoke after encrypted snapshots are reconciled at unlock. */ reconcile: async id => serialize(id, async () => { @@ -145,6 +202,10 @@ export function createModelAccountLifecycle({ unsaved.delete(id); return { state: "deleted" }; } + if (getEncryptedCache) { + const restored = await restore(id); + if (restored.state !== "restored") return restored; + } const result = await authHelper.invoke({ method: "account-status", accountId: id }); return result.state === "authenticated" ? persist(id, currentGeneration(id)) : { state: "reauthentication-required" }; }), diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 449fbdb4..37ab14e4 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -111,3 +111,42 @@ test("deletion invalidates a retry and removes the account cache", async t => { await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); assert.ok(f.methods.includes("logout")); }); + +test("unlock reconciliation restores the encrypted Codex cache before checking account status", async t => { + const persisted = new TextEncoder().encode(JSON.stringify({ tokens: { access_token: "restored-access", refresh_token: "restored-refresh" } })); + const encryptedPayload = `cache.${Buffer.from(persisted).toString("base64")}`; + const f = await fixture(t, { options: { + getEncryptedCache: async id => ({ + accountId: id, ownerId: "owner", keyGeneration: "generation", revision: 8, encryptedPayload, + }), + decryptCache: async (payload, key, binding) => { + assert.equal(payload, encryptedPayload); + assert.equal(key, "synthetic-unlock-key"); + assert.deepEqual(binding, { + ownerId: "owner", recordId: accountId, recordType: "model", destination: "factorize", + revision: 8, keyGeneration: "generation", + }); + return new Uint8Array(persisted); + }, + } }); + + assert.deepEqual(await f.lifecycle.reconcile(accountId), { state: "saved" }); + assert.deepEqual(JSON.parse(await readFile(join(f.home, "auth.json"), "utf8")), { + tokens: { access_token: "restored-access", refresh_token: "restored-refresh" }, + }); + assert.ok(f.methods.indexOf("account-status") > -1); + assert.equal(f.events.find(event => event.type === "persist").request.expectedRevision, 4); +}); + +test("unlock reconciliation removes stale cache when the account key generation changed", async t => { + const f = await fixture(t, { options: { + getEncryptedCache: async id => ({ + accountId: id, ownerId: "owner", keyGeneration: "old-generation", revision: 8, encryptedPayload: "cache.payload", + }), + decryptCache: async () => { throw new Error("must not decrypt stale cache"); }, + } }); + + assert.deepEqual(await f.lifecycle.reconcile(accountId), { state: "reauthentication-required" }); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.equal(f.methods.includes("account-status"), false); +}); From d27d95a4462f67f120e7721f9a88cd1737de350a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 01:35:23 +0000 Subject: [PATCH 034/102] GEN-2824 persist encrypted model caches by revision --- .../encrypted-integration-repository.ts | 37 ++++++++++++++++++ packages/api/src/proxy-control.ts | 38 +++++++++++++++++-- .../credential-proxy/src/control-client.mjs | 35 +++++++++++++++++ .../src/model-account-lifecycle.mjs | 2 +- 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/packages/api/src/postgres/encrypted-integration-repository.ts b/packages/api/src/postgres/encrypted-integration-repository.ts index 095f6d03..dcbb3ddc 100644 --- a/packages/api/src/postgres/encrypted-integration-repository.ts +++ b/packages/api/src/postgres/encrypted-integration-repository.ts @@ -126,6 +126,43 @@ export class EncryptedIntegrationRepository { return Promise.all(rows.map(async row => ({ id: row.id, kind: row.kind, name: row.name, origin: row.origin, description: row.description, revision: safeRevision(row.revision), encryptedPayload: await decrypt(row.encrypted_payload, this.serverEncryptionKey) }))); } + /** + * Replace only a model account's user-encrypted Codex cache. The account + * revision is independent of the collection revision, so unrelated bearer + * edits do not invalidate the cache's authenticated-encryption binding. + */ + async saveModelCache(input: { id: string; expectedRevision: number; encryptedPayload: string; idempotencyKey: string }): Promise<{ revision: number; collectionRevision: number; duplicate: boolean }> { + const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (!uuid.test(input.id) || !uuid.test(input.idempotencyKey) || !Number.isSafeInteger(input.expectedRevision) || input.expectedRevision < 0 + || !isUserCiphertext(input.encryptedPayload)) throw new IntegrationValidationError("model cache binding is invalid"); + const encryptedPayload = await encrypt(input.encryptedPayload, this.serverEncryptionKey); + return this.database.transaction(async client => { + await client.query("INSERT INTO app.integration_collections(tenant_id) VALUES ($1) ON CONFLICT DO NOTHING", [this.tenantId]); + const collection = (await client.query<{ revision: string | number }>("SELECT revision FROM app.integration_collections WHERE tenant_id=$1 FOR UPDATE", [this.tenantId])).rows[0]!; + const currentCollectionRevision = safeRevision(collection.revision); + const current = (await client.query<{ kind: EncryptedIntegrationKind; name: string; origin: string | null; description: string | null; revision: string | number; encrypted_payload: string | null; deleted_at: Date | null }>( + "SELECT kind,name,origin,description,revision,encrypted_payload,deleted_at FROM app.integration_records WHERE tenant_id=$1 AND id=$2 FOR UPDATE", [this.tenantId, input.id])).rows[0]; + if (!current || current.kind !== "model" || current.deleted_at || current.revision === null) throw new IntegrationValidationError("model account is unavailable"); + const currentRevision = safeRevision(current.revision); + const previousChange = (await client.query<{ revision: string | number; record_id: string; operation: string; encrypted_payload: string | null }>( + "SELECT revision,record_id,operation,encrypted_payload FROM app.integration_changes WHERE tenant_id=$1 AND idempotency_key=$2", [this.tenantId,input.idempotencyKey])).rows[0]; + if (previousChange) { + const same = previousChange.record_id === input.id && previousChange.operation === "upsert" && previousChange.encrypted_payload !== null + && await decrypt(previousChange.encrypted_payload, this.serverEncryptionKey) === input.encryptedPayload; + if (!same) throw new IntegrationIdempotencyConflictError(); + return { revision: Math.max(0, input.expectedRevision + 1), collectionRevision: safeRevision(previousChange.revision), duplicate: true }; + } + if (currentRevision !== input.expectedRevision) throw new StaleIntegrationRevisionError(currentRevision); + const collectionRevision = currentCollectionRevision + 1; + const revision = currentRevision + 1; + await client.query("UPDATE app.integration_collections SET revision=$2,updated_at=now() WHERE tenant_id=$1", [this.tenantId, collectionRevision]); + await client.query("UPDATE app.integration_records SET revision=$3,encrypted_payload=$4,updated_at=now() WHERE tenant_id=$1 AND id=$2", [this.tenantId,input.id,revision,encryptedPayload]); + await client.query(`INSERT INTO app.integration_changes(tenant_id,revision,id,idempotency_key,operation,record_id,kind,name,origin,description,encrypted_payload) + VALUES ($1,$2,$3,$4,'upsert',$5,'model',$6,NULL,$7,$8)`, [this.tenantId,collectionRevision,crypto.randomUUID(),input.idempotencyKey,input.id,current.name,current.description,encryptedPayload]); + return { revision, collectionRevision, duplicate: false }; + }); + } + async changesSince(revision: number, limit = 100): Promise { if (!Number.isSafeInteger(revision) || revision < 0) throw new IntegrationValidationError("revision must be a non-negative safe integer"); if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500) throw new IntegrationValidationError("limit must be between 1 and 500"); diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index 18ef39f3..a1653350 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -9,14 +9,15 @@ const json = (body: unknown, status = 200) => Response.json(body, { status, head /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot", "/internal/credential-proxy/revocations"].includes(url.pathname)) return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot", "/internal/credential-proxy/model-account-cache", "/internal/credential-proxy/revocations"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; const token = authorization.startsWith("Bearer ") ? authorization.slice(7) : ""; if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return json({ error: "unauthorized" }, 401); const declaredLength = Number(request.headers.get("content-length") ?? "0"); - if (declaredLength > 1024) return json({ error: "invalid_request" }, 413); + const maxBodyBytes = url.pathname === "/internal/credential-proxy/model-account-cache" ? 70_000 : 1024; + if (declaredLength > maxBodyBytes) return json({ error: "invalid_request" }, 413); let body: unknown; try { const reader = request.body?.getReader(); @@ -24,7 +25,7 @@ export async function proxyControlFetch(request: Request, env: Parameters 1024) { await reader.cancel(); return json({ error: "invalid_request" }, 413); } + if (length > maxBodyBytes) { await reader.cancel(); return json({ error: "invalid_request" }, 413); } chunks.push(value); } const bytes = new Uint8Array(length); let offset = 0; @@ -49,6 +50,37 @@ export async function proxyControlFetch(request: Request, env: Parameters row.run_id) }); } + if (url.pathname === "/internal/credential-proxy/model-account-cache") { + if (!body || typeof body !== "object" || Array.isArray(body)) return json({ error: "invalid_request" }, 400); + const input = body as Record; + const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (input.operation === "read" && Object.keys(input).sort().join(",") === "accountId,operation" + && typeof input.accountId === "string" && uuid.test(input.accountId)) { + const account = (await new EncryptedIntegrationRepository(databaseFor(env), proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("model")) + .find(record => record.id === input.accountId); + if (!account) return json({ error: "model_account_unavailable" }, 404); + const keyState = await databaseFor(env).pool.query<{ generation_id: string }>("SELECT generation_id FROM app.unlock_key_verifications WHERE tenant_id=$1", [proxy.tenantId]); + if (!keyState.rows[0]) return json({ error: "model_account_unavailable" }, 404); + return json({ ownerId: proxy.tenantId, account: { ...account, keyGeneration: keyState.rows[0].generation_id } }); + } + if (input.operation !== "save" || Object.keys(input).sort().join(",") !== "accountId,encryptedPayload,expectedRevision,idempotencyKey,operation" + || typeof input.accountId !== "string" || !uuid.test(input.accountId) + || typeof input.idempotencyKey !== "string" || !uuid.test(input.idempotencyKey) + || !Number.isSafeInteger(input.expectedRevision) || (input.expectedRevision as number) < 0 + || typeof input.encryptedPayload !== "string") return json({ error: "invalid_request" }, 400); + try { + const saved = await new EncryptedIntegrationRepository(databaseFor(env), proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).saveModelCache({ + id: input.accountId, expectedRevision: input.expectedRevision as number, + idempotencyKey: input.idempotencyKey, encryptedPayload: input.encryptedPayload, + }); + return json({ ...saved, accountId: input.accountId }); + } catch (error) { + if (error instanceof Error && error.name === "StaleIntegrationRevisionError") return json({ error: "stale_revision" }, 409); + if (error instanceof Error && error.name === "IntegrationIdempotencyConflictError") return json({ error: "idempotency_conflict" }, 409); + if (error instanceof Error && error.name === "IntegrationValidationError") return json({ error: "model_account_unavailable" }, 404); + throw error; + } + } if (url.pathname === "/internal/credential-proxy/integration-snapshot") { if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "grantRevision"].includes(key)) || typeof (body as any).runId !== "string" || !Number.isSafeInteger((body as any).grantRevision) || (body as any).grantRevision < 0) return json({ error: "invalid_request" }, 400); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index f36c3fa4..e8d89798 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -76,6 +76,41 @@ export async function readProxyIntegrationSnapshot({ controlOrigin, token, bindi return result; } +/** Concrete proxy-side adapters for revision-bound encrypted model caches. */ +export function createModelAccountPersistenceAdapter({ controlOrigin, controlCredential, ownerId, fetchImpl = fetch } = {}) { + if (typeof ownerId !== "string" || !ownerId || !/^fzc_[A-Za-z0-9_-]{43}$/.test(controlCredential ?? "")) { + throw new TypeError("model account persistence requires the registered proxy identity"); + } + const read = async accountId => { + if (!uuidPattern.test(accountId ?? "")) throw new TypeError("invalid model account id"); + const result = await proxyControlRequest({ controlOrigin, token: controlCredential, path: "/internal/credential-proxy/model-account-cache", body: { operation: "read", accountId }, fetchImpl }); + const account = result?.account; + if (result?.ownerId !== ownerId || !account || account.id !== accountId || account.kind !== "model" + || account.origin !== null || typeof account.encryptedPayload !== "string" + || !Number.isSafeInteger(account.revision) || !uuidPattern.test(account.keyGeneration ?? "")) { + throw new Error("invalid model account persistence response"); + } + return { ...account, ownerId }; + }; + return Object.freeze({ + getAccount: read, + async getEncryptedCache(accountId) { + const account = await read(accountId); + return { accountId, ownerId, keyGeneration: account.keyGeneration, revision: account.revision, encryptedPayload: account.encryptedPayload }; + }, + async persistEncrypted({ id, expectedRevision, encryptedPayload, idempotencyKey }) { + if (!uuidPattern.test(id ?? "") || !uuidPattern.test(idempotencyKey ?? "") || !Number.isSafeInteger(expectedRevision) + || typeof encryptedPayload !== "string" || encryptedPayload.length > 65_536) throw new TypeError("invalid encrypted model account update"); + const result = await proxyControlRequest({ controlOrigin, token: controlCredential, path: "/internal/credential-proxy/model-account-cache", + body: { operation: "save", accountId: id, expectedRevision, encryptedPayload, idempotencyKey }, fetchImpl }); + if (result?.accountId !== id || result.revision !== expectedRevision + 1 || typeof result.duplicate !== "boolean") { + throw new Error("invalid model account persistence response"); + } + return result; + }, + }); +} + /** Read durable revocations for the registered proxy boot. Replaying the full * set is intentional: it closes runs revoked while this proxy was offline and * runtime.revokeRun is idempotent. */ diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index 0ee1bcc3..8e13d05f 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -82,7 +82,7 @@ export function createModelAccountLifecycle({ plain.fill(0); } if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); - const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted }; + const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted, idempotencyKey: randomUUID() }; for (let attempt = 0; ; attempt++) { if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); try { From df221e5b3af486e51772b0b02dba7fe9a4f087f1 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:00:55 +0000 Subject: [PATCH 035/102] GEN-2824 compose model cache lifecycle into proxy startup --- packages/api/src/proxy-control.ts | 9 ++++++- packages/api/test/proxy-control.test.ts | 2 +- .../credential-proxy/src/control-client.mjs | 15 +++++++++-- packages/credential-proxy/src/main.mjs | 25 +++++++++++++++++ .../test/control-client.test.mjs | 27 ++++++++++++++++--- 5 files changed, 71 insertions(+), 7 deletions(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index a1653350..e947d35b 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -37,7 +37,7 @@ export async function proxyControlFetch(request: Request, env: Parameters key !== "bootEpoch") || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); const registration = await repository.registerBoot(token, (body as any).bootEpoch); if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); - return json({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch, controlCredential: registration.controlCredential }); + return json({ registered: true, tenantId: registration.tenantId, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch, controlCredential: registration.controlCredential }); } const proxy = await repository.authenticateProxy(token); @@ -54,6 +54,13 @@ export async function proxyControlFetch(request: Request, env: Parameters; const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (input.operation === "list" && Object.keys(input).length === 1) { + const accounts = await new EncryptedIntegrationRepository(databaseFor(env), proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("model"); + const keyState = await databaseFor(env).pool.query<{ generation_id: string }>("SELECT generation_id FROM app.unlock_key_verifications WHERE tenant_id=$1", [proxy.tenantId]); + if (!keyState.rows[0]) return json({ error: "model_account_unavailable" }, 404); + return json({ ownerId: proxy.tenantId, keyGeneration: keyState.rows[0].generation_id, + accounts: accounts.map(({ id, revision }) => ({ id, revision, kind: "model", origin: null })) }); + } if (input.operation === "read" && Object.keys(input).sort().join(",") === "accountId,operation" && typeof input.accountId === "string" && uuid.test(input.accountId)) { const account = (await new EncryptedIntegrationRepository(databaseFor(env), proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("model")) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 42a02a58..ae33a978 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -30,7 +30,7 @@ describe("dedicated proxy boot registration endpoint", () => { const env = envFor(); const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: expect.stringMatching(/^fzc_[A-Za-z0-9_-]{43}$/) }); + expect(await response.json()).toMatchObject({ registered: true, tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: expect.stringMatching(/^fzc_[A-Za-z0-9_-]{43}$/) }); expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(4); expect(env.transactionQuery.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(env.transactionQuery.mock.calls[2][0]).toContain("consumed_at=now()"); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index e8d89798..6ea36bab 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -22,8 +22,9 @@ export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetch }); if (!response.ok) throw new Error("proxy boot registration rejected"); const result = await response.json(); - if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(result.controlCredential ?? "")) throw new Error("invalid proxy boot registration response"); - return { proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch, controlCredential: result.controlCredential }; + if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string" + || !uuidPattern.test(result.tenantId ?? "") || !/^fzc_[A-Za-z0-9_-]{43}$/.test(result.controlCredential ?? "")) throw new Error("invalid proxy boot registration response"); + return { tenantId: result.tenantId, proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch, controlCredential: result.controlCredential }; } finally { credential = ""; } @@ -93,6 +94,16 @@ export function createModelAccountPersistenceAdapter({ controlOrigin, controlCre return { ...account, ownerId }; }; return Object.freeze({ + async listAccounts() { + const result = await proxyControlRequest({ controlOrigin, token: controlCredential, path: "/internal/credential-proxy/model-account-cache", body: { operation: "list" }, fetchImpl }); + if (result?.ownerId !== ownerId || !Array.isArray(result.accounts) + || !uuidPattern.test(result.keyGeneration ?? "") + || result.accounts.some(account => !account || !uuidPattern.test(account.id ?? "") || account.kind !== "model" + || account.origin !== null || !Number.isSafeInteger(account.revision))) { + throw new Error("invalid model account list response"); + } + return result.accounts.map(account => ({ ...account, ownerId, keyGeneration: result.keyGeneration })); + }, getAccount: read, async getEncryptedCache(accountId) { const account = await read(accountId); diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 60ab7333..4b2765bb 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -6,6 +6,10 @@ import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; import { createProxyIntegrationGrantResolver } from "./decrypted-grants.mjs"; import { startAuthenticatedProxyTunnel } from "./tunnel-transport.mjs"; import { createProxyTlsAuthority } from "./tls-identity.mjs"; +import { createCodexAuthHelper } from "./codex-auth-helper.mjs"; +import { createModelAccountLifecycle } from "./model-account-lifecycle.mjs"; +import { createModelAccountPersistenceAdapter } from "./control-client.mjs"; +import { decryptModelAccountCache, encryptModelAccountCache } from "./model-account-crypto.mjs"; const epochIndex = process.argv.indexOf("--boot-epoch"); const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; @@ -20,9 +24,26 @@ let controlCredential; let controlOrigin; let tunnel; let revocationMonitor; +let modelAccounts; +let modelAccountLifecycle; async function activateRegistration(registration, origin) { if (tunnel) throw new Error("proxy boot is already registered"); + const persistence = createModelAccountPersistenceAdapter({ + controlOrigin: origin, + controlCredential: registration.controlCredential, + ownerId: registration.tenantId, + }); + modelAccounts = persistence; + modelAccountLifecycle = createModelAccountLifecycle({ + authHelper: createCodexAuthHelper(), + getUnlockKey: () => runtime.getUnlockKey(), + getAccount: persistence.getAccount, + getEncryptedCache: persistence.getEncryptedCache, + persistEncrypted: persistence.persistEncrypted, + encryptCache: encryptModelAccountCache, + decryptCache: decryptModelAccountCache, + }); const resolveGrants = createProxyIntegrationGrantResolver({ runtime, controlOrigin: origin, controlCredential: registration.controlCredential }); const started = await startAuthenticatedProxyTunnel({ tls: tlsAuthority.tls, @@ -96,6 +117,10 @@ lines.on("line", async (line) => { if (command?.type === "status") respondStatus(); else if (command?.type === "unlock") { runtime.unlock(command.key); + if (modelAccountLifecycle && modelAccounts) { + const accounts = await modelAccounts.listAccounts(); + for (const account of accounts) await modelAccountLifecycle.reconcile(account.id); + } respondStatus(); } else if (command?.type === "lock") { await runtime.lock(); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index b3ab4cbd..e25342cf 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, readProxyRunRevocations, registerProxyBoot, startProxyRunRevocationMonitor } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createModelAccountPersistenceAdapter, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, readProxyRunRevocations, registerProxyBoot, startProxyRunRevocationMonitor } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -11,9 +11,9 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an let request; const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async (url, init) => { request = { url: String(url), init }; - return Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); + return Response.json({ registered: true, tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); } }); - assert.deepEqual(result, { proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); + assert.deepEqual(result, { tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); assert.equal(request.url, "https://factorize.example/internal/credential-proxy/register-boot"); assert.equal(request.init.headers.Authorization, `Bearer ${token}`); assert.equal(request.init.redirect, "error"); @@ -35,6 +35,27 @@ test("proxy snapshot reader requests only the bound run revision and validates e await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ...snapshot, modelAccount: { ...snapshot.modelAccount, kind: "bearer" } }) }), /invalid proxy integration snapshot response/); }); +test("model account persistence adapter lists owner-bound account revisions over proxy authentication", async () => { + const ownerId = "11111111-1111-4111-8111-111111111111"; + let request; + const adapter = createModelAccountPersistenceAdapter({ + controlOrigin: "https://factorize.example", controlCredential: token, ownerId, + fetchImpl: async (url, init) => { + request = { url: String(url), init, body: JSON.parse(init.body) }; + return Response.json({ ownerId, keyGeneration: "22222222-2222-4222-8222-222222222222", + accounts: [{ id: "33333333-3333-4333-8333-333333333333", kind: "model", origin: null, revision: 4 }] }); + }, + }); + assert.deepEqual(await adapter.listAccounts(), [{ id: "33333333-3333-4333-8333-333333333333", kind: "model", origin: null, revision: 4, ownerId, keyGeneration: "22222222-2222-4222-8222-222222222222" }]); + assert.equal(request.url, "https://factorize.example/internal/credential-proxy/model-account-cache"); + assert.deepEqual(request.body, { operation: "list" }); + assert.equal(request.init.headers.Authorization, `Bearer ${token}`); + await assert.rejects(adapter.listAccounts && createModelAccountPersistenceAdapter({ + controlOrigin: "https://factorize.example", controlCredential: token, ownerId, + fetchImpl: async () => Response.json({ ownerId: "22222222-2222-4222-8222-222222222222", keyGeneration: "22222222-2222-4222-8222-222222222222", accounts: [] }), + }).listAccounts(), /invalid model account list response/); +}); + test("proxy revocation monitor applies durable run revocations and tolerates an outage", async () => { const requests = []; const revoked = []; From a20d434024f88b969cb8b5bc792819f5363a1608 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:18:46 +0000 Subject: [PATCH 036/102] GEN-2824 add private browser model authorization control --- .../credential-proxy/src/browser-control.mjs | 150 ++++++++++++++++++ .../test/browser-control.test.mjs | 76 +++++++++ 2 files changed, 226 insertions(+) create mode 100644 packages/credential-proxy/src/browser-control.mjs create mode 100644 packages/credential-proxy/test/browser-control.test.mjs diff --git a/packages/credential-proxy/src/browser-control.mjs b/packages/credential-proxy/src/browser-control.mjs new file mode 100644 index 00000000..5911ba14 --- /dev/null +++ b/packages/credential-proxy/src/browser-control.mjs @@ -0,0 +1,150 @@ +import { createServer } from "node:http"; +import { createPublicKey, verify } from "node:crypto"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const MAX_BODY_BYTES = 4096; + +function decodePart(value) { + if (!/^[A-Za-z0-9_-]+$/.test(value)) throw new Error("invalid token"); + return Buffer.from(value, "base64url"); +} + +function parseCapability(token, publicKey) { + if (typeof token !== "string" || token.length > 4096) throw new Error("invalid capability"); + const parts = token.split("."); + if (parts.length !== 3) throw new Error("invalid capability"); + const [headerPart, claimsPart, signaturePart] = parts; + const header = JSON.parse(decodePart(headerPart).toString("utf8")); + const claims = JSON.parse(decodePart(claimsPart).toString("utf8")); + if (header?.alg !== "EdDSA" || header?.typ !== "JWT" + || !verify(null, Buffer.from(`${headerPart}.${claimsPart}`), publicKey, decodePart(signaturePart))) { + throw new Error("invalid capability"); + } + return claims; +} + +function json(response, status, body, origin) { + const headers = { "Cache-Control": "no-store", "Content-Type": "application/json; charset=utf-8" }; + if (origin) { + headers["Access-Control-Allow-Origin"] = origin; + headers.Vary = "Origin"; + } + response.writeHead(status, headers).end(JSON.stringify(body)); +} + +async function readJson(request) { + const chunks = []; + let size = 0; + for await (const chunk of request) { + size += chunk.byteLength; + if (size > MAX_BODY_BYTES) throw Object.assign(new Error("request too large"), { status: 413 }); + chunks.push(chunk); + } + try { return JSON.parse(Buffer.concat(chunks).toString("utf8")); } + catch { throw Object.assign(new Error("invalid request"), { status: 400 }); } +} + +/** Private Fly service endpoint for owner-browser model-account operations. + * The Fly edge terminates public HTTPS; this listener must only be reachable + * through that service and never through the agent SSH integration channel. */ +export async function startBrowserControl({ + runtime, + lifecycle, + publicKey, + proxyId, + ownerId, + generation, + appOrigin, + host = "0.0.0.0", + port = 8080, + now = () => Math.floor(Date.now() / 1000), +} = {}) { + if (!runtime || !lifecycle || !publicKey || typeof proxyId !== "string" || !proxyId + || !UUID.test(ownerId ?? "") || !Number.isSafeInteger(generation) || generation < 1 + || !Number.isInteger(port) || port < 0 || port > 65535) { + throw new TypeError("browser control configuration is invalid"); + } + const expectedOrigin = new URL(appOrigin); + if (expectedOrigin.protocol !== "https:" || expectedOrigin.origin !== appOrigin) throw new TypeError("browser control origin must be canonical HTTPS"); + const verifyKey = publicKey?.type === "public" ? publicKey : createPublicKey(publicKey); + const consumed = new Map(); + const authorizations = new Map(); + + const server = createServer(async (request, response) => { + const origin = request.headers.origin; + if (origin !== appOrigin) return json(response, 403, { error: "origin_rejected" }); + if (request.method === "OPTIONS") { + if (request.headers["access-control-request-method"] !== "POST" + || request.headers["access-control-request-headers"]?.split(",").map(value => value.trim().toLowerCase()).sort().join(",") !== "authorization,content-type") { + return json(response, 403, { error: "preflight_rejected" }); + } + response.writeHead(204, { + "Access-Control-Allow-Origin": appOrigin, + "Access-Control-Allow-Methods": "POST", + "Access-Control-Allow-Headers": "Authorization, Content-Type", + "Access-Control-Max-Age": "60", + "Cache-Control": "no-store", + Vary: "Origin", + }).end(); + return; + } + if (request.method !== "POST") return json(response, 405, { error: "method_not_allowed" }, appOrigin); + const match = /^\/model-accounts\/([0-9a-f-]{36})\/(authorize|status|cancel)$/.exec(new URL(request.url ?? "/", "http://proxy.internal").pathname); + if (!match || !UUID.test(match[1])) return json(response, 404, { error: "not_found" }, appOrigin); + const [, accountId, operation] = match; + if (request.headers["content-type"]?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") { + return json(response, 415, { error: "unsupported_media_type" }, appOrigin); + } + const authorization = request.headers.authorization ?? ""; + if (!authorization.startsWith("Bearer ")) return json(response, 401, { error: "invalid_capability" }, appOrigin); + let claims; + try { claims = parseCapability(authorization.slice(7), verifyKey); } + catch { return json(response, 401, { error: "invalid_capability" }, appOrigin); } + const expectedAction = `model-account:${operation}`; + if (claims.iss !== "factorize" || claims.aud !== proxyId || claims.sub !== ownerId + || claims.action !== expectedAction || claims.accountId !== accountId || claims.origin !== appOrigin + || claims.generation !== generation || !Number.isSafeInteger(claims.exp) || claims.exp <= now() + || claims.exp > now() + 60 || typeof claims.jti !== "string" || !/^[A-Za-z0-9_-]{16,128}$/.test(claims.jti)) { + return json(response, 401, { error: "invalid_capability" }, appOrigin); + } + for (const [id, expiry] of consumed) if (expiry <= now()) consumed.delete(id); + if (consumed.has(claims.jti)) return json(response, 401, { error: "capability_replayed" }, appOrigin); + consumed.set(claims.jti, claims.exp); + + try { + const body = await readJson(request); + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).length) { + return json(response, 400, { error: "invalid_request" }, appOrigin); + } + if (operation === "authorize") { + if (runtime.state !== "unlocked") return json(response, 423, { error: "proxy_locked" }, appOrigin); + const login = await lifecycle.deviceLogin(accountId); + const state = { state: "pending" }; + authorizations.set(accountId, state); + Promise.resolve(login.completion).then(result => { + if (authorizations.get(accountId) === state) authorizations.set(accountId, { ...result }); + }, () => { + if (authorizations.get(accountId) === state) authorizations.set(accountId, { state: "failed" }); + }); + return json(response, 202, state, appOrigin); + } + if (operation === "status") return json(response, 200, authorizations.get(accountId) ?? { state: "idle" }, appOrigin); + const result = await lifecycle.cancel(accountId); + authorizations.set(accountId, { state: "canceled" }); + return json(response, 200, { state: result?.state === "idle" ? "idle" : "canceled" }, appOrigin); + } catch (error) { + return json(response, Number.isInteger(error?.status) ? error.status : 400, + { error: "model_account_operation_failed" }, appOrigin); + } + }); + server.on("connect", (_request, socket) => socket.end("HTTP/1.1 405 Method Not Allowed\r\nConnection: close\r\n\r\n")); + server.on("upgrade", (_request, socket) => socket.destroy()); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { server.removeListener("error", reject); resolve(); }); + }); + return Object.freeze({ + address: server.address(), + close: () => new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())), + }); +} diff --git a/packages/credential-proxy/test/browser-control.test.mjs b/packages/credential-proxy/test/browser-control.test.mjs new file mode 100644 index 00000000..0c7af413 --- /dev/null +++ b/packages/credential-proxy/test/browser-control.test.mjs @@ -0,0 +1,76 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { generateKeyPairSync, sign } from "node:crypto"; +import { startBrowserControl } from "../src/browser-control.mjs"; + +const ownerId = "11111111-1111-4111-8111-111111111111"; +const accountId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const proxyId = "fly:factorize-proxy:machine-1"; +const generation = 4; +const appOrigin = "https://factorize.example"; +const { publicKey, privateKey } = generateKeyPairSync("ed25519"); + +function capability({ action, exp = 1_900_000_030, aud = proxyId, sub = ownerId, origin = appOrigin, gen = generation, account = accountId, jti = `nonce-${crypto.randomUUID()}` }) { + const head = Buffer.from(JSON.stringify({ alg: "EdDSA", typ: "JWT" })).toString("base64url"); + const claims = Buffer.from(JSON.stringify({ iss: "factorize", aud, sub, action, origin, generation: gen, exp, accountId: account, jti })).toString("base64url"); + const input = `${head}.${claims}`; + return `${input}.${sign(null, Buffer.from(input), privateKey).toString("base64url")}`; +} + +test("private browser control requires exact origin and one-use owner/proxy/action capabilities", async t => { + let now = 1_900_000_000; + const calls = []; + const runtime = { state: "unlocked" }; + const lifecycle = { + async deviceLogin(id) { calls.push(["authorize", id]); return { state: "pending", completion: Promise.resolve({ state: "saved" }) }; }, + async cancel(id) { calls.push(["cancel", id]); return { state: "canceled" }; }, + }; + const server = await startBrowserControl({ runtime, lifecycle, publicKey, proxyId, ownerId, generation, appOrigin, port: 0, host: "127.0.0.1", now: () => now }); + t.after(() => server.close()); + const origin = `http://127.0.0.1:${server.address.port}`; + const post = (action, token, path = `/model-accounts/${accountId}/${action.split(":")[1]}`, requestOrigin = appOrigin) => fetch(origin + path, { + method: "POST", headers: { Origin: requestOrigin, "Content-Type": "application/json", Authorization: `Bearer ${token}` }, body: "{}", + }); + + const preflight = await fetch(origin + `/model-accounts/${accountId}/authorize`, { + method: "OPTIONS", headers: { Origin: appOrigin, "Access-Control-Request-Method": "POST", "Access-Control-Request-Headers": "content-type,authorization" }, + }); + assert.equal(preflight.status, 204); + assert.equal(preflight.headers.get("access-control-allow-origin"), appOrigin); + const foreign = await fetch(origin + `/model-accounts/${accountId}/authorize`, { method: "OPTIONS", headers: { Origin: "https://attacker.example", "Access-Control-Request-Method": "POST", "Access-Control-Request-Headers": "authorization,content-type" } }); + assert.equal(foreign.status, 403); + assert.equal(foreign.headers.get("access-control-allow-origin"), null); + + const authToken = capability({ action: "model-account:authorize" }); + const started = await post("model-account:authorize", authToken); + assert.equal(started.status, 202); + assert.deepEqual(await started.json(), { state: "pending" }); + assert.deepEqual(calls[0], ["authorize", accountId]); + const replay = await post("model-account:authorize", authToken); + assert.equal(replay.status, 401); + + const status = await post("model-account:status", capability({ action: "model-account:status" })); + assert.deepEqual(await status.json(), { state: "saved" }); + const wrongAudience = await post("model-account:cancel", capability({ action: "model-account:cancel", aud: "fly:other:machine" })); + assert.equal(wrongAudience.status, 401); + const expired = await post("model-account:cancel", capability({ action: "model-account:cancel", exp: now - 1 })); + assert.equal(expired.status, 401); + const wrongOrigin = await post("model-account:cancel", capability({ action: "model-account:cancel" }), undefined, "https://attacker.example"); + assert.equal(wrongOrigin.status, 403); + + const canceled = await post("model-account:cancel", capability({ action: "model-account:cancel" })); + assert.deepEqual(await canceled.json(), { state: "canceled" }); + assert.deepEqual(calls.at(-1), ["cancel", accountId]); + now += 61; +}); + +test("authorization is refused while locked and expiry remains bounded to one minute", async t => { + const runtime = { state: "locked" }; + const lifecycle = { async deviceLogin() { assert.fail("locked proxy must not start authorization"); }, async cancel() { return { state: "idle" }; } }; + const server = await startBrowserControl({ runtime, lifecycle, publicKey, proxyId, ownerId, generation, appOrigin, port: 0, host: "127.0.0.1", now: () => 1_900_000_000 }); + t.after(() => server.close()); + const response = await fetch(`http://127.0.0.1:${server.address.port}/model-accounts/${accountId}/authorize`, { + method: "POST", headers: { Origin: appOrigin, "Content-Type": "application/json", Authorization: `Bearer ${capability({ action: "model-account:authorize", exp: 1_900_000_061 })}` }, body: "{}", + }); + assert.equal(response.status, 401); +}); From c876176c3e23e975fe1b36d295ca95a7c31e3929 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:47:15 +0000 Subject: [PATCH 037/102] GEN-2824 issue paired model account browser capabilities --- packages/api-client/src/schema.d.ts | 66 ++++++++++++++++ .../0038_proxy_browser_control_pairings.sql | 8 ++ packages/api/src/api-contract.ts | 6 ++ packages/api/src/flow-service.ts | 18 +++++ .../src/fly-credential-proxy-provisioner.ts | 6 ++ .../src/postgres/proxy-control-repository.ts | 17 ++++- packages/api/src/postgres/schema.ts | 5 ++ packages/api/src/proxy-browser-capability.ts | 38 ++++++++++ packages/api/src/proxy-control.ts | 4 +- packages/api/src/proxy-provisioner.ts | 5 +- packages/api/src/types.ts | 3 + packages/api/test/api-contract.test.ts | 9 +++ .../fly-credential-proxy-provisioner.test.ts | 7 +- .../api/test/proxy-browser-capability.test.ts | 25 ++++++ .../api/test/proxy-control-repository.test.ts | 9 ++- packages/api/test/proxy-control.test.ts | 5 +- packages/api/test/proxy-provisioner.test.ts | 2 + .../src/bootstrap-control.mjs | 7 +- .../credential-proxy/src/browser-control.mjs | 2 +- .../credential-proxy/src/control-client.mjs | 6 +- packages/credential-proxy/src/main.mjs | 22 +++++- .../test/bootstrap-control.test.mjs | 14 ++-- .../test/control-client.test.mjs | 10 +-- packages/docs/openapi.yaml | 76 +++++++++++++++++++ 24 files changed, 336 insertions(+), 34 deletions(-) create mode 100644 packages/api/migrations/0038_proxy_browser_control_pairings.sql create mode 100644 packages/api/src/proxy-browser-capability.ts create mode 100644 packages/api/test/proxy-browser-capability.test.ts diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index a910f642..e7c92965 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -1,5 +1,29 @@ // Generated from packages/docs/openapi.yaml. Do not edit. export interface paths { + "/api/v1/model-accounts/{accountId}/browser-capability": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Issue a short-lived browser capability for a paired model account + * @description Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens. + * + * Requires the integrations:write scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + post: operations["post_api_v1_model_accounts_accountId_browser_capability"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/unlock-key/verification": { parameters: { query?: never; @@ -2683,6 +2707,48 @@ export interface components { } export type $defs = Record; export interface operations { + post_api_v1_model_accounts_accountId_browser_capability: { + parameters: { + query?: never; + header?: never; + path: { + accountId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + /** @enum {string} */ + action: "authorize" | "status" | "cancel"; + }; + }; + }; + responses: { + /** @description Paired proxy endpoint and one-use capability */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** Format: uri */ + endpoint: string; + capability: string; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 409: components["responses"]["Error409"]; + 415: components["responses"]["Error415"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; get_api_v1_unlock_key_verification: { parameters: { query?: never; diff --git a/packages/api/migrations/0038_proxy_browser_control_pairings.sql b/packages/api/migrations/0038_proxy_browser_control_pairings.sql new file mode 100644 index 00000000..45a8eb2e --- /dev/null +++ b/packages/api/migrations/0038_proxy_browser_control_pairings.sql @@ -0,0 +1,8 @@ +CREATE TABLE app.credential_proxy_browser_pairings ( + tenant_id uuid PRIMARY KEY REFERENCES app.tenants(id) ON DELETE CASCADE, + proxy_identity text NOT NULL, + boot_epoch uuid NOT NULL, + browser_origin text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (browser_origin ~ '^https://[^/]+$') +); diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index aa6e4b45..aae9e10a 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -135,6 +135,7 @@ const bearerIntegrationWriteResult = z.object({ revision: z.number().int().nonne const bearerIntegrationDeleteResult = z.object({ id: z.uuid(), deleted: z.literal(true), revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const unlockKeyVerification = z.object({ ownerId: z.string(), verification: z.object({ generationId: z.uuid(), encryptedVerification: z.string() }).strict().nullable() }).strict(); const unlockKeyVerificationInput = z.object({ generationId: z.uuid(), encryptedVerification: z.string().min(1).max(4096) }).strict(); +const modelAccountBrowserCapabilityInput = z.object({ action: z.enum(["authorize", "status", "cancel"]) }).strict(); function route( method: string, @@ -213,6 +214,11 @@ function authRoute(method: string, path: string, action: AuthAction, summary: st } export const API_ROUTES: RouteContract[] = [ + route("POST", "/api/v1/model-accounts/{accountId}/browser-capability", "integrations:write", { + summary: "Issue a short-lived browser capability for a paired model account", + description: "Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens.", + responses: { "200": { description: "Paired proxy endpoint and one-use capability", content: { "application/json": { schema: jsonSchema(z.object({ endpoint: z.url(), capability: z.string() }), "output") } } } }, + }, (service, { params, body }) => service.issueModelAccountBrowserCapability(params.accountId, body.action), { body: modelAccountBrowserCapabilityInput, maxBodyBytes: 1024, ownerSession: true, parameters: z.object({ accountId: z.uuid() }) }), route("GET", "/api/v1/unlock-key/verification", "integrations:read", { summary: "Get the current encrypted unlock-key verification record", description: "Requires an interactive owner browser session. Returns user-encrypted verification ciphertext and its key generation; the unlock key is never accepted or returned.", responses: { "200": { description: "Encrypted verification record or empty setup state", content: { "application/json": { schema: jsonSchema(unlockKeyVerification, "output") } } } } }, service => service.getUnlockKeyVerification(), { ownerSession: true }), route("POST", "/api/v1/unlock-key/verification", "integrations:write", { summary: "Create the encrypted unlock-key verification record", description: "Requires an interactive owner browser session. Stores only browser-encrypted verification ciphertext and permits one setup per account.", responses: { "200": { description: "Created verification record", content: { "application/json": { schema: jsonSchema(z.object({ generationId: z.uuid() }), "output") } } } } }, (service, { body }) => service.createUnlockKeyVerification(body), { body: unlockKeyVerificationInput, maxBodyBytes: 8192, ownerSession: true }), route("GET", "/api/v1/integrations/bearer", "flows:read", { diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index 0b1ede59..c2c7253a 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -17,6 +17,8 @@ import type { JobPageQuery } from "./job-contracts"; import { PostgresJobRepository, StaleJobEdit } from "./postgres/job-repository"; import { ConnectionRepository } from "./postgres/connection-repository"; import { EncryptedIntegrationRepository, IntegrationIdempotencyConflictError, IntegrationValidationError, StaleIntegrationRevisionError } from "./postgres/encrypted-integration-repository"; +import { ProxyControlRepository } from "./postgres/proxy-control-repository"; +import { issueModelAccountCapability } from "./proxy-browser-capability"; import { decrypt, encrypt } from "./crypto"; import { InvocationService, type Job, type Trigger } from "./job-domain"; import { RunQueryRepository } from "./postgres/run-query-repository"; @@ -571,6 +573,22 @@ export class ApiService { if (!rows.length) throw new ServiceError(409, "conflict", "An unlock key is already set up for this account."); return { generationId: input.generationId }; } + async issueModelAccountBrowserCapability(accountId: string, action: "authorize" | "status" | "cancel") { + await this.authorizeOwnerSession("integrations:write"); + if (!this.env.MODEL_ACCOUNT_CAPABILITY_PRIVATE_KEY) throw new ServiceError(503, "operation_failed", "Model-account browser authorization is unavailable."); + const [pairing, accounts] = await Promise.all([ + new ProxyControlRepository(databaseFor(this.env)).browserPairing(this.auth.tenantId), + this.encryptedIntegrations().records("model"), + ]); + if (!pairing) throw new ServiceError(503, "operation_failed", "The credential proxy is not paired for browser authorization."); + if (!accounts.some(account => account.id === accountId)) throw new ServiceError(404, "not_found", "Model account not found."); + const capability = await issueModelAccountCapability({ + privateKeyPem: this.env.MODEL_ACCOUNT_CAPABILITY_PRIVATE_KEY, + proxyIdentity: pairing.proxyIdentity, ownerId: this.auth.tenantId, accountId, action, + appOrigin: this.env.APP_ORIGIN, generation: pairing.bootEpoch, + }); + return { endpoint: `${pairing.browserOrigin}/model-accounts/${accountId}/${action}`, capability }; + } async removeTailIntegration(id: string) { await this.authorize("integrations:write"); if (!await this.connections().delete(`cloudflare-tail:${id}`)) throw new ServiceError(404, "not_found", "Integration not found"); return { id, deleted: true }; } async linearProjects() { await this.authorize("integrations:read"); return publicValue(z.array(namedOption), await this.providers().linearProjects()); } async linearOptions() { await this.authorize("integrations:read"); return publicValue(providerOptions, await this.providers().linearOptions()); } diff --git a/packages/api/src/fly-credential-proxy-provisioner.ts b/packages/api/src/fly-credential-proxy-provisioner.ts index 101b0ba5..c959537e 100644 --- a/packages/api/src/fly-credential-proxy-provisioner.ts +++ b/packages/api/src/fly-credential-proxy-provisioner.ts @@ -13,6 +13,7 @@ export type FlyCredentialProxyConfig = { image: string; region: string; size: string; + browserCapabilityPublicKey: string; memoryMb?: number; controlOrigin: string; /** Stable across retries of one provisioning operation. */ @@ -47,6 +48,7 @@ export function createFlyCredentialProxyGuest(options: { const result = await transport.run(["/usr/local/bin/factorize-proxy-startup"], `${JSON.stringify({ type: "register-boot", controlOrigin: input.controlOrigin, + browserOrigin: input.browserOrigin, bootEpoch: input.bootEpoch, credential: input.credential, })}\n`); @@ -86,6 +88,7 @@ function validate(input: FlyCredentialProxyConfig): void { if (!IMAGE_DIGEST.test(input.image)) throw new TypeError("Credential proxy image must be pinned by sha256 digest"); if (!/^[a-z0-9]{3}$/.test(input.region)) throw new TypeError("Invalid credential proxy region"); if (!/^[a-z0-9-]{1,64}$/.test(input.size)) throw new TypeError("Invalid credential proxy size"); + if (!input.browserCapabilityPublicKey.startsWith("-----BEGIN PUBLIC KEY-----")) throw new TypeError("Proxy browser capability public key is required"); if (input.memoryMb !== undefined && (!Number.isInteger(input.memoryMb) || input.memoryMb < 256 || input.memoryMb > 8192)) throw new TypeError("Credential proxy memory must be between 256 and 8192 MiB"); } @@ -113,7 +116,9 @@ export function createFlyCredentialProxyProvisioner(options: FlyCredentialProxyP env: { FACTORIZE_PROXY_TUNNEL_SERVER_NAME: `${input.app}.internal`, FACTORIZE_PROXY_TUNNEL_PORT: "9090", + FACTORIZE_PROXY_BROWSER_CAPABILITY_PUBLIC_KEY: input.browserCapabilityPublicKey, }, + services: [{ protocol: "tcp", internal_port: 8080, ports: [{ port: 443, handlers: ["tls", "http"] }], force_instance_key: true }], metadata: { factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId, @@ -136,6 +141,7 @@ export function createFlyCredentialProxyProvisioner(options: FlyCredentialProxyP proxyIdentity, bootEpoch, controlOrigin: origin.origin, + browserOrigin: `https://${input.app}.fly.dev`, }; try { await provisionCredentialProxy(bootstrap, { diff --git a/packages/api/src/postgres/proxy-control-repository.ts b/packages/api/src/postgres/proxy-control-repository.ts index 09fd28c9..047d0837 100644 --- a/packages/api/src/postgres/proxy-control-repository.ts +++ b/packages/api/src/postgres/proxy-control-repository.ts @@ -26,8 +26,10 @@ export class ProxyControlRepository { return row ? { tenantId: row.tenant_id, proxyIdentity: row.proxy_identity } : null; } - async registerBoot(token: string, bootEpoch: string): Promise { + async registerBoot(token: string, bootEpoch: string, browserOrigin: string): Promise { if (!uuid.test(bootEpoch)) throw new TypeError("bootEpoch must be a UUID"); + const parsedOrigin = new URL(browserOrigin); + if (parsedOrigin.protocol !== "https:" || parsedOrigin.origin !== browserOrigin || parsedOrigin.username || parsedOrigin.password) throw new TypeError("browser origin must be canonical HTTPS"); const tokenDigest = Buffer.from(await digest(token)); return this.database.transaction(async client => { const credential = await client.query<{ tenant_id: string; proxy_identity: string }>(`SELECT tenant_id,proxy_identity @@ -45,10 +47,23 @@ export class ProxyControlRepository { WHERE token_digest=$1 AND boot_epoch=$2 AND consumed_at IS NULL`, [tokenDigest, bootEpoch, Buffer.from(await digest(proxyToken))]); await client.query(`UPDATE app.credential_run_grants SET revoked_at=now() WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch<>$3 AND revoked_at IS NULL`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); + await client.query(`INSERT INTO app.credential_proxy_browser_pairings(tenant_id,proxy_identity,boot_epoch,browser_origin) + VALUES ($1,$2,$3,$4) ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity, + boot_epoch=excluded.boot_epoch,browser_origin=excluded.browser_origin,updated_at=now()`, + [identity.tenant_id, identity.proxy_identity, bootEpoch, browserOrigin]); return { tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch, controlCredential: proxyToken }; }); } + async browserPairing(tenantId: string): Promise<{ proxyIdentity: string; bootEpoch: string; browserOrigin: string } | null> { + const result = await this.database.pool.query<{ proxy_identity: string; boot_epoch: string; browser_origin: string }>( + `SELECT b.proxy_identity,b.boot_epoch,b.browser_origin FROM app.credential_proxy_browser_pairings b + JOIN app.credential_proxy_states p ON p.tenant_id=b.tenant_id AND p.proxy_identity=b.proxy_identity AND p.boot_epoch=b.boot_epoch + WHERE b.tenant_id=$1 AND p.disabled_at IS NULL AND p.revoked_at IS NULL`, [tenantId]); + const row = result.rows[0]; + return row ? { proxyIdentity: row.proxy_identity, bootEpoch: row.boot_epoch, browserOrigin: row.browser_origin } : null; + } + /** Authenticates the proxy's memory-held control credential against its current registered boot. */ async authenticateProxy(token: string): Promise | null> { if (typeof token !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return null; diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 2fbfed29..5092f713 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -53,6 +53,11 @@ export const integrationChanges = app.table("integration_changes", { export const credentialProxyStates = app.table("credential_proxy_states", { tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), disabledAt: timestamp("disabled_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), updatedAt: updatedAt(), }); +export const credentialProxyBrowserPairings = app.table("credential_proxy_browser_pairings", { + tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), + proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), + browserOrigin: text("browser_origin").notNull(), updatedAt: updatedAt(), +}); export const credentialProxyControlCredentials = app.table("credential_proxy_control_credentials", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), tokenDigest: bytea("token_digest").notNull().unique(), proxyTokenDigest: bytea("proxy_token_digest").unique(), revokedAt: timestamp("revoked_at", { withTimezone: true }), consumedAt: timestamp("consumed_at", { withTimezone: true }), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.proxyIdentity] })]); export const credentialRunGrants = app.table("credential_run_grants", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), vmIdentity: text("vm_identity").notNull(), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), grantRevision: bigint("grant_revision", { mode: "number" }).notNull(), revokedAt: timestamp("revoked_at", { withTimezone: true }), createdAt: createdAt(), diff --git a/packages/api/src/proxy-browser-capability.ts b/packages/api/src/proxy-browser-capability.ts new file mode 100644 index 00000000..fcb3ed9e --- /dev/null +++ b/packages/api/src/proxy-browser-capability.ts @@ -0,0 +1,38 @@ +const encoder = new TextEncoder(); +const base64url = (bytes: Uint8Array) => btoa(String.fromCharCode(...bytes)).replace(/=/g, "").replace(/\+/g, "-").replace(/\//g, "_"); + +function decodePem(pem: string): Uint8Array { + const body = pem.replace(/-----BEGIN PRIVATE KEY-----|-----END PRIVATE KEY-----|\s/g, ""); + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(body)) throw new TypeError("invalid browser capability signing key"); + const binary = atob(body); + return Uint8Array.from(binary, character => character.charCodeAt(0)); +} + +/** Issue a one-use capability accepted only by the paired proxy listener. */ +export async function issueModelAccountCapability(input: { + privateKeyPem: string; + proxyIdentity: string; + ownerId: string; + accountId: string; + action: "authorize" | "status" | "cancel"; + appOrigin: string; + generation: number | string; + now?: number; + jti?: string; +}): Promise { + const now = input.now ?? Math.floor(Date.now() / 1000); + const header = base64url(encoder.encode(JSON.stringify({ alg: "EdDSA", typ: "JWT" }))); + const claims = base64url(encoder.encode(JSON.stringify({ + iss: "factorize", aud: input.proxyIdentity, sub: input.ownerId, + action: `model-account:${input.action}`, accountId: input.accountId, + origin: input.appOrigin, generation: input.generation, exp: now + 60, + jti: input.jti ?? base64url(crypto.getRandomValues(new Uint8Array(24))), + }))); + const payload = `${header}.${claims}`; + const rawKey = decodePem(input.privateKeyPem); + const keyData = new ArrayBuffer(rawKey.byteLength); + new Uint8Array(keyData).set(rawKey); + const key = await crypto.subtle.importKey("pkcs8", keyData, { name: "Ed25519" }, false, ["sign"]); + const signature = await crypto.subtle.sign("Ed25519", key, encoder.encode(payload)); + return `${payload}.${base64url(new Uint8Array(signature))}`; +} diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index e947d35b..64db6ad6 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -34,8 +34,8 @@ export async function proxyControlFetch(request: Request, env: Parameters key !== "bootEpoch") || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); - const registration = await repository.registerBoot(token, (body as any).bootEpoch); + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).sort().join(",") !== "bootEpoch,browserOrigin" || typeof (body as any).bootEpoch !== "string" || typeof (body as any).browserOrigin !== "string") return json({ error: "invalid_request" }, 400); + const registration = await repository.registerBoot(token, (body as any).bootEpoch, (body as any).browserOrigin); if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); return json({ registered: true, tenantId: registration.tenantId, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch, controlCredential: registration.controlCredential }); } diff --git a/packages/api/src/proxy-provisioner.ts b/packages/api/src/proxy-provisioner.ts index f6d4957a..c32e7e5d 100644 --- a/packages/api/src/proxy-provisioner.ts +++ b/packages/api/src/proxy-provisioner.ts @@ -6,6 +6,7 @@ export type ProxyBootstrapProvisioning = { /** A new UUID for every process boot. */ bootEpoch: string; controlOrigin: string; + browserOrigin: string; }; /** @@ -25,9 +26,11 @@ export async function provisionCredentialProxy( if (origin.protocol !== "https:" || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) { throw new TypeError("controlOrigin must be an HTTPS origin"); } + const browserOrigin = new URL(input.browserOrigin); + if (browserOrigin.protocol !== "https:" || browserOrigin.origin !== input.browserOrigin || browserOrigin.username || browserOrigin.password) throw new TypeError("browserOrigin must be an HTTPS origin"); const credential = await dependencies.control.provisionCredential(input.tenantId, input.proxyIdentity, input.bootEpoch); try { - await dependencies.startProxy({ ...input, controlOrigin: origin.origin, credential }); + await dependencies.startProxy({ ...input, controlOrigin: origin.origin, browserOrigin: browserOrigin.origin, credential }); } catch (error) { // A failed/ambiguous start must not leave a valid bootstrap credential. await dependencies.control.revokeCredential(input.tenantId, input.proxyIdentity).catch(() => false); diff --git a/packages/api/src/types.ts b/packages/api/src/types.ts index 3aa3b959..250dfe35 100644 --- a/packages/api/src/types.ts +++ b/packages/api/src/types.ts @@ -15,6 +15,9 @@ export interface Env { CLICKUP_CLIENT_SECRET?: string; CREDENTIAL_ENCRYPTION_KEY: string; SESSION_SIGNING_SECRET: string; + /** PKCS#8 Ed25519 key used only for short-lived proxy browser capabilities. */ + MODEL_ACCOUNT_CAPABILITY_PRIVATE_KEY?: string; + MODEL_ACCOUNT_CAPABILITY_PUBLIC_KEY?: string; GITHUB_APP_ID?: string; GITHUB_APP_SLUG?: string; GITHUB_APP_PRIVATE_KEY?: string; diff --git a/packages/api/test/api-contract.test.ts b/packages/api/test/api-contract.test.ts index 688d7528..7a803c25 100644 --- a/packages/api/test/api-contract.test.ts +++ b/packages/api/test/api-contract.test.ts @@ -64,6 +64,15 @@ describe("versioned API contract", () => { expect(post?.body?.safeParse({ generationId: "00000000-0000-4000-8000-000000000001", encryptedVerification: "ciphertext", key: "plain" }).success).toBe(false); expect(openapi.paths["/api/v1/unlock-key/verification"].post.requestBody).toBeDefined(); }); + it("publishes model-account proxy capabilities as owner-only API operations", () => { + const route = API_ROUTES.find(item => item.method === "POST" && item.path === "/api/v1/model-accounts/{accountId}/browser-capability"); + expect(route?.scope).toBe("integrations:write"); + expect(route?.ownerSession).toBe(true); + expect(route?.parameters.shape.accountId).toBeDefined(); + expect(route?.body?.safeParse({ action: "authorize" }).success).toBe(true); + expect(route?.body?.safeParse({ action: "authorize", accountId: "attacker" }).success).toBe(false); + expect(openapi.paths["/api/v1/model-accounts/{accountId}/browser-capability"].post.requestBody).toBeDefined(); + }); it("accepts code jobs without agent prompts or models and rejects missing scripts", () => { const code = { name: "Export", slug: "export", executionType: "code", script: "#!/bin/sh\nprintf '{\"status\":\"succeeded\"}' > \"$FACTORIZE_RESULT_PATH\"", executionConfigId: "exe", triggers: [] }; expect(jobInputSchema.safeParse(code).success).toBe(true); diff --git a/packages/api/test/fly-credential-proxy-provisioner.test.ts b/packages/api/test/fly-credential-proxy-provisioner.test.ts index d2b1a976..098dcc06 100644 --- a/packages/api/test/fly-credential-proxy-provisioner.test.ts +++ b/packages/api/test/fly-credential-proxy-provisioner.test.ts @@ -9,6 +9,7 @@ const input = { image: `ghcr.io/generationsoftware/credential-proxy@sha256:${"a".repeat(64)}`, region: "iad", size: "shared-cpu-1x", + browserCapabilityPublicKey: "-----BEGIN PUBLIC KEY-----\nfixture\n-----END PUBLIC KEY-----", controlOrigin: "https://factorize.example", operationId: "tenant-proxy-start-1", }; @@ -37,7 +38,8 @@ describe("Fly credential proxy startup", () => { image: input.image, region: input.region, cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", epoch], - env: { FACTORIZE_PROXY_TUNNEL_SERVER_NAME: "factorize-proxies.internal", FACTORIZE_PROXY_TUNNEL_PORT: "9090" }, + env: { FACTORIZE_PROXY_TUNNEL_SERVER_NAME: "factorize-proxies.internal", FACTORIZE_PROXY_TUNNEL_PORT: "9090", FACTORIZE_PROXY_BROWSER_CAPABILITY_PUBLIC_KEY: input.browserCapabilityPublicKey }, + services: [{ protocol: "tcp", internal_port: 8080, ports: [{ port: 443, handlers: ["tls", "http"] }], force_instance_key: true }], restart: { policy: "no" }, auto_start: false, metadata: expect.objectContaining({ factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId }), @@ -52,6 +54,7 @@ describe("Fly credential proxy startup", () => { proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, + browserOrigin: `https://${input.app}.fly.dev`, credential: `fzc_${"b".repeat(43)}`, })); }); @@ -88,7 +91,7 @@ describe("Fly credential proxy guest adapter", () => { return child; }) as any; const guest = createFlyCredentialProxyGuest({ app: input.app, machine: machine.id, token: "server-token-fixture", spawnProcess }); - await guest.startProxy({ tenantId: input.tenantId, proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, credential: `fzc_${"c".repeat(43)}` }); + await guest.startProxy({ tenantId: input.tenantId, proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, browserOrigin: `https://${input.app}.fly.dev`, credential: `fzc_${"c".repeat(43)}` }); expect(calls[0]?.[1]).toContain("/usr/local/bin/factorize-proxy-startup"); expect(String(calls[1]?.[1])).toContain(`"type":"register-boot"`); expect(String(calls[1]?.[1])).toContain(`"credential":"fzc_${"c".repeat(43)}"`); diff --git a/packages/api/test/proxy-browser-capability.test.ts b/packages/api/test/proxy-browser-capability.test.ts new file mode 100644 index 00000000..3729c98a --- /dev/null +++ b/packages/api/test/proxy-browser-capability.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { generateKeyPairSync, verify } from "node:crypto"; +import { issueModelAccountCapability } from "../src/proxy-browser-capability"; + +describe("proxy browser capabilities", () => { + it("signs an account/action/origin/boot-bound one-minute EdDSA token", async () => { + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + const token = await issueModelAccountCapability({ + privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(), + proxyIdentity: "fly:proxy:machine-1", ownerId: "11111111-1111-4111-8111-111111111111", + accountId: "22222222-2222-4222-8222-222222222222", action: "authorize", + appOrigin: "https://factorize.example", generation: "33333333-3333-4333-8333-333333333333", + now: 1_900_000_000, jti: "one-use-fixture-nonce-123456", + }); + const [header, claims, signature] = token.split("."); + expect(JSON.parse(Buffer.from(header!, "base64url").toString())).toEqual({ alg: "EdDSA", typ: "JWT" }); + expect(JSON.parse(Buffer.from(claims!, "base64url").toString())).toMatchObject({ + iss: "factorize", aud: "fly:proxy:machine-1", sub: "11111111-1111-4111-8111-111111111111", + action: "model-account:authorize", accountId: "22222222-2222-4222-8222-222222222222", + origin: "https://factorize.example", generation: "33333333-3333-4333-8333-333333333333", + exp: 1_900_000_060, jti: "one-use-fixture-nonce-123456", + }); + expect(verify(null, Buffer.from(`${header}.${claims}`), publicKey, Buffer.from(signature!, "base64url"))).toBe(true); + }); +}); diff --git a/packages/api/test/proxy-control-repository.test.ts b/packages/api/test/proxy-control-repository.test.ts index 9508486a..b622e0df 100644 --- a/packages/api/test/proxy-control-repository.test.ts +++ b/packages/api/test/proxy-control-repository.test.ts @@ -33,21 +33,22 @@ describe("dedicated credential proxy control identity", () => { const tx = { query: vi.fn().mockImplementation((sql: string) => sql.includes("credential_proxy_control_credentials") ? credentialQuery() : sql.includes("INSERT INTO app.credential_proxy_states") ? stateQuery() : Promise.resolve({ rows: [], rowCount: 1 })) }; (repository as any).database = { transaction: (work: (client: any) => Promise) => work(tx) }; const epoch = "22222222-2222-4222-8222-222222222222"; - const registration = await repository.registerBoot("fzc_" + "a".repeat(43), epoch); + const registration = await repository.registerBoot("fzc_" + "a".repeat(43), epoch, "https://proxy.example"); expect(registration).toMatchObject({ tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch: epoch }); expect(registration?.controlCredential).toMatch(/^fzc_[A-Za-z0-9_-]{43}$/); - expect(tx.query.mock.calls.map(call => call[0])).toHaveLength(4); + expect(tx.query.mock.calls.map(call => call[0])).toHaveLength(5); expect(tx.query.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(tx.query.mock.calls[2][0]).toContain("consumed_at=now()"); expect(tx.query.mock.calls[3][0]).toContain("boot_epoch<>$3"); + expect(tx.query.mock.calls[4][0]).toContain("credential_proxy_browser_pairings"); credentialQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 }); - expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch)).toBeNull(); + expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch, "https://proxy.example")).toBeNull(); }); it("rejects malformed boot epochs before opening a transaction", async () => { const transaction = vi.fn(); const repository = new ProxyControlRepository({ transaction, pool: { query: vi.fn() } } as any); - await expect(repository.registerBoot("fzc_" + "a".repeat(43), "old-boot")).rejects.toThrow("bootEpoch must be a UUID"); + await expect(repository.registerBoot("fzc_" + "a".repeat(43), "old-boot", "https://proxy.example")).rejects.toThrow("bootEpoch must be a UUID"); expect(transaction).not.toHaveBeenCalled(); }); }); diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index ae33a978..3bf692dd 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -21,7 +21,7 @@ function envFor({ credentialFound = true, stateSaved = true } = {}) { return { DATABASE: { pool: { query }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery, query } as any; } -function request({ authorization, body = { bootEpoch }, method = "POST", path = "/internal/credential-proxy/register-boot" }: { authorization?: string; body?: unknown; method?: string; path?: string } = {}) { +function request({ authorization, body = { bootEpoch, browserOrigin: "https://proxy.example" }, method = "POST", path = "/internal/credential-proxy/register-boot" }: { authorization?: string; body?: unknown; method?: string; path?: string } = {}) { return new Request(`https://factorize.example${path}`, { method, headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}) }, body: method === "POST" ? JSON.stringify(body) : undefined }); } @@ -31,9 +31,10 @@ describe("dedicated proxy boot registration endpoint", () => { const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ registered: true, tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: expect.stringMatching(/^fzc_[A-Za-z0-9_-]{43}$/) }); - expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(4); + expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(5); expect(env.transactionQuery.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(env.transactionQuery.mock.calls[2][0]).toContain("consumed_at=now()"); + expect(env.transactionQuery.mock.calls[4][0]).toContain("credential_proxy_browser_pairings"); }); it("rejects guest job tokens before database access", async () => { diff --git a/packages/api/test/proxy-provisioner.test.ts b/packages/api/test/proxy-provisioner.test.ts index 1a107b53..bc6e9eca 100644 --- a/packages/api/test/proxy-provisioner.test.ts +++ b/packages/api/test/proxy-provisioner.test.ts @@ -6,6 +6,7 @@ const input = { proxyIdentity: "fly-app/proxy-machine", bootEpoch: "22222222-2222-4222-8222-222222222222", controlOrigin: "https://factorize.example/", + browserOrigin: "https://proxy.example", }; describe("credential proxy bootstrap provisioning", () => { @@ -40,6 +41,7 @@ describe("credential proxy bootstrap provisioning", () => { const control = { provisionCredential: vi.fn(), revokeCredential: vi.fn() }; await expect(provisionCredentialProxy({ ...input, controlOrigin: "http://factorize.example" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); await expect(provisionCredentialProxy({ ...input, controlOrigin: "https://factorize.example/path" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); + await expect(provisionCredentialProxy({ ...input, browserOrigin: "http://proxy.example" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); expect(control.provisionCredential).not.toHaveBeenCalled(); }); }); diff --git a/packages/credential-proxy/src/bootstrap-control.mjs b/packages/credential-proxy/src/bootstrap-control.mjs index 8d17234c..68ba7eb1 100644 --- a/packages/credential-proxy/src/bootstrap-control.mjs +++ b/packages/credential-proxy/src/bootstrap-control.mjs @@ -54,8 +54,9 @@ export async function startProxyBootstrapControl({ runtime, socketPath = "/run/f } if (request?.type !== "register-boot") return finish({ error: "invalid bootstrap request" }); if (!request || typeof request !== "object" || Array.isArray(request) - || Object.keys(request).sort().join(",") !== "bootEpoch,controlOrigin,credential,type" + || Object.keys(request).sort().join(",") !== "bootEpoch,browserOrigin,controlOrigin,credential,type" || request.bootEpoch !== runtime.bootEpoch || typeof request.credential !== "string" + || typeof request.browserOrigin !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(request.credential)) { return finish({ error: "invalid bootstrap request" }); } @@ -64,10 +65,10 @@ export async function startProxyBootstrapControl({ runtime, socketPath = "/run/f socket.removeAllListeners("data"); let credential = request.credential; try { - const registration = await register({ controlOrigin: request.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + const registration = await register({ controlOrigin: request.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch, browserOrigin: request.browserOrigin }); // Keep the exchanged proxy credential inside the proxy process. The // one-shot startup command receives only the public registration result. - await onRegistered(registration, { controlOrigin: request.controlOrigin, bootEpoch: runtime.bootEpoch }); + await onRegistered(registration, { controlOrigin: request.controlOrigin, browserOrigin: request.browserOrigin, bootEpoch: runtime.bootEpoch }); finish({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { finish({ error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/src/browser-control.mjs b/packages/credential-proxy/src/browser-control.mjs index 5911ba14..e87e1218 100644 --- a/packages/credential-proxy/src/browser-control.mjs +++ b/packages/credential-proxy/src/browser-control.mjs @@ -60,7 +60,7 @@ export async function startBrowserControl({ now = () => Math.floor(Date.now() / 1000), } = {}) { if (!runtime || !lifecycle || !publicKey || typeof proxyId !== "string" || !proxyId - || !UUID.test(ownerId ?? "") || !Number.isSafeInteger(generation) || generation < 1 + || !UUID.test(ownerId ?? "") || !(Number.isSafeInteger(generation) && generation > 0 || UUID.test(generation ?? "")) || !Number.isInteger(port) || port < 0 || port > 65535) { throw new TypeError("browser control configuration is invalid"); } diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 6ea36bab..9b4656c9 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -7,16 +7,18 @@ function validateOrigin(controlOrigin) { return origin; } -export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetchImpl = fetch }) { +export async function registerProxyBoot({ controlOrigin, token, bootEpoch, browserOrigin, fetchImpl = fetch }) { const origin = validateOrigin(controlOrigin); if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token ?? "")) throw new TypeError("invalid proxy control credential"); if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); + const browser = new URL(browserOrigin); + if (browser.protocol !== "https:" || browser.origin !== browserOrigin || browser.username || browser.password) throw new TypeError("invalid browser control origin"); let credential = token; try { const response = await fetchImpl(new URL("/internal/credential-proxy/register-boot", origin), { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${credential}` }, - body: JSON.stringify({ bootEpoch }), + body: JSON.stringify({ bootEpoch, browserOrigin }), signal: AbortSignal.timeout(10_000), redirect: "error", }); diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 4b2765bb..016c80a3 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -10,6 +10,7 @@ import { createCodexAuthHelper } from "./codex-auth-helper.mjs"; import { createModelAccountLifecycle } from "./model-account-lifecycle.mjs"; import { createModelAccountPersistenceAdapter } from "./control-client.mjs"; import { decryptModelAccountCache, encryptModelAccountCache } from "./model-account-crypto.mjs"; +import { startBrowserControl } from "./browser-control.mjs"; const epochIndex = process.argv.indexOf("--boot-epoch"); const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; @@ -26,8 +27,9 @@ let tunnel; let revocationMonitor; let modelAccounts; let modelAccountLifecycle; +let browserControl; -async function activateRegistration(registration, origin) { +async function activateRegistration(registration, origin, browserOrigin) { if (tunnel) throw new Error("proxy boot is already registered"); const persistence = createModelAccountPersistenceAdapter({ controlOrigin: origin, @@ -44,6 +46,16 @@ async function activateRegistration(registration, origin) { encryptCache: encryptModelAccountCache, decryptCache: decryptModelAccountCache, }); + if (!process.env.FACTORIZE_PROXY_BROWSER_CAPABILITY_PUBLIC_KEY) throw new Error("proxy browser capability verification key is unavailable"); + browserControl = await startBrowserControl({ + runtime, + lifecycle: modelAccountLifecycle, + publicKey: process.env.FACTORIZE_PROXY_BROWSER_CAPABILITY_PUBLIC_KEY, + proxyId: registration.proxyIdentity, + ownerId: registration.tenantId, + generation: registration.bootEpoch, + appOrigin: process.env.FACTORIZE_APP_ORIGIN ?? "https://app.factorize.sh", + }); const resolveGrants = createProxyIntegrationGrantResolver({ runtime, controlOrigin: origin, controlCredential: registration.controlCredential }); const started = await startAuthenticatedProxyTunnel({ tls: tlsAuthority.tls, @@ -68,7 +80,7 @@ async function activateRegistration(registration, origin) { const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET, - onRegistered: (registration, request) => activateRegistration(registration, request.controlOrigin), + onRegistered: (registration, request) => activateRegistration(registration, request.controlOrigin, request.browserOrigin), issueVmCertificate: ({ vmIdentity, csr }) => { if (!tunnel) throw new Error("proxy tunnel is not registered"); return tlsAuthority.signVmCsr({ vmIdentity, csr }); @@ -93,6 +105,8 @@ async function stop() { controlOrigin = undefined; await revocationMonitor?.close(); revocationMonitor = undefined; + await browserControl?.close().catch(() => {}); + browserControl = undefined; await runtime.shutdown(); await tunnel?.close().catch(() => {}); tunnel = undefined; @@ -129,8 +143,8 @@ lines.on("line", async (line) => { let credential = command.credential; command.credential = undefined; try { - const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); - await activateRegistration(registration, command.controlOrigin); + const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, browserOrigin: command.browserOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + await activateRegistration(registration, command.controlOrigin, command.browserOrigin); respond({ type: "boot-registered", proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { respond({ type: "error", error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/test/bootstrap-control.test.mjs b/packages/credential-proxy/test/bootstrap-control.test.mjs index 5e7956ac..de122d5f 100644 --- a/packages/credential-proxy/test/bootstrap-control.test.mjs +++ b/packages/credential-proxy/test/bootstrap-control.test.mjs @@ -42,12 +42,12 @@ test("local startup control redeems the one-use credential and leaves a private async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }; }, (value, request) => registered.push({ value, request }), ); - const result = await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + const result = await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch, credential: token }); assert.deepEqual(result, { registered: true, proxyIdentity: "fly:app:machine", bootEpoch: epoch }); - assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch }]); + assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch, browserOrigin: "https://proxy.example" }]); assert.deepEqual(registered, [{ value: { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }, - request: { controlOrigin: "https://factorize.example", bootEpoch: epoch }, + request: { controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch }, }]); assert.equal(JSON.stringify(result).includes(token), false, "startup response never returns the proxy control credential"); assert.equal((await stat(socketPath)).mode & 0o777, 0o600); @@ -57,9 +57,9 @@ test("local startup control rejects run credentials, epoch substitution, and ext const calls = []; const socketPath = await fixture(async value => { calls.push(value); return {}; }); for (const payload of [ - { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, - { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, - { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, + { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, + { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, + { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, ]) assert.deepEqual(await request(socketPath, payload), { error: "invalid bootstrap request" }); assert.deepEqual(calls, []); }); @@ -77,7 +77,7 @@ test("local provider channel returns only a boot-bound VM certificate and public ); const payload = { type: "issue-vm-certificate", bootEpoch: epoch, vmIdentity: "fly:job-1", csr }; assert.deepEqual(await request(socketPath, payload), { error: "certificate provisioning failed" }); - await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch, credential: token }); assert.deepEqual(await request(socketPath, payload), { issued: true, bootEpoch: epoch, vmIdentity: "fly:job-1", certificate: "cert-for:fly:job-1", trustBundle: "public-ca", }); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index e25342cf..449cf991 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -9,7 +9,7 @@ const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R" test("boot registration uses a one-call control credential, HTTPS, exact path and fresh process epoch", async () => { let request; - const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async (url, init) => { + const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", token, bootEpoch, fetchImpl: async (url, init) => { request = { url: String(url), init }; return Response.json({ registered: true, tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); } }); @@ -17,7 +17,7 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an assert.equal(request.url, "https://factorize.example/internal/credential-proxy/register-boot"); assert.equal(request.init.headers.Authorization, `Bearer ${token}`); assert.equal(request.init.redirect, "error"); - assert.deepEqual(JSON.parse(request.init.body), { bootEpoch }); + assert.deepEqual(JSON.parse(request.init.body), { bootEpoch, browserOrigin: "https://proxy.example" }); }); test("proxy snapshot reader requests only the bound run revision and validates encrypted records", async () => { @@ -84,9 +84,9 @@ test("proxy revocation monitor applies durable run revocations and tolerates an }); test("boot registration rejects unsafe origins, invalid credentials and mismatched response bindings", async () => { - await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch }), /HTTPS origin/); - await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); - await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async () => Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch: "33333333-3333-4333-8333-333333333333" }) }), /invalid proxy boot registration response/); + await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch, browserOrigin: "https://proxy.example" }), /HTTPS origin/); + await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch, browserOrigin: "https://proxy.example" }), /credential/); + await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, browserOrigin: "https://proxy.example", fetchImpl: async () => Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch: "33333333-3333-4333-8333-333333333333" }) }), /invalid proxy boot registration response/); }); test("proxy control client requests and consumes a one-use enrollment over authenticated HTTPS", async () => { diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index 62e4f652..cb5d0509 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -1806,6 +1806,82 @@ components: schema: $ref: "#/components/schemas/Error" paths: + /api/v1/model-accounts/{accountId}/browser-capability: + post: + summary: Issue a short-lived browser capability for a paired model account + description: |- + Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens. + + Requires the integrations:write scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "200": + description: Paired proxy endpoint and one-use capability + content: + application/json: + schema: + type: object + properties: + endpoint: + type: string + format: uri + capability: + type: string + required: + - endpoint + - capability + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + "415": + $ref: "#/components/responses/Error415" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: post_api_v1_model_accounts_accountId_browser_capability + tags: + - model-accounts + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: integrations:write + x-owner-session-required: true + parameters: + - name: accountId + in: path + required: true + schema: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + action: + type: string + enum: + - authorize + - status + - cancel + required: + - action + additionalProperties: false /api/v1/unlock-key/verification: get: summary: Get the current encrypted unlock-key verification record From dd266ccaaa311f5f2c0e1a2ad1cd21f33971f4e0 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:03:23 +0000 Subject: [PATCH 038/102] GEN-2824 connect model account browser authorization --- packages/api-client/src/schema.d.ts | 60 +++++++++++++ packages/api/src/api-contract.ts | 6 ++ packages/api/src/flow-service.ts | 5 ++ packages/api/test/api-contract.test.ts | 7 ++ .../src/features/settings/integrations.tsx | 3 +- .../src/features/settings/model-accounts.tsx | 90 +++++++++++++++++++ packages/app/test/settings.browser.test.mjs | 29 ++++++ .../credential-proxy/src/browser-control.mjs | 23 ++++- .../src/codex-auth-helper.mjs | 7 +- .../src/model-account-lifecycle.mjs | 2 +- .../test/browser-control.test.mjs | 6 +- .../test/codex-auth-helper.test.mjs | 3 + packages/docs/openapi.yaml | 69 ++++++++++++++ 13 files changed, 300 insertions(+), 10 deletions(-) create mode 100644 packages/app/src/features/settings/model-accounts.tsx diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index e7c92965..56992fde 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -1,5 +1,29 @@ // Generated from packages/docs/openapi.yaml. Do not edit. export interface paths { + "/api/v1/model-accounts": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List model accounts available to the owner + * @description Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned. + * + * Requires the integrations:read scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + get: operations["get_api_v1_model_accounts"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/model-accounts/{accountId}/browser-capability": { parameters: { query?: never; @@ -2707,6 +2731,42 @@ export interface components { } export type $defs = Record; export interface operations { + get_api_v1_model_accounts: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Model account metadata */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + accounts: { + /** Format: uuid */ + id: string; + name: string; + description: string | null; + revision: number; + }[]; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 409: components["responses"]["Error409"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; post_api_v1_model_accounts_accountId_browser_capability: { parameters: { query?: never; diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index aae9e10a..36474829 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -131,6 +131,7 @@ const bearerIntegrationCreate = z.object({ const bearerIntegrationUpdate = bearerIntegrationCreate.omit({ id: true }); const bearerIntegrationDelete = z.object({ expectedRevision: z.number().int().nonnegative(), idempotencyKey: z.uuid() }).strict(); const bearerIntegrationList = z.object({ revision: z.number().int().nonnegative(), integrations: z.array(bearerIntegrationMetadata) }); +const modelAccountMetadata = z.object({ id: z.uuid(), name: z.string(), description: z.string().nullable(), revision: z.number().int().nonnegative() }); const bearerIntegrationWriteResult = z.object({ revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const bearerIntegrationDeleteResult = z.object({ id: z.uuid(), deleted: z.literal(true), revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const unlockKeyVerification = z.object({ ownerId: z.string(), verification: z.object({ generationId: z.uuid(), encryptedVerification: z.string() }).strict().nullable() }).strict(); @@ -214,6 +215,11 @@ function authRoute(method: string, path: string, action: AuthAction, summary: st } export const API_ROUTES: RouteContract[] = [ + route("GET", "/api/v1/model-accounts", "integrations:read", { + summary: "List model accounts available to the owner", + description: "Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned.", + responses: { "200": { description: "Model account metadata", content: { "application/json": { schema: jsonSchema(z.object({ accounts: z.array(modelAccountMetadata) }), "output") } } } }, + }, service => service.listModelAccounts(), { ownerSession: true }), route("POST", "/api/v1/model-accounts/{accountId}/browser-capability", "integrations:write", { summary: "Issue a short-lived browser capability for a paired model account", description: "Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens.", diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index c2c7253a..364742d9 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -589,6 +589,11 @@ export class ApiService { }); return { endpoint: `${pairing.browserOrigin}/model-accounts/${accountId}/${action}`, capability }; } + async listModelAccounts() { + await this.authorizeOwnerSession("integrations:read"); + const accounts = await this.encryptedIntegrations().records("model"); + return { accounts: accounts.map(({ id, name, description, revision }) => ({ id, name, description, revision })) }; + } async removeTailIntegration(id: string) { await this.authorize("integrations:write"); if (!await this.connections().delete(`cloudflare-tail:${id}`)) throw new ServiceError(404, "not_found", "Integration not found"); return { id, deleted: true }; } async linearProjects() { await this.authorize("integrations:read"); return publicValue(z.array(namedOption), await this.providers().linearProjects()); } async linearOptions() { await this.authorize("integrations:read"); return publicValue(providerOptions, await this.providers().linearOptions()); } diff --git a/packages/api/test/api-contract.test.ts b/packages/api/test/api-contract.test.ts index 7a803c25..bebc6035 100644 --- a/packages/api/test/api-contract.test.ts +++ b/packages/api/test/api-contract.test.ts @@ -73,6 +73,13 @@ describe("versioned API contract", () => { expect(route?.body?.safeParse({ action: "authorize", accountId: "attacker" }).success).toBe(false); expect(openapi.paths["/api/v1/model-accounts/{accountId}/browser-capability"].post.requestBody).toBeDefined(); }); + it("lists only safe model account metadata for the owner browser", () => { + const route = API_ROUTES.find(item => item.method === "GET" && item.path === "/api/v1/model-accounts"); + expect(route?.scope).toBe("integrations:read"); + expect(route?.ownerSession).toBe(true); + expect(route?.documentation.responses["200"].content["application/json"].schema.properties.accounts.items.properties.encryptedPayload).toBeUndefined(); + expect(openapi.paths["/api/v1/model-accounts"].get).toBeDefined(); + }); it("accepts code jobs without agent prompts or models and rejects missing scripts", () => { const code = { name: "Export", slug: "export", executionType: "code", script: "#!/bin/sh\nprintf '{\"status\":\"succeeded\"}' > \"$FACTORIZE_RESULT_PATH\"", executionConfigId: "exe", triggers: [] }; expect(jobInputSchema.safeParse(code).success).toBe(true); diff --git a/packages/app/src/features/settings/integrations.tsx b/packages/app/src/features/settings/integrations.tsx index dfa08fbc..f8c72b9a 100644 --- a/packages/app/src/features/settings/integrations.tsx +++ b/packages/app/src/features/settings/integrations.tsx @@ -10,6 +10,7 @@ import { IntegrationForm } from "./integration-form"; import { messageOf } from "../auth/session"; import { UnlockKeySetup } from "./unlock-key-setup"; import { BearerIntegrations } from "./bearer-integrations"; +import { ModelAccounts } from "./model-accounts"; const linkClass = "inline-flex min-h-10 items-center rounded-lg border border-stone-200 px-3 py-2 text-sm font-semibold hover:bg-stone-100 dark:border-slate-700 dark:hover:bg-slate-800"; @@ -53,7 +54,7 @@ export function Integrations() { {status.data?.cloudflareTail.installations.map(c =>

Cloudflare Tail · {c.name}

Referenced by {c.referencedJobCount} {c.referencedJobCount === 1 ? "job" : "jobs"}

Connected{buttons("cloudflareTail", c.integrationId)}
{manage(`tail:${c.integrationId}`, )}
)} } - +

Available integrations

Connect a provider to add it to your workspace. Connected providers remain available for additional connections.

setOpenSetup(open ? "linear-new" : null)}>{status.data?.linear ? "Add another Linear account" : "Connect Linear"} setOpenSetup(open ? "clickup-new" : null)}>{status.data?.clickup ? "Add another ClickUp workspace" : "Connect ClickUp"} diff --git a/packages/app/src/features/settings/model-accounts.tsx b/packages/app/src/features/settings/model-accounts.tsx new file mode 100644 index 00000000..b94e0806 --- /dev/null +++ b/packages/app/src/features/settings/model-accounts.tsx @@ -0,0 +1,90 @@ +import { useEffect, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { api } from "factorize-api-client"; +import { Button, Card } from "../../shared/ui"; +import { messageOf } from "../auth/session"; + +type DeviceState = { state: string; verificationUrl?: string; userCode?: string }; +type ModelAccount = { id: string; name: string; description: string | null; revision: number }; +const accountsQuery = { queryKey: ["settings", "model-accounts"], queryFn: async ({ signal }: { signal: AbortSignal }) => { + const { data, error } = await api.GET("/api/v1/model-accounts", { signal }); + if (!data || error) throw new Error(messageOf(error)); + return data.accounts; +} }; + +async function proxyAction(accountId: string, action: "authorize" | "status" | "cancel"): Promise { + const capability = await api.POST("/api/v1/model-accounts/{accountId}/browser-capability", { + params: { path: { accountId } }, body: { action }, + }); + if (!capability.data || capability.error) throw new Error(messageOf(capability.error)); + const response = await fetch(capability.data.endpoint, { + method: "POST", mode: "cors", credentials: "omit", cache: "no-store", + headers: { Authorization: `Bearer ${capability.data.capability}`, "Content-Type": "application/json" }, + body: "{}", + }); + let result: DeviceState; + try { result = await response.json() as DeviceState; } + catch { throw new Error("The credential proxy returned an invalid response."); } + if (!response.ok) { + const messages: Record = { 423: "Unlock the credential proxy before connecting a model account.", 503: "The credential proxy is unavailable. Try again shortly." }; + throw new Error(messages[response.status] ?? "The credential proxy could not complete this request."); + } + return result; +} + +export function ModelAccounts() { + const accounts = useQuery(accountsQuery); + const [active, setActive] = useState(null); + const [states, setStates] = useState>({}); + const [error, setError] = useState(""); + + useEffect(() => { + if (!active) return; + let stopped = false; + const poll = async () => { + try { + const next = await proxyAction(active, "status"); + if (stopped) return; + setStates(current => ({ ...current, [active]: next })); + if (next.state !== "pending") setActive(null); + } catch (reason) { + if (!stopped) setError(reason instanceof Error ? reason.message : "Could not check model account authorization."); + } + }; + const timer = window.setInterval(poll, 2000); + return () => { stopped = true; window.clearInterval(timer); }; + }, [active]); + + const run = async (account: ModelAccount, action: "authorize" | "cancel") => { + setError(""); + try { + const next = await proxyAction(account.id, action); + setStates(current => ({ ...current, [account.id]: next })); + setActive(action === "authorize" && next.state === "pending" ? account.id : null); + } catch (reason) { + setError(reason instanceof Error ? reason.message : "Could not connect the model account."); + } + }; + + return
+

ChatGPT model accounts

Connect each account through Codex device authorization. Credentials stay in the credential proxy.

+ {accounts.isPending &&

Loading model accounts…

} + {accounts.error &&

{accounts.error.message}

} + {error &&

{error}

} + {!accounts.isPending && accounts.data?.length === 0 &&

No model accounts are configured yet.

} + {accounts.data?.map(account => { + const state = states[account.id] ?? { state: "idle" }; + return +

{account.name}

{account.description &&

{account.description}

} + {state.state === "pending" &&

Waiting for device authorization.

{state.userCode &&

Code: {state.userCode}

}{state.verificationUrl && OpenAI device authorization}
} + {state.state === "authorized" &&

ChatGPT account connected.

} + {state.state === "authorized-unsaved" &&

ChatGPT account connected, but its encrypted token cache could not be saved. Keep the proxy running and retry after checking its connection.

} + {state.state === "reauthentication-required" &&

Sign in again to reconnect this account.

} + {state.state === "failed" &&

Device authorization failed. You can try again.

} + {state.state === "canceled" &&

Device authorization canceled.

} +
+ {state.state === "pending" ? : } + ; + })} +
; +} diff --git a/packages/app/test/settings.browser.test.mjs b/packages/app/test/settings.browser.test.mjs index bc3368e5..83db10d0 100644 --- a/packages/app/test/settings.browser.test.mjs +++ b/packages/app/test/settings.browser.test.mjs @@ -18,6 +18,35 @@ test("integration setup tests credentials, submits typed fields, clears secrets await form.getByLabel("exe.dev account token").fill("private-input"); await form.getByRole("button", { name: "Test credentials" }).click(); await form.getByText("✓ Credentials verified.", { exact: true }).waitFor(); await form.getByLabel("VM tags").selectOption(["dev"]); await form.getByRole("button", { name: "Save integration", exact: true }).click(); await form.getByText("Integration saved", { exact: true }).waitFor(); assert.deepEqual(body.tags, ["dev"]); assert.equal(body.agentKind, "codex"); assert.equal(await form.getByLabel("exe.dev account token").inputValue(), ""); assert.equal(await page.evaluate(() => JSON.stringify(localStorage).includes("private-input")), false); await page.getByText("dev", { exact: true }).waitFor(); await context.close(); }); +test("model account settings issue short-lived capabilities and display the bounded device challenge", async () => { + const { context, page } = await setup(); + const accountId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", proxyOrigin = "https://proxy.example"; + await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); + await page.route("**/api/v1/integrations", route => route.fulfill({ json: { linear: null, clickup: null, exe: null, exeConnections: [], ampConnections: [], cloudflareTail: { count: 0, installations: [] } } })); + await page.route("**/api/v1/model-accounts", async route => { + if (route.request().method() === "GET") return route.fulfill({ json: { accounts: [{ id: accountId, name: "Work account", description: null, revision: 0 }] } }); + const action = route.request().postDataJSON().action; + return route.fulfill({ json: { endpoint: `${proxyOrigin}/model-accounts/${accountId}/${action}`, capability: `one-use-${action}` } }); + }); + const proxyCalls = []; + await page.route(`${proxyOrigin}/model-accounts/**`, async route => { + const request = route.request(); + if (request.method() === "OPTIONS") return route.fulfill({ status: 204, headers: { "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Methods": "POST", "Access-Control-Allow-Headers": "Authorization, Content-Type" } }); + proxyCalls.push({ path: new URL(request.url()).pathname, authorization: request.headers().authorization, body: request.postData() }); + const response = request.url().endsWith("/authorize") ? { state: "pending", verificationUrl: "https://auth.openai.com/codex/device", userCode: "ABCD-1234" } : { state: "canceled" }; + return route.fulfill({ json: response, headers: { "Access-Control-Allow-Origin": origin, "Cache-Control": "no-store" } }); + }); + await page.goto(origin + "/settings/integrations"); + const models = page.getByRole("region", { name: "ChatGPT model accounts" }); + await models.getByRole("button", { name: "Connect ChatGPT" }).click(); + await models.getByText("ABCD-1234", { exact: false }).waitFor(); + await models.getByRole("link", { name: "OpenAI device authorization" }).waitFor(); + await models.getByRole("button", { name: "Cancel authorization" }).click(); + await models.getByText("Device authorization canceled.", { exact: true }).waitFor(); + assert.deepEqual(proxyCalls.map(call => call.authorization), ["Bearer one-use-authorize", "Bearer one-use-cancel"]); + assert.deepEqual(proxyCalls.map(call => call.body), ["{}", "{}"]); + await context.close(); +}); test("unlock-key setup requires acknowledgement, stores ciphertext only, verifies entered keys and survives refresh without a saved key", async () => { const { context, page } = await setup(); await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); diff --git a/packages/credential-proxy/src/browser-control.mjs b/packages/credential-proxy/src/browser-control.mjs index e87e1218..930498b8 100644 --- a/packages/credential-proxy/src/browser-control.mjs +++ b/packages/credential-proxy/src/browser-control.mjs @@ -32,6 +32,18 @@ function json(response, status, body, origin) { response.writeHead(status, headers).end(JSON.stringify(body)); } +function publicAuthorization(value) { + if (!value || typeof value !== "object") return { state: "idle" }; + const result = { state: value.state }; + const challenge = value.challenge; + if (challenge?.verificationUrl === "https://auth.openai.com/codex/device" + && typeof challenge.userCode === "string" && (/^[A-Z0-9]{4,6}-[A-Z0-9]{4,6}$/u.test(challenge.userCode) || /^[A-Z0-9]{8,12}$/u.test(challenge.userCode))) { + result.verificationUrl = challenge.verificationUrl; + result.userCode = challenge.userCode; + } + return result; +} + async function readJson(request) { const chunks = []; let size = 0; @@ -119,16 +131,19 @@ export async function startBrowserControl({ if (operation === "authorize") { if (runtime.state !== "unlocked") return json(response, 423, { error: "proxy_locked" }, appOrigin); const login = await lifecycle.deviceLogin(accountId); - const state = { state: "pending" }; + const state = { state: "pending", challenge: login.challenge }; authorizations.set(accountId, state); Promise.resolve(login.completion).then(result => { - if (authorizations.get(accountId) === state) authorizations.set(accountId, { ...result }); + if (authorizations.get(accountId) === state) { + const next = result?.state === "saved" ? "authorized" : result?.state === "unsaved" ? "authorized-unsaved" : result?.state ?? "reauthentication-required"; + authorizations.set(accountId, { state: next }); + } }, () => { if (authorizations.get(accountId) === state) authorizations.set(accountId, { state: "failed" }); }); - return json(response, 202, state, appOrigin); + return json(response, 202, publicAuthorization(state), appOrigin); } - if (operation === "status") return json(response, 200, authorizations.get(accountId) ?? { state: "idle" }, appOrigin); + if (operation === "status") return json(response, 200, publicAuthorization(authorizations.get(accountId)), appOrigin); const result = await lifecycle.cancel(accountId); authorizations.set(accountId, { state: "canceled" }); return json(response, 200, { state: result?.state === "idle" ? "idle" : "canceled" }, appOrigin); diff --git a/packages/credential-proxy/src/codex-auth-helper.mjs b/packages/credential-proxy/src/codex-auth-helper.mjs index 199d906f..8f888875 100644 --- a/packages/credential-proxy/src/codex-auth-helper.mjs +++ b/packages/credential-proxy/src/codex-auth-helper.mjs @@ -99,6 +99,7 @@ export function createCodexAuthHelper({ if (logins.has(accountId)) throw new Error("device authorization is already running for this account"); const home = await prepareHome(accountId); let child; + const challenge = {}; const completion = new Promise((resolveRun, rejectRun) => { child = spawnProcess(codexPath, ["login", "--device-auth"], { env: { PATH: process.env.PATH ?? "", CODEX_HOME: home, HOME: home }, @@ -109,6 +110,10 @@ export function createCodexAuthHelper({ timer.unref?.(); child.stdout.on("data", chunk => { stdout = (stdout + chunk.toString("utf8")).slice(-16_384); + const url = stdout.match(/https:\/\/auth\.openai\.com\/codex\/device(?:\b|\/)/u)?.[0]; + const code = stdout.match(/\b(?:user\s+)?code\s*:\s*([A-Z0-9]{4,6}-[A-Z0-9]{4,6}|[A-Z0-9]{8,12})\b/iu)?.[1]; + if (url) challenge.verificationUrl = "https://auth.openai.com/codex/device"; + if (code) challenge.userCode = code.toUpperCase(); }); child.stderr.on("data", chunk => { // Retain no provider error text, which could include device details. @@ -124,7 +129,7 @@ export function createCodexAuthHelper({ const entry = { child, completion }; logins.set(accountId, entry); completion.finally(() => { if (logins.get(accountId) === entry) logins.delete(accountId); }).catch(() => {}); - return { state: "pending", completion }; + return { state: "pending", challenge, completion }; } if (request.method === "cancel") { const login = logins.get(accountId); diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index 8e13d05f..fc6a1308 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -120,7 +120,7 @@ export function createModelAccountLifecycle({ if (outcome?.state !== "authorized") return { state: outcome?.state ?? "reauthentication-required" }; return persist(id, generation); }); - return { state: "pending", completion }; + return { state: "pending", challenge: result.challenge, completion }; }); } diff --git a/packages/credential-proxy/test/browser-control.test.mjs b/packages/credential-proxy/test/browser-control.test.mjs index 0c7af413..918c6995 100644 --- a/packages/credential-proxy/test/browser-control.test.mjs +++ b/packages/credential-proxy/test/browser-control.test.mjs @@ -22,7 +22,7 @@ test("private browser control requires exact origin and one-use owner/proxy/acti const calls = []; const runtime = { state: "unlocked" }; const lifecycle = { - async deviceLogin(id) { calls.push(["authorize", id]); return { state: "pending", completion: Promise.resolve({ state: "saved" }) }; }, + async deviceLogin(id) { calls.push(["authorize", id]); return { state: "pending", challenge: { verificationUrl: "https://auth.openai.com/codex/device", userCode: "ABCD-1234" }, completion: Promise.resolve({ state: "saved" }) }; }, async cancel(id) { calls.push(["cancel", id]); return { state: "canceled" }; }, }; const server = await startBrowserControl({ runtime, lifecycle, publicKey, proxyId, ownerId, generation, appOrigin, port: 0, host: "127.0.0.1", now: () => now }); @@ -44,13 +44,13 @@ test("private browser control requires exact origin and one-use owner/proxy/acti const authToken = capability({ action: "model-account:authorize" }); const started = await post("model-account:authorize", authToken); assert.equal(started.status, 202); - assert.deepEqual(await started.json(), { state: "pending" }); + assert.deepEqual(await started.json(), { state: "pending", verificationUrl: "https://auth.openai.com/codex/device", userCode: "ABCD-1234" }); assert.deepEqual(calls[0], ["authorize", accountId]); const replay = await post("model-account:authorize", authToken); assert.equal(replay.status, 401); const status = await post("model-account:status", capability({ action: "model-account:status" })); - assert.deepEqual(await status.json(), { state: "saved" }); + assert.deepEqual(await status.json(), { state: "authorized" }); const wrongAudience = await post("model-account:cancel", capability({ action: "model-account:cancel", aud: "fly:other:machine" })); assert.equal(wrongAudience.status, 401); const expired = await post("model-account:cancel", capability({ action: "model-account:cancel", exp: now - 1 })); diff --git a/packages/credential-proxy/test/codex-auth-helper.test.mjs b/packages/credential-proxy/test/codex-auth-helper.test.mjs index 1b386c1d..9528f76f 100644 --- a/packages/credential-proxy/test/codex-auth-helper.test.mjs +++ b/packages/credential-proxy/test/codex-auth-helper.test.mjs @@ -63,8 +63,11 @@ test("device authorization runs as a separate cancellable login process", async const helper = createCodexAuthHelper({ accountHomeRoot: directory, spawnProcess: fake.spawnProcess }); const result = await helper.invoke({ method: "device-login", accountId: accountA }); assert.equal(result.state, "pending"); + assert.deepEqual(result.challenge, {}); assert.deepEqual(fake.calls.map(call => call.args), [["--version"], ["login", "--device-auth"]]); assert.equal(fake.calls[1].options.env.CODEX_HOME, join(directory, accountA)); + fake.children[1].stdout.write("Open this URL in your browser: https://auth.openai.com/codex/device\nEnter user code: abcd-1234\n"); + assert.deepEqual(result.challenge, { verificationUrl: "https://auth.openai.com/codex/device", userCode: "ABCD-1234" }); assert.deepEqual(await helper.invoke({ method: "cancel", accountId: accountA }), { state: "canceled" }); await assert.rejects(result.completion, /stopped/); assert.deepEqual(await helper.invoke({ method: "cancel", accountId: accountA }), { state: "idle" }); diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index cb5d0509..6089764f 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -1806,6 +1806,75 @@ components: schema: $ref: "#/components/schemas/Error" paths: + /api/v1/model-accounts: + get: + summary: List model accounts available to the owner + description: |- + Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned. + + Requires the integrations:read scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "200": + description: Model account metadata + content: + application/json: + schema: + type: object + properties: + accounts: + type: array + items: + type: object + properties: + id: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + name: + type: string + description: + type: + - string + - "null" + revision: + type: integer + minimum: 0 + maximum: 9007199254740991 + required: + - id + - name + - description + - revision + additionalProperties: false + required: + - accounts + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: get_api_v1_model_accounts + tags: + - model-accounts + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: integrations:read + x-owner-session-required: true /api/v1/model-accounts/{accountId}/browser-capability: post: summary: Issue a short-lived browser capability for a paired model account From 2ccb27b00f59e51388e03382ad7ce2a6b0fc3f8c Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:07:01 +0000 Subject: [PATCH 039/102] GEN-2824 cover capability endpoint in browser test --- packages/app/test/settings.browser.test.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/app/test/settings.browser.test.mjs b/packages/app/test/settings.browser.test.mjs index 83db10d0..81903180 100644 --- a/packages/app/test/settings.browser.test.mjs +++ b/packages/app/test/settings.browser.test.mjs @@ -28,6 +28,10 @@ test("model account settings issue short-lived capabilities and display the boun const action = route.request().postDataJSON().action; return route.fulfill({ json: { endpoint: `${proxyOrigin}/model-accounts/${accountId}/${action}`, capability: `one-use-${action}` } }); }); + await page.route("**/api/v1/model-accounts/*/browser-capability", async route => { + const action = route.request().postDataJSON().action; + return route.fulfill({ json: { endpoint: `${proxyOrigin}/model-accounts/${accountId}/${action}`, capability: `one-use-${action}` } }); + }); const proxyCalls = []; await page.route(`${proxyOrigin}/model-accounts/**`, async route => { const request = route.request(); From 87341ad22b547a4b9c3c55df8e5e56cb5e133bce Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:29:09 +0000 Subject: [PATCH 040/102] GEN-2824 add model account create and delete --- packages/api-client/src/schema.d.ts | 130 +++++++++++- packages/api/src/api-contract.ts | 21 +- packages/api/src/flow-service.ts | 46 ++++- .../encrypted-integration-repository.ts | 27 +++ packages/api/test/api-contract.test.ts | 9 + ...encrypted-integration-api.postgres.test.ts | 37 ++++ .../src/features/settings/model-accounts.tsx | 70 ++++++- packages/app/src/lib/unlock-crypto.ts | 16 ++ packages/app/test/settings.browser.test.mjs | 66 +++++- packages/app/test/unlock-crypto.test.mjs | 21 +- packages/docs/openapi.yaml | 190 +++++++++++++++++- 11 files changed, 615 insertions(+), 18 deletions(-) diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index 56992fde..1e4bd5e7 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -9,7 +9,7 @@ export interface paths { }; /** * List model accounts available to the owner - * @description Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned. + * @description Requires an interactive owner browser session. Returns account metadata and the collection revision only; encrypted account caches and credentials are never returned. * * Requires the integrations:read scope. * @@ -17,13 +17,45 @@ export interface paths { */ get: operations["get_api_v1_model_accounts"]; put?: never; - post?: never; + /** + * Create a model account + * @description Requires an interactive owner browser session and the current collection revision. Accepts only user-encrypted initial account state; credentials are established directly with the credential proxy. + * + * Requires the integrations:write scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + post: operations["post_api_v1_model_accounts"]; delete?: never; options?: never; head?: never; patch?: never; trace?: never; }; + "/api/v1/model-accounts/{accountId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + /** + * Delete a model account + * @description Requires an interactive owner browser session and the current collection revision. Deletion is recorded as a tombstone so future snapshots and unlock reconciliation cannot restore the account. + * + * Requires the integrations:write scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + delete: operations["delete_api_v1_model_accounts_accountId"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/model-accounts/{accountId}/browser-capability": { parameters: { query?: never; @@ -2740,13 +2772,14 @@ export interface operations { }; requestBody?: never; responses: { - /** @description Model account metadata */ + /** @description Model account metadata and collection revision */ 200: { headers: { [name: string]: unknown; }; content: { "application/json": { + revision: number; accounts: { /** Format: uuid */ id: string; @@ -2767,6 +2800,97 @@ export interface operations { 503: components["responses"]["Error503"]; }; }; + post_api_v1_model_accounts: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + /** Format: uuid */ + id: string; + expectedRevision: number; + /** Format: uuid */ + idempotencyKey: string; + name: string; + description: string | null; + encryptedPayload: string; + }; + }; + }; + responses: { + /** @description Created model account */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + revision: number; + duplicate: boolean; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 409: components["responses"]["Error409"]; + 415: components["responses"]["Error415"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; + delete_api_v1_model_accounts_accountId: { + parameters: { + query?: never; + header?: never; + path: { + accountId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + expectedRevision: number; + /** Format: uuid */ + idempotencyKey: string; + }; + }; + }; + responses: { + /** @description Deleted model account */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** Format: uuid */ + id: string; + /** @constant */ + deleted: true; + revision: number; + duplicate: boolean; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 409: components["responses"]["Error409"]; + 415: components["responses"]["Error415"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; post_api_v1_model_accounts_accountId_browser_capability: { parameters: { query?: never; diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index 36474829..6ea38cdc 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -132,6 +132,13 @@ const bearerIntegrationUpdate = bearerIntegrationCreate.omit({ id: true }); const bearerIntegrationDelete = z.object({ expectedRevision: z.number().int().nonnegative(), idempotencyKey: z.uuid() }).strict(); const bearerIntegrationList = z.object({ revision: z.number().int().nonnegative(), integrations: z.array(bearerIntegrationMetadata) }); const modelAccountMetadata = z.object({ id: z.uuid(), name: z.string(), description: z.string().nullable(), revision: z.number().int().nonnegative() }); +const modelAccountCreate = z.object({ + id: z.uuid(), expectedRevision: z.number().int().nonnegative(), idempotencyKey: z.uuid(), + name: z.string().trim().min(1).max(120), description: z.string().max(500).nullable(), + encryptedPayload: z.string().min(1).max(65_536), +}).strict(); +const modelAccountDelete = z.object({ expectedRevision: z.number().int().nonnegative(), idempotencyKey: z.uuid() }).strict(); +const modelAccountList = z.object({ revision: z.number().int().nonnegative(), accounts: z.array(modelAccountMetadata) }); const bearerIntegrationWriteResult = z.object({ revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const bearerIntegrationDeleteResult = z.object({ id: z.uuid(), deleted: z.literal(true), revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const unlockKeyVerification = z.object({ ownerId: z.string(), verification: z.object({ generationId: z.uuid(), encryptedVerification: z.string() }).strict().nullable() }).strict(); @@ -217,9 +224,19 @@ function authRoute(method: string, path: string, action: AuthAction, summary: st export const API_ROUTES: RouteContract[] = [ route("GET", "/api/v1/model-accounts", "integrations:read", { summary: "List model accounts available to the owner", - description: "Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned.", - responses: { "200": { description: "Model account metadata", content: { "application/json": { schema: jsonSchema(z.object({ accounts: z.array(modelAccountMetadata) }), "output") } } } }, + description: "Requires an interactive owner browser session. Returns account metadata and the collection revision only; encrypted account caches and credentials are never returned.", + responses: { "200": { description: "Model account metadata and collection revision", content: { "application/json": { schema: jsonSchema(modelAccountList, "output") } } } }, }, service => service.listModelAccounts(), { ownerSession: true }), + route("POST", "/api/v1/model-accounts", "integrations:write", { + summary: "Create a model account", + description: "Requires an interactive owner browser session and the current collection revision. Accepts only user-encrypted initial account state; credentials are established directly with the credential proxy.", + responses: { "201": { description: "Created model account", content: { "application/json": { schema: jsonSchema(bearerIntegrationWriteResult, "output") } } } }, + }, (service, { body }) => service.createModelAccount(body), { body: modelAccountCreate, maxBodyBytes: 70_000, ownerSession: true }), + route("DELETE", "/api/v1/model-accounts/{accountId}", "integrations:write", { + summary: "Delete a model account", + description: "Requires an interactive owner browser session and the current collection revision. Deletion is recorded as a tombstone so future snapshots and unlock reconciliation cannot restore the account.", + responses: { "200": { description: "Deleted model account", content: { "application/json": { schema: jsonSchema(bearerIntegrationDeleteResult, "output") } } } }, + }, (service, { params, body }) => service.deleteModelAccount(params.accountId, body), { body: modelAccountDelete, maxBodyBytes: 4096, ownerSession: true, parameters: z.object({ accountId: z.uuid() }) }), route("POST", "/api/v1/model-accounts/{accountId}/browser-capability", "integrations:write", { summary: "Issue a short-lived browser capability for a paired model account", description: "Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens.", diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index 364742d9..896dfd42 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -591,8 +591,50 @@ export class ApiService { } async listModelAccounts() { await this.authorizeOwnerSession("integrations:read"); - const accounts = await this.encryptedIntegrations().records("model"); - return { accounts: accounts.map(({ id, name, description, revision }) => ({ id, name, description, revision })) }; + const repository = this.encryptedIntegrations(); + const [revision, accounts] = await Promise.all([repository.revision(), repository.records("model")]); + return { revision, accounts: accounts.map(({ id, name, description, revision }) => ({ id, name, description, revision })) }; + } + private async modelAccountWrite(input: { id: string; expectedRevision: number; idempotencyKey: string; name: string; description: string | null; encryptedPayload?: string; operation: "upsert" | "delete" }) { + try { return await this.encryptedIntegrations().write({ ...input, kind: "model", origin: null }); } + catch (error) { + if (error instanceof StaleIntegrationRevisionError) throw new ServiceError(409, "stale_revision", `The model account collection changed. Current revision is ${error.currentRevision}.`); + if (error instanceof IntegrationIdempotencyConflictError) throw new ServiceError(409, "conflict", "The idempotency key was already used for a different model account change."); + if (error instanceof IntegrationValidationError) throw new ServiceError(400, "invalid_request", error.message); + throw error; + } + } + async createModelAccount(input: { id: string; expectedRevision: number; idempotencyKey: string; name: string; description: string | null; encryptedPayload: string }) { + await this.authorizeOwnerSession("integrations:write"); + const repository = this.encryptedIntegrations(); + try { + const replay = await repository.replay({ ...input, kind: "model", origin: null, operation: "upsert" }); + if (replay) return replay; + } catch (error) { + if (error instanceof IntegrationIdempotencyConflictError) throw new ServiceError(409, "conflict", "The idempotency key was already used for a different model account change."); + if (error instanceof IntegrationValidationError) throw new ServiceError(400, "invalid_request", error.message); + throw error; + } + if ((await repository.records()).some(record => record.id === input.id)) throw new ServiceError(409, "conflict", "An integration with this id already exists."); + return this.modelAccountWrite({ ...input, operation: "upsert" }); + } + async deleteModelAccount(id: string, input: { expectedRevision: number; idempotencyKey: string }) { + await this.authorizeOwnerSession("integrations:write"); + const repository = this.encryptedIntegrations(); + const existing = (await repository.records("model")).find(record => record.id === id); + if (!existing) { + try { + const replay = await repository.replayDelete(id, "model", input.idempotencyKey); + if (replay) return { id, deleted: true as const, ...replay }; + } catch (error) { + if (error instanceof IntegrationIdempotencyConflictError) throw new ServiceError(409, "conflict", "The idempotency key was already used for a different model account change."); + if (error instanceof IntegrationValidationError) throw new ServiceError(400, "invalid_request", error.message); + throw error; + } + throw new ServiceError(404, "not_found", "Model account not found."); + } + const result = await this.modelAccountWrite({ id, name: existing.name, description: existing.description, ...input, operation: "delete" }); + return { id, deleted: true as const, ...result }; } async removeTailIntegration(id: string) { await this.authorize("integrations:write"); if (!await this.connections().delete(`cloudflare-tail:${id}`)) throw new ServiceError(404, "not_found", "Integration not found"); return { id, deleted: true }; } async linearProjects() { await this.authorize("integrations:read"); return publicValue(z.array(namedOption), await this.providers().linearProjects()); } diff --git a/packages/api/src/postgres/encrypted-integration-repository.ts b/packages/api/src/postgres/encrypted-integration-repository.ts index dcbb3ddc..04bf9dad 100644 --- a/packages/api/src/postgres/encrypted-integration-repository.ts +++ b/packages/api/src/postgres/encrypted-integration-repository.ts @@ -77,6 +77,33 @@ export class EncryptedIntegrationRepository { return rows.length ? safeRevision(rows[0]!.revision) : 0; } + /** Recognize an exact retry before collection-level create checks run. */ + async replay(input: IntegrationWrite): Promise<{ revision: number; duplicate: true } | null> { + validate(input); + const { rows } = await this.database.pool.query("SELECT revision,operation,record_id,kind,name,origin,description,encrypted_payload FROM app.integration_changes WHERE tenant_id=$1 AND idempotency_key=$2", [this.tenantId, input.idempotencyKey]); + const row = rows[0]; + if (!row) return null; + const sameCiphertext = row.encrypted_payload === null + ? input.operation === "delete" && input.encryptedPayload === undefined + : input.operation === "upsert" && typeof input.encryptedPayload === "string" && await decrypt(row.encrypted_payload, this.serverEncryptionKey) === input.encryptedPayload; + if (row.record_id !== input.id || row.kind !== input.kind || row.name !== input.name || row.origin !== input.origin + || row.description !== input.description || row.operation !== input.operation || !sameCiphertext) { + throw new IntegrationIdempotencyConflictError(); + } + return { revision: safeRevision(row.revision), duplicate: true }; + } + + /** Recognize a delete retry after its active record has already become a tombstone. */ + async replayDelete(id: string, kind: EncryptedIntegrationKind, idempotencyKey: string): Promise<{ revision: number; duplicate: true } | null> { + const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (!uuid.test(id) || !uuid.test(idempotencyKey)) throw new IntegrationValidationError("id and idempotencyKey must be UUIDs"); + const { rows } = await this.database.pool.query("SELECT revision,operation,record_id,kind,name,origin,description,encrypted_payload FROM app.integration_changes WHERE tenant_id=$1 AND idempotency_key=$2", [this.tenantId, idempotencyKey]); + const row = rows[0]; + if (!row) return null; + if (row.record_id !== id || row.kind !== kind || row.operation !== "delete" || row.encrypted_payload !== null) throw new IntegrationIdempotencyConflictError(); + return { revision: safeRevision(row.revision), duplicate: true }; + } + async write(input: IntegrationWrite): Promise<{ revision: number; duplicate: boolean }> { validate(input); const encryptedPayload = input.operation === "upsert" ? await encrypt(input.encryptedPayload!, this.serverEncryptionKey) : null; diff --git a/packages/api/test/api-contract.test.ts b/packages/api/test/api-contract.test.ts index bebc6035..abfbcb01 100644 --- a/packages/api/test/api-contract.test.ts +++ b/packages/api/test/api-contract.test.ts @@ -75,9 +75,18 @@ describe("versioned API contract", () => { }); it("lists only safe model account metadata for the owner browser", () => { const route = API_ROUTES.find(item => item.method === "GET" && item.path === "/api/v1/model-accounts"); + const create = API_ROUTES.find(item => item.method === "POST" && item.path === "/api/v1/model-accounts"); + const remove = API_ROUTES.find(item => item.method === "DELETE" && item.path === "/api/v1/model-accounts/{accountId}"); expect(route?.scope).toBe("integrations:read"); expect(route?.ownerSession).toBe(true); expect(route?.documentation.responses["200"].content["application/json"].schema.properties.accounts.items.properties.encryptedPayload).toBeUndefined(); + expect(create?.scope).toBe("integrations:write"); expect(create?.ownerSession).toBe(true); + expect(create?.body?.safeParse({ id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", expectedRevision: 0, idempotencyKey: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", name: "Account", description: null, encryptedPayload: "cipher" }).success).toBe(true); + expect(create?.body?.safeParse({ id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", expectedRevision: 0, idempotencyKey: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", name: "Account", description: null, encryptedPayload: "cipher", unlockKey: "must-not-be-accepted" }).success).toBe(false); + expect(remove?.scope).toBe("integrations:write"); expect(remove?.ownerSession).toBe(true); + expect(remove?.parameters.shape.accountId).toBeDefined(); + expect(openapi.paths["/api/v1/model-accounts"].post).toBeDefined(); + expect(openapi.paths["/api/v1/model-accounts/{accountId}"].delete).toBeDefined(); expect(openapi.paths["/api/v1/model-accounts"].get).toBeDefined(); }); it("accepts code jobs without agent prompts or models and rejects missing scripts", () => { diff --git a/packages/api/test/encrypted-integration-api.postgres.test.ts b/packages/api/test/encrypted-integration-api.postgres.test.ts index 7aa42ad5..4a5622bd 100644 --- a/packages/api/test/encrypted-integration-api.postgres.test.ts +++ b/packages/api/test/encrypted-integration-api.postgres.test.ts @@ -76,6 +76,43 @@ describe.skipIf(!databaseUrl)("browser-only encrypted bearer integration API (Po } finally { await f.cleanup(); } }); + it("creates and deletes model accounts through owner-only revision-checked writes", async () => { + const f = await loadingFixture(databaseUrl!); + try { + const path = f.env.APP_ORIGIN + "/api/v1/model-accounts", id = crypto.randomUUID(); + const request = (url: string, method = "GET", body?: unknown, cookie = f.cookie) => worker.fetch(new Request(url, { + method, headers: { Cookie: cookie, Origin: f.env.APP_ORIGIN, "Content-Type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }), f.env, { waitUntil() {} } as any); + const ciphertext = await userEncrypt("{}"), createBody = { + id, expectedRevision: 0, idempotencyKey: crypto.randomUUID(), name: "Work account", + description: "Personal subscription", encryptedPayload: ciphertext, + }; + const created = await request(path, "POST", createBody); + expect(created.status).toBe(201); + expect(await created.json()).toEqual({ revision: 1, duplicate: false }); + expect(await (await request(path, "POST", createBody)).json()).toEqual({ revision: 1, duplicate: true }); + + const listed = await request(path); + expect(listed.status).toBe(200); + expect(await listed.json()).toEqual({ revision: 1, accounts: [{ id, name: "Work account", description: "Personal subscription", revision: 1 }] }); + + const stale = await request(path + `/${id}`, "DELETE", { expectedRevision: 0, idempotencyKey: crypto.randomUUID() }); + expect(stale.status).toBe(409); + expect((await stale.json() as any).error.code).toBe("stale_revision"); + + const deleteBody = { expectedRevision: 1, idempotencyKey: crypto.randomUUID() }; + const deleted = await request(path + `/${id}`, "DELETE", deleteBody); + expect(deleted.status).toBe(200); + expect(await deleted.json()).toEqual({ id, deleted: true, revision: 2, duplicate: false }); + expect(await (await request(path + `/${id}`, "DELETE", deleteBody)).json()).toEqual({ id, deleted: true, revision: 2, duplicate: true }); + expect(await (await request(path)).json()).toEqual({ revision: 2, accounts: [] }); + + const agent = await protectedApiFetch(new Request(path, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ ...createBody, id: crypto.randomUUID() }) }), f.env, { tenantId: f.tenantId, userId: f.userId, sessionVersion: 0, scopes: ["integrations:write"], authMethod: "access_token" } as any, {} as any); + expect(agent.status).toBe(403); + } finally { await f.cleanup(); } + }); + it("requires a browser session and isolates records across tenants", async () => { const f = await loadingFixture(databaseUrl!); try { diff --git a/packages/app/src/features/settings/model-accounts.tsx b/packages/app/src/features/settings/model-accounts.tsx index b94e0806..3283878f 100644 --- a/packages/app/src/features/settings/model-accounts.tsx +++ b/packages/app/src/features/settings/model-accounts.tsx @@ -1,7 +1,9 @@ import { useEffect, useState } from "react"; -import { useQuery } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { api } from "factorize-api-client"; -import { Button, Card } from "../../shared/ui"; +import { Button, Card, Input, Label } from "../../shared/ui"; +import { encryptModelAccountCacheRecord, verifyUnlockKey } from "../../lib/unlock-crypto"; +import { getUnlockVerification } from "../../lib/unlock-key"; import { messageOf } from "../auth/session"; type DeviceState = { state: string; verificationUrl?: string; userCode?: string }; @@ -9,8 +11,9 @@ type ModelAccount = { id: string; name: string; description: string | null; revi const accountsQuery = { queryKey: ["settings", "model-accounts"], queryFn: async ({ signal }: { signal: AbortSignal }) => { const { data, error } = await api.GET("/api/v1/model-accounts", { signal }); if (!data || error) throw new Error(messageOf(error)); - return data.accounts; + return data; } }; +const verificationQuery = { queryKey: ["unlock-key-verification"], queryFn: ({ signal }: { signal: AbortSignal }) => getUnlockVerification(signal), retry: false as const }; async function proxyAction(accountId: string, action: "authorize" | "status" | "cancel"): Promise { const capability = await api.POST("/api/v1/model-accounts/{accountId}/browser-capability", { @@ -33,10 +36,53 @@ async function proxyAction(accountId: string, action: "authorize" | "status" | " } export function ModelAccounts() { - const accounts = useQuery(accountsQuery); + const cache = useQueryClient(), accounts = useQuery(accountsQuery), verification = useQuery(verificationQuery); const [active, setActive] = useState(null); const [states, setStates] = useState>({}); const [error, setError] = useState(""); + const [name, setName] = useState(""), [description, setDescription] = useState(""), [key, setKey] = useState(""); + + const create = useMutation({ retry: false, mutationFn: async () => { + const cleanName = name.trim(), cleanDescription = description.trim(); + if (!cleanName || cleanName.length > 120) throw new Error("Enter a name up to 120 characters."); + if (cleanDescription.length > 500) throw new Error("Description must be at most 500 characters."); + const setup = verification.data?.verification; + if (!setup || !verification.data) throw new Error("Set up an unlock key before adding a model account."); + if (!accounts.data) throw new Error("Model accounts are still loading. Try again shortly."); + try { await verifyUnlockKey(key, verification.data.ownerId, setup.generationId, JSON.parse(setup.encryptedVerification)); } + catch { throw new Error("Unlock key could not be verified for this account and generation."); } + const id = crypto.randomUUID(), expectedRevision = accounts.data.revision; + const encryptedPayload = await encryptModelAccountCacheRecord(new TextEncoder().encode("{}"), key, { + ownerId: verification.data.ownerId, recordId: id, recordType: "model", destination: "factorize", + keyGeneration: setup.generationId, revision: expectedRevision + 1, + }); + const { data, error } = await api.POST("/api/v1/model-accounts", { body: { + id, expectedRevision, idempotencyKey: crypto.randomUUID(), name: cleanName, + description: cleanDescription || null, encryptedPayload, + } }); + if (!data || error) throw new Error(messageOf(error)); + return data; + }, onSuccess: async () => { + setName(""); setDescription(""); setKey(""); setError(""); + await cache.invalidateQueries({ queryKey: accountsQuery.queryKey }); + }, onError: async reason => { + setKey(""); setError(reason.message); + if (reason.message.includes("changed") || reason.message.includes("revision")) await cache.invalidateQueries({ queryKey: accountsQuery.queryKey }); + } }); + + const remove = useMutation({ retry: false, mutationFn: async (account: ModelAccount) => { + if (!accounts.data) throw new Error("Model accounts are still loading. Try again shortly."); + const { data, error } = await api.DELETE("/api/v1/model-accounts/{accountId}", { + params: { path: { accountId: account.id } }, + body: { expectedRevision: accounts.data.revision, idempotencyKey: crypto.randomUUID() }, + }); + if (!data || error) throw new Error(messageOf(error)); + return data; + }, onSuccess: async () => { + setError(""); await cache.invalidateQueries({ queryKey: accountsQuery.queryKey }); + }, onError: async reason => { + setError(reason.message); await cache.invalidateQueries({ queryKey: accountsQuery.queryKey }); + } }); useEffect(() => { if (!active) return; @@ -71,8 +117,8 @@ export function ModelAccounts() { {accounts.isPending &&

Loading model accounts…

} {accounts.error &&

{accounts.error.message}

} {error &&

{error}

} - {!accounts.isPending && accounts.data?.length === 0 &&

No model accounts are configured yet.

} - {accounts.data?.map(account => { + {!accounts.isPending && accounts.data?.accounts.length === 0 &&

No model accounts are configured yet.

} + {accounts.data?.accounts.map(account => { const state = states[account.id] ?? { state: "idle" }; return

{account.name}

{account.description &&

{account.description}

} @@ -83,8 +129,18 @@ export function ModelAccounts() { {state.state === "failed" &&

Device authorization failed. You can try again.

} {state.state === "canceled" &&

Device authorization canceled.

}
- {state.state === "pending" ? : } + {state.state === "pending" ? : } + ; })} +

Add a model account

+ {!verification.data?.verification && verification.data &&

Set up your unlock key above before adding a model account.

} +
{ event.preventDefault(); setError(""); create.mutate(); }}> + + + + +
+
; } diff --git a/packages/app/src/lib/unlock-crypto.ts b/packages/app/src/lib/unlock-crypto.ts index 900aabcf..7648d973 100644 --- a/packages/app/src/lib/unlock-crypto.ts +++ b/packages/app/src/lib/unlock-crypto.ts @@ -91,6 +91,22 @@ export async function encryptRecord(plaintext: Uint8Array, key: string, binding: return { version: VERSION, algorithm: "A256GCM", nonce: encodeBase64Url(nonce), ciphertext: encodeBase64Url(new Uint8Array(ciphertext)) }; } +/** Encrypt an initial model-account cache with the same key-generation-bound AAD used by the proxy. */ +export async function encryptModelAccountCacheRecord(plaintext: Uint8Array, key: string, binding: RecordBinding & { keyGeneration: string }): Promise { + if (!(plaintext instanceof Uint8Array) || !binding || typeof binding.ownerId !== "string" || !binding.ownerId + || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(binding.recordId) + || typeof binding.keyGeneration !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(binding.keyGeneration) + || !Number.isSafeInteger(binding.revision) || binding.revision < 1) { + throw new TypeError("Malformed model-account encryption binding"); + } + const aad = encoder.encode(JSON.stringify(["factorize-record-v1", binding.ownerId, binding.recordId, "model", "factorize", binding.keyGeneration, binding.revision])); + const nonce = crypto.getRandomValues(new Uint8Array(NONCE_BYTES)); + const aesKey = await deriveAesKey(key, ["encrypt"]); + const ciphertext = await crypto.subtle.encrypt({ name: "AES-GCM", iv: toArrayBuffer(nonce), additionalData: toArrayBuffer(aad), tagLength: 128 }, aesKey, toArrayBuffer(plaintext)); + const encode = (bytes: Uint8Array) => btoa(Array.from(bytes, byte => String.fromCharCode(byte)).join("")); + return `${encode(nonce)}.${encode(new Uint8Array(ciphertext))}`; +} + /** Decrypt a supported envelope; malformed metadata, bindings and authentication fail closed. */ export async function decryptRecord(envelope: unknown, key: string, binding: RecordBinding): Promise { if (!envelope || typeof envelope !== "object" || Array.isArray(envelope)) throw new TypeError("Malformed encrypted envelope"); diff --git a/packages/app/test/settings.browser.test.mjs b/packages/app/test/settings.browser.test.mjs index 81903180..0d5ca6f0 100644 --- a/packages/app/test/settings.browser.test.mjs +++ b/packages/app/test/settings.browser.test.mjs @@ -24,7 +24,7 @@ test("model account settings issue short-lived capabilities and display the boun await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); await page.route("**/api/v1/integrations", route => route.fulfill({ json: { linear: null, clickup: null, exe: null, exeConnections: [], ampConnections: [], cloudflareTail: { count: 0, installations: [] } } })); await page.route("**/api/v1/model-accounts", async route => { - if (route.request().method() === "GET") return route.fulfill({ json: { accounts: [{ id: accountId, name: "Work account", description: null, revision: 0 }] } }); + if (route.request().method() === "GET") return route.fulfill({ json: { revision: 1, accounts: [{ id: accountId, name: "Work account", description: null, revision: 1 }] } }); const action = route.request().postDataJSON().action; return route.fulfill({ json: { endpoint: `${proxyOrigin}/model-accounts/${accountId}/${action}`, capability: `one-use-${action}` } }); }); @@ -51,6 +51,70 @@ test("model account settings issue short-lived capabilities and display the boun assert.deepEqual(proxyCalls.map(call => call.body), ["{}", "{}"]); await context.close(); }); +test("model account create/delete encrypts initial state and uses the current collection revision", async () => { + const { context, page } = await setup(); + await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); + await page.route("**/api/v1/integrations", route => route.fulfill({ json: { linear: null, clickup: null, exe: null, exeConnections: [], ampConnections: [], cloudflareTail: { count: 0, installations: [] } } })); + let verification = null, revision = 0, account = null, createBody = null, deleteBodies = [], staleOnce = true; + await page.route("**/api/v1/unlock-key/verification", async route => { + if (route.request().method() === "GET") return route.fulfill({ json: { ownerId: "owner", verification } }); + const body = route.request().postDataJSON(); verification = { generationId: body.generationId, encryptedVerification: body.encryptedVerification }; + return route.fulfill({ json: { generationId: body.generationId } }); + }); + await page.route("**/api/v1/model-accounts", async route => { + if (route.request().method() === "GET") return route.fulfill({ json: { revision, accounts: account ? [account] : [] } }); + createBody = route.request().postDataJSON(); + assert.deepEqual(Object.keys(createBody).sort(), ["description", "encryptedPayload", "expectedRevision", "id", "idempotencyKey", "name"]); + assert.equal(createBody.expectedRevision, revision); + assert.equal(createBody.name, "Work account"); + assert.equal(createBody.encryptedPayload.includes("{}"), false); + assert.equal(createBody.encryptedPayload.includes("fzk1."), false); + assert.match(createBody.encryptedPayload, /^[A-Za-z0-9+/]+=*\.[A-Za-z0-9+/]+=*$/u); + revision++; account = { id: createBody.id, name: createBody.name, description: createBody.description, revision }; + return route.fulfill({ status: 201, json: { revision, duplicate: false } }); + }); + await page.route("**/api/v1/model-accounts/*", async route => { + assert.equal(route.request().method(), "DELETE"); + const body = route.request().postDataJSON(); deleteBodies.push(body); + assert.deepEqual(Object.keys(body).sort(), ["expectedRevision", "idempotencyKey"]); + if (staleOnce) { + staleOnce = false; revision++; + return route.fulfill({ status: 409, json: { error: { code: "stale_revision", message: "The model account collection changed." } } }); + } + assert.equal(body.expectedRevision, revision); + revision++; account = null; + return route.fulfill({ json: { id: createBody.id, deleted: true, revision, duplicate: false } }); + }); + + await page.goto(origin + "/settings/integrations"); + await page.getByRole("button", { name: "Generate unlock key" }).click(); + const unlockKey = await page.getByLabel("New unlock key").inputValue(); + await page.getByRole("checkbox", { name: "I saved this key somewhere safe." }).check(); + await page.getByRole("button", { name: "I saved my key" }).click(); + await page.getByText("Unlock key verification is set up.", { exact: false }).waitFor(); + + const models = page.getByRole("region", { name: "ChatGPT model accounts" }); + await models.getByLabel("Name").fill("Work account"); + await models.getByLabel("Description (optional)").fill("Personal account"); + await models.getByLabel("Unlock key").fill(unlockKey); + await models.getByRole("button", { name: "Add model account" }).click(); + await models.getByText("Work account", { exact: true }).waitFor(); + assert.equal(createBody.expectedRevision, 0); + assert.equal(createBody.description, "Personal account"); + + page.once("dialog", dialog => dialog.accept()); + const refreshAfterConflict = page.waitForResponse(response => response.url().includes("/api/v1/model-accounts") && response.request().method() === "GET" && response.status() === 200); + await models.getByRole("button", { name: "Delete account" }).click(); + await models.getByRole("alert").filter({ hasText: "collection changed" }).waitFor(); + await refreshAfterConflict; + page.once("dialog", dialog => dialog.accept()); + await models.getByRole("button", { name: "Delete account" }).click(); + await models.getByText("No model accounts are configured yet.", { exact: true }).waitFor(); + assert.equal(deleteBodies.length, 2); + assert.equal(deleteBodies[0].expectedRevision, 1); + assert.equal(deleteBodies[1].expectedRevision, 2); + await context.close(); +}); test("unlock-key setup requires acknowledgement, stores ciphertext only, verifies entered keys and survives refresh without a saved key", async () => { const { context, page } = await setup(); await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); diff --git a/packages/app/test/unlock-crypto.test.mjs b/packages/app/test/unlock-crypto.test.mjs index 2ac54691..ab04eb4a 100644 --- a/packages/app/test/unlock-crypto.test.mjs +++ b/packages/app/test/unlock-crypto.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { createUnlockVerification, decryptRecord, encryptAfterUnlockVerification, encryptRecord, generateUnlockKey, verifyUnlockKey } from "../src/lib/unlock-crypto.ts"; +import { createUnlockVerification, decryptRecord, encryptAfterUnlockVerification, encryptModelAccountCacheRecord, encryptRecord, generateUnlockKey, verifyUnlockKey } from "../src/lib/unlock-crypto.ts"; const key = "fzk1.AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8"; const binding = { ownerId: "owner-vector", recordId: "record-vector", recordType: "integration", destination: "https://api.example.test", revision: 7 }; @@ -25,6 +25,25 @@ test("fresh nonces make repeated encryptions differ", async () => { assert.notEqual(first.ciphertext, second.ciphertext); }); +test("initial model-account cache uses the proxy key-generation and record-revision binding", async () => { + const unlockKey = generateUnlockKey(), modelBinding = { + ownerId: "owner-a", recordId: "11111111-1111-4111-8111-111111111111", recordType: "model", + destination: "factorize", keyGeneration: "22222222-2222-4222-8222-222222222222", revision: 4, + }; + const encrypted = await encryptModelAccountCacheRecord(encoder.encode("{}"), unlockKey, modelBinding); + const [nonce, ciphertext] = encrypted.split(".").map(value => Uint8Array.from(atob(value), char => char.charCodeAt(0))); + const raw = Uint8Array.from(atob(unlockKey.slice(5).replaceAll("-", "+").replaceAll("_", "/") + "="), char => char.charCodeAt(0)); + try { + const material = await crypto.subtle.importKey("raw", raw, "HKDF", false, ["deriveKey"]); + const aes = await crypto.subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt: encoder.encode("factorize.unlock-key.v1"), info: encoder.encode("factorize.record.aes-256-gcm.v1") }, material, { name: "AES-GCM", length: 256 }, false, ["decrypt"]); + const aad = encoder.encode(JSON.stringify(["factorize-record-v1", modelBinding.ownerId, modelBinding.recordId, "model", "factorize", modelBinding.keyGeneration, modelBinding.revision])); + const plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv: nonce, additionalData: aad, tagLength: 128 }, aes, ciphertext); + assert.equal(new TextDecoder().decode(plain), "{}"); + const staleAad = encoder.encode(JSON.stringify(["factorize-record-v1", modelBinding.ownerId, modelBinding.recordId, "model", "factorize", modelBinding.keyGeneration, modelBinding.revision + 1])); + await assert.rejects(crypto.subtle.decrypt({ name: "AES-GCM", iv: nonce, additionalData: staleAad, tagLength: 128 }, aes, ciphertext)); + } finally { raw.fill(0); } +}); + test("wrong key and every authenticated binding change fail closed", async () => { await assert.rejects(decryptRecord(vector, "fzk1.AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE", binding)); for (const field of Object.keys(binding)) { diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index 6089764f..32c63f8c 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -1810,19 +1810,23 @@ paths: get: summary: List model accounts available to the owner description: |- - Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned. + Requires an interactive owner browser session. Returns account metadata and the collection revision only; encrypted account caches and credentials are never returned. Requires the integrations:read scope. An interactive owner session is required; bearer tokens cannot call this operation. responses: "200": - description: Model account metadata + description: Model account metadata and collection revision content: application/json: schema: type: object properties: + revision: + type: integer + minimum: 0 + maximum: 9007199254740991 accounts: type: array items: @@ -1849,6 +1853,7 @@ paths: - revision additionalProperties: false required: + - revision - accounts additionalProperties: false "400": @@ -1875,6 +1880,187 @@ paths: - cookieAuth: [] x-required-scope: integrations:read x-owner-session-required: true + post: + summary: Create a model account + description: |- + Requires an interactive owner browser session and the current collection revision. Accepts only user-encrypted initial account state; credentials are established directly with the credential proxy. + + Requires the integrations:write scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "201": + description: Created model account + content: + application/json: + schema: + type: object + properties: + revision: + type: integer + minimum: 0 + maximum: 9007199254740991 + duplicate: + type: boolean + required: + - revision + - duplicate + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + "415": + $ref: "#/components/responses/Error415" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: post_api_v1_model_accounts + tags: + - model-accounts + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: integrations:write + x-owner-session-required: true + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + id: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + expectedRevision: + type: integer + minimum: 0 + maximum: 9007199254740991 + idempotencyKey: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + name: + type: string + minLength: 1 + maxLength: 120 + description: + anyOf: + - type: string + maxLength: 500 + - type: "null" + encryptedPayload: + type: string + minLength: 1 + maxLength: 65536 + required: + - id + - expectedRevision + - idempotencyKey + - name + - description + - encryptedPayload + additionalProperties: false + /api/v1/model-accounts/{accountId}: + delete: + summary: Delete a model account + description: |- + Requires an interactive owner browser session and the current collection revision. Deletion is recorded as a tombstone so future snapshots and unlock reconciliation cannot restore the account. + + Requires the integrations:write scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "200": + description: Deleted model account + content: + application/json: + schema: + type: object + properties: + id: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + deleted: + type: boolean + const: true + revision: + type: integer + minimum: 0 + maximum: 9007199254740991 + duplicate: + type: boolean + required: + - id + - deleted + - revision + - duplicate + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + "415": + $ref: "#/components/responses/Error415" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: delete_api_v1_model_accounts_accountId + tags: + - model-accounts + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: integrations:write + x-owner-session-required: true + parameters: + - name: accountId + in: path + required: true + schema: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + expectedRevision: + type: integer + minimum: 0 + maximum: 9007199254740991 + idempotencyKey: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + required: + - expectedRevision + - idempotencyKey + additionalProperties: false /api/v1/model-accounts/{accountId}/browser-capability: post: summary: Issue a short-lived browser capability for a paired model account From 4e4a5c0a7e8bc2b1a477a4206c6267ac9b005ec8 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:48:17 +0000 Subject: [PATCH 041/102] GEN-2824 recover stale model account cache writes --- .../credential-proxy/src/control-client.mjs | 10 ++- .../src/model-account-lifecycle.mjs | 86 +++++++++++++------ .../test/model-account-lifecycle.test.mjs | 42 ++++++++- 3 files changed, 109 insertions(+), 29 deletions(-) diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 9b4656c9..0a4c889e 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -39,7 +39,15 @@ async function proxyControlRequest({ controlOrigin, token, path, body, fetchImpl method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(10_000), redirect: "error", }); - if (!response.ok) throw new Error("proxy control request rejected"); + if (!response.ok) { + let code; + try { code = (await response.clone().json())?.error; } catch { /* Keep error handling bounded to the HTTP status. */ } + const error = new Error("proxy control request rejected"); + error.name = "ProxyControlRequestError"; + error.status = response.status; + if (typeof code === "string") error.code = code; + throw error; + } return response.json(); } diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index fc6a1308..cb5af18e 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -25,6 +25,7 @@ export function createModelAccountLifecycle({ persistEncrypted, onUnsaved = () => {}, retryDelaysMs = [100, 500, 2_000], + staleRevisionRetries = 3, sleep = ms => new Promise(resolveSleep => setTimeout(resolveSleep, ms)), removeCache = rm, readCache = readFile, @@ -37,7 +38,8 @@ export function createModelAccountLifecycle({ if (!authHelper || typeof authHelper.invoke !== "function" || typeof getUnlockKey !== "function" || typeof getAccount !== "function" || typeof encryptCache !== "function" || typeof persistEncrypted !== "function" || !Array.isArray(retryDelaysMs) || retryDelaysMs.length > 5 - || retryDelaysMs.some(ms => !Number.isSafeInteger(ms) || ms < 0 || ms > 30_000)) { + || retryDelaysMs.some(ms => !Number.isSafeInteger(ms) || ms < 0 || ms > 30_000) + || !Number.isSafeInteger(staleRevisionRetries) || staleRevisionRetries < 0 || staleRevisionRetries > 5) { throw new TypeError("model account lifecycle configuration is invalid"); } if ((getEncryptedCache === undefined) !== (decryptCache === undefined) @@ -59,7 +61,7 @@ export function createModelAccountLifecycle({ } async function persist(id, generation) { - const account = await getAccount(id); + let account = await getAccount(id); if (!account || account.id !== id || account.kind !== "model" || !Number.isSafeInteger(account.revision) || typeof account.ownerId !== "string" || !account.ownerId || typeof account.keyGeneration !== "string" || !account.keyGeneration) { throw new Error("model account persistence binding is unavailable"); @@ -68,36 +70,66 @@ export function createModelAccountLifecycle({ if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const plain = await readCache(cachePath(accountHomeRoot, id)); if (!Buffer.isBuffer(plain) || plain.byteLength === 0 || plain.byteLength > 65_536) throw new Error("Codex token cache is missing or exceeds its limit"); - let encrypted; + let staleRetries = 0; + let attempts = 0; try { - encrypted = await encryptCache(new Uint8Array(plain), unlockKey, { - ownerId: account.ownerId, - recordId: id, - recordType: "model", - destination: "factorize", - revision: account.revision + 1, - keyGeneration: account.keyGeneration, - }); - } finally { - plain.fill(0); - } - if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); - const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted, idempotencyKey: randomUUID() }; - for (let attempt = 0; ; attempt++) { - if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); - try { - await persistEncrypted(request); - unsaved.delete(id); - onUnsaved({ accountId: id, unsaved: false }); - return { state: "saved" }; - } catch (error) { - if (attempt >= retryDelaysMs.length) { + for (;;) { + let encrypted; + try { + encrypted = await encryptCache(new Uint8Array(plain), unlockKey, { + ownerId: account.ownerId, + recordId: id, + recordType: "model", + destination: "factorize", + revision: account.revision + 1, + keyGeneration: account.keyGeneration, + }); + } catch (error) { + throw error; + } + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted, idempotencyKey: randomUUID() }; + let retryWithLatestRevision = false; + for (let attempt = 0; ; attempt++) { + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + attempts++; + try { + await persistEncrypted(request); + unsaved.delete(id); + onUnsaved({ accountId: id, unsaved: false }); + return { state: "saved" }; + } catch (error) { + if (error?.code === "stale_revision") { + if (staleRetries >= staleRevisionRetries) break; + let latest; + try { latest = await getAccount(id); } + catch { throw new Error("model account changed before token persistence"); } + if (!latest || latest.id !== id || latest.kind !== "model" || !Number.isSafeInteger(latest.revision) + || latest.revision <= account.revision || latest.ownerId !== account.ownerId + || latest.keyGeneration !== account.keyGeneration) { + throw new Error("model account changed before token persistence"); + } + account = latest; + staleRetries++; + retryWithLatestRevision = true; + break; + } + if (attempt >= retryDelaysMs.length) { + unsaved.add(id); + onUnsaved({ accountId: id, unsaved: true }); + return { state: "unsaved", attempts }; + } + await sleep(retryDelaysMs[attempt]); + } + } + if (!retryWithLatestRevision) { unsaved.add(id); onUnsaved({ accountId: id, unsaved: true }); - return { state: "unsaved", attempts: attempt + 1 }; + return { state: "unsaved", attempts }; } - await sleep(retryDelaysMs[attempt]); } + } finally { + plain.fill(0); } } diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 37ab14e4..2028d069 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -18,7 +18,7 @@ async function fixture(t, overrides = {}) { const helper = { invoke: async request => { methods.push(request.method); if (request.method === "device-login") return { state: "pending", completion: overrides.completion ?? Promise.resolve({ state: "authorized" }) }; - if (request.method === "refresh") { calls.push("refresh"); return { state: "refreshed" }; } + if (request.method === "refresh") { calls.push("refresh"); return overrides.refreshResult ?? { state: "refreshed" }; } if (request.method === "account-status") return { state: "authenticated" }; return { state: request.method === "cancel" ? "canceled" : "logged-out" }; } }; @@ -67,6 +67,12 @@ test("denied or expired device authorization does not persist token state", asyn assert.equal(f.saves, 0); }); +test("expired refresh explicitly requires reauthentication without saving stale tokens", async t => { + const f = await fixture(t, { refreshResult: { state: "reauthentication-required" } }); + assert.deepEqual(await f.lifecycle.refresh(accountId), { state: "reauthentication-required" }); + assert.equal(f.saves, 0); +}); + test("refresh keeps the working local cache and reports bounded encrypted-save failures", async t => { const f = await fixture(t, { retryDelaysMs: [0, 0], persistEncrypted: async () => { throw new Error("fixture outage"); } }); assert.deepEqual(await f.lifecycle.refresh(accountId), { state: "unsaved", attempts: 3 }); @@ -93,6 +99,28 @@ test("refresh requests for one account serialize and use the latest revision", a assert.equal(f.saves, 2); }); +test("refresh re-encrypts against a newer account revision after a concurrent edit", async t => { + let revision = 4; + const f = await fixture(t, { + getAccount: () => ({ ...account, revision }), + persistEncrypted: async request => { + if (request.expectedRevision !== revision || f.saves === 1) { + revision++; + const error = new Error("stale model account revision"); + error.code = "stale_revision"; + throw error; + } + revision++; + }, + }); + assert.deepEqual(await f.lifecycle.refresh(accountId), { state: "saved" }); + const writes = f.events.filter(event => event.type === "persist").map(event => event.request); + assert.deepEqual(writes.map(request => request.expectedRevision), [4, 5]); + assert.notEqual(writes[0].idempotencyKey, writes[1].idempotencyKey); + const encryptions = f.events.filter(event => event.type === "encrypt"); + assert.deepEqual(encryptions.map(event => event.binding.revision), [5, 6]); +}); + test("deletion invalidates a retry and removes the account cache", async t => { let release; const waiting = new Promise(resolve => { release = resolve; }); @@ -112,6 +140,18 @@ test("deletion invalidates a retry and removes the account cache", async t => { assert.ok(f.methods.includes("logout")); }); +test("deletion racing pending device login prevents a late authorization from restoring its cache", async t => { + let complete; + const f = await fixture(t, { completion: new Promise(resolve => { complete = resolve; }) }); + const login = await f.lifecycle.deviceLogin(accountId); + const deleting = f.lifecycle.deleteAccount(accountId); + complete({ state: "authorized" }); + assert.deepEqual(await login.completion, { state: "deleted" }); + assert.deepEqual(await deleting, { state: "deleted" }); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.equal(f.saves, 0); +}); + test("unlock reconciliation restores the encrypted Codex cache before checking account status", async t => { const persisted = new TextEncoder().encode(JSON.stringify({ tokens: { access_token: "restored-access", refresh_token: "restored-refresh" } })); const encryptedPayload = `cache.${Buffer.from(persisted).toString("base64")}`; From e5a6a97c9d71aea2b67defffe874bf15f09f6d29 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:55:44 +0000 Subject: [PATCH 042/102] GEN-2824 clear deleted model caches on unlock --- packages/credential-proxy/src/main.mjs | 9 +++- .../src/model-account-lifecycle.mjs | 49 ++++++++++++++----- .../test/model-account-lifecycle.test.mjs | 25 ++++++++++ 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 016c80a3..50f1ba2c 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -132,8 +132,13 @@ lines.on("line", async (line) => { else if (command?.type === "unlock") { runtime.unlock(command.key); if (modelAccountLifecycle && modelAccounts) { - const accounts = await modelAccounts.listAccounts(); - for (const account of accounts) await modelAccountLifecycle.reconcile(account.id); + try { + const accounts = await modelAccounts.listAccounts(); + await modelAccountLifecycle.reconcileAll(accounts); + } catch (error) { + await runtime.lock(); + throw error; + } } respondStatus(); } else if (command?.type === "lock") { diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index cb5af18e..14179ce8 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -1,4 +1,4 @@ -import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; import { randomUUID } from "node:crypto"; import { join, resolve, sep } from "node:path"; @@ -31,6 +31,7 @@ export function createModelAccountLifecycle({ readCache = readFile, writeCache = writeFile, makeDirectory = mkdir, + readDirectory = readdir, replaceFile = rename, getEncryptedCache, decryptCache, @@ -220,15 +221,8 @@ export function createModelAccountLifecycle({ return { state: "restored", revision: encrypted.revision }; } - return Object.freeze({ - deviceLogin, - cancel, - refresh, - deleteAccount, - restore, - isUnsaved: id => unsaved.has(id), - /** Invoke after encrypted snapshots are reconciled at unlock. */ - reconcile: async id => serialize(id, async () => { + async function reconcile(id) { + return serialize(id, async () => { if (!await getAccount(id)) { await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); unsaved.delete(id); @@ -240,7 +234,40 @@ export function createModelAccountLifecycle({ } const result = await authHelper.invoke({ method: "account-status", accountId: id }); return result.state === "authenticated" ? persist(id, currentGeneration(id)) : { state: "reauthentication-required" }; - }), + }); + } + + return Object.freeze({ + deviceLogin, + cancel, + refresh, + deleteAccount, + restore, + reconcile, + isUnsaved: id => unsaved.has(id), + /** Reconcile a complete server snapshot after unlock. A reset removes every + * prior model account from that snapshot, so stale local Codex homes must + * be logged out and removed before any current account is admitted. */ + reconcileAll: async accounts => { + if (!Array.isArray(accounts) || accounts.some(account => !account || !UUID.test(account.id ?? "") || account.kind !== "model") + || new Set(accounts.map(account => account.id)).size !== accounts.length) { + throw new TypeError("model account snapshot is invalid"); + } + const activeIds = new Set(accounts.map(account => account.id)); + const entries = await readDirectory(accountHomeRoot, { withFileTypes: true }).catch(error => { + if (error?.code === "ENOENT") return []; + throw error; + }); + const removed = []; + for (const entry of entries) { + if (!UUID.test(entry.name) || activeIds.has(entry.name)) continue; + await deleteAccount(entry.name); + removed.push(entry.name); + } + const states = []; + for (const account of accounts) states.push({ accountId: account.id, ...(await reconcile(account.id)) }); + return { removed, accounts: states }; + }, }); } diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 2028d069..342886ed 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -6,6 +6,7 @@ import { join } from "node:path"; import { createModelAccountLifecycle } from "../src/model-account-lifecycle.mjs"; const accountId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const resetAccountId = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; const account = { id: accountId, kind: "model", ownerId: "owner", keyGeneration: "generation", revision: 4 }; async function fixture(t, overrides = {}) { @@ -190,3 +191,27 @@ test("unlock reconciliation removes stale cache when the account key generation await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); assert.equal(f.methods.includes("account-status"), false); }); + +test("unlock snapshot reconciliation logs out and removes cached accounts deleted by reset", async t => { + const f = await fixture(t); + const staleHome = join(f.root, resetAccountId); + await mkdir(staleHome, { recursive: true }); + await writeFile(join(staleHome, "auth.json"), JSON.stringify({ tokens: { access_token: "stale-reset-token" } })); + + assert.deepEqual(await f.lifecycle.reconcileAll([account]), { + removed: [resetAccountId], + accounts: [{ accountId, state: "saved" }], + }); + await assert.rejects(readFile(join(staleHome, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); + assert.equal(f.saves, 1); +}); + +test("unlock snapshot rejects duplicate account identities before touching local state", async t => { + const f = await fixture(t); + await assert.rejects(f.lifecycle.reconcileAll([account, account]), /snapshot is invalid/); + assert.deepEqual(JSON.parse(await readFile(join(f.home, "auth.json"), "utf8")), { + tokens: { access_token: "synthetic-access", refresh_token: "synthetic-refresh" }, + }); + assert.equal(f.methods.length, 0); +}); From 29cd5488a0bff7d955a0ca1fa69806edc1dbee82 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 04:19:13 +0000 Subject: [PATCH 043/102] GEN-2824 cover model cache key rotation race --- .../test/model-account-lifecycle.test.mjs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 342886ed..8d90fa1d 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -122,6 +122,23 @@ test("refresh re-encrypts against a newer account revision after a concurrent ed assert.deepEqual(encryptions.map(event => event.binding.revision), [5, 6]); }); +test("refresh does not retry a stale cache write after the unlock-key generation rotates", async t => { + let generation = "generation"; + const f = await fixture(t, { + getAccount: () => ({ ...account, keyGeneration: generation }), + persistEncrypted: async () => { + generation = "rotated-generation"; + const error = new Error("stale model account revision"); + error.code = "stale_revision"; + throw error; + }, + }); + + await assert.rejects(f.lifecycle.refresh(accountId), /changed before token persistence/); + assert.equal(f.saves, 1); + assert.deepEqual(f.events.filter(event => event.type === "encrypt").map(event => event.binding.keyGeneration), ["generation"]); +}); + test("deletion invalidates a retry and removes the account cache", async t => { let release; const waiting = new Promise(resolve => { release = resolve; }); From 7a4fdfad6577e62b85b10de52d735b521604f404 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 04:34:34 +0000 Subject: [PATCH 044/102] GEN-2824 cover model reset refresh race --- .../test/model-account-lifecycle.test.mjs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 8d90fa1d..0438a445 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -158,6 +158,34 @@ test("deletion invalidates a retry and removes the account cache", async t => { assert.ok(f.methods.includes("logout")); }); +test("reset snapshot invalidates a refresh cache write already in flight", async t => { + let release; + let entered; + const waiting = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const f = await fixture(t, { + persistEncrypted: async () => { + entered(); + await waiting; + const error = new Error("stale model account revision"); + error.code = "stale_revision"; + throw error; + }, + }); + + const refreshing = f.lifecycle.refresh(accountId); + await started; + const resetting = f.lifecycle.reconcileAll([]); + await new Promise(resolve => setImmediate(resolve)); + release(); + + await assert.rejects(refreshing, /changed before token persistence/); + assert.deepEqual(await resetting, { removed: [accountId], accounts: [] }); + assert.equal(f.saves, 1); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); +}); + test("deletion racing pending device login prevents a late authorization from restoring its cache", async t => { let complete; const f = await fixture(t, { completion: new Promise(resolve => { complete = resolve; }) }); From 5a5f01a4f52bfe574ee51bf2f7b10067d09f0ded Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 04:47:45 +0000 Subject: [PATCH 045/102] GEN-2824 cover reset during refresh retry --- .../test/model-account-lifecycle.test.mjs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 0438a445..f188b7bb 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -186,6 +186,29 @@ test("reset snapshot invalidates a refresh cache write already in flight", async assert.ok(f.methods.includes("logout")); }); +test("deletion during refresh persistence backoff prevents a delayed retry", async t => { + let releaseBackoff; + let enteredBackoff; + const waiting = new Promise(resolve => { releaseBackoff = resolve; }); + const started = new Promise(resolve => { enteredBackoff = resolve; }); + const f = await fixture(t, { + retryDelaysMs: [1], + options: { sleep: async () => { enteredBackoff(); await waiting; } }, + persistEncrypted: async () => { throw new Error("temporary persistence failure"); }, + }); + + const refreshing = f.lifecycle.refresh(accountId); + await started; + const deleting = f.lifecycle.deleteAccount(accountId); + releaseBackoff(); + + await assert.rejects(refreshing, /changed before token persistence/); + assert.deepEqual(await deleting, { state: "deleted" }); + assert.equal(f.saves, 1); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); +}); + test("deletion racing pending device login prevents a late authorization from restoring its cache", async t => { let complete; const f = await fixture(t, { completion: new Promise(resolve => { complete = resolve; }) }); From 8960123273adc4ab31bf34268dee82ea38d89d16 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 05:09:24 +0000 Subject: [PATCH 046/102] GEN-2824 invalidate refresh retries after model rotation --- .../src/model-account-lifecycle.mjs | 8 ++++ .../test/model-account-lifecycle.test.mjs | 47 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index 14179ce8..565abde8 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -121,6 +121,14 @@ export function createModelAccountLifecycle({ return { state: "unsaved", attempts }; } await sleep(retryDelaysMs[attempt]); + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + let latest; + try { latest = await getAccount(id); } + catch { throw new Error("model account changed before token persistence"); } + if (!latest || latest.id !== id || latest.kind !== "model" || latest.ownerId !== account.ownerId + || latest.keyGeneration !== account.keyGeneration) { + throw new Error("model account changed before token persistence"); + } } } if (!retryWithLatestRevision) { diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index f188b7bb..310b4dfb 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -139,6 +139,20 @@ test("refresh does not retry a stale cache write after the unlock-key generation assert.deepEqual(f.events.filter(event => event.type === "encrypt").map(event => event.binding.keyGeneration), ["generation"]); }); +test("refresh does not retry a transient cache write after the key generation rotates during backoff", async t => { + let generation = "generation"; + const f = await fixture(t, { + retryDelaysMs: [1], + getAccount: () => ({ ...account, keyGeneration: generation }), + options: { sleep: async () => { generation = "rotated-generation"; } }, + persistEncrypted: async () => { throw new Error("temporary persistence failure"); }, + }); + + await assert.rejects(f.lifecycle.refresh(accountId), /changed before token persistence/); + assert.equal(f.saves, 1); + assert.deepEqual(f.events.filter(event => event.type === "encrypt").map(event => event.binding.keyGeneration), ["generation"]); +}); + test("deletion invalidates a retry and removes the account cache", async t => { let release; const waiting = new Promise(resolve => { release = resolve; }); @@ -209,6 +223,39 @@ test("deletion during refresh persistence backoff prevents a delayed retry", asy assert.ok(f.methods.includes("logout")); }); +test("reset during refresh persistence backoff prevents a delayed retry and clears the cache", async t => { + let releaseBackoff; + let enteredBackoff; + let releaseSnapshot; + let enteredSnapshot; + const waiting = new Promise(resolve => { releaseBackoff = resolve; }); + const started = new Promise(resolve => { enteredBackoff = resolve; }); + const snapshotWaiting = new Promise(resolve => { releaseSnapshot = resolve; }); + const snapshotStarted = new Promise(resolve => { enteredSnapshot = resolve; }); + const f = await fixture(t, { + retryDelaysMs: [1], + options: { + sleep: async () => { enteredBackoff(); await waiting; }, + readDirectory: async () => { enteredSnapshot(); await snapshotWaiting; return [{ name: accountId }]; }, + }, + persistEncrypted: async () => { throw new Error("temporary persistence failure"); }, + }); + + const refreshing = f.lifecycle.refresh(accountId); + await started; + const resetting = f.lifecycle.reconcileAll([]); + await snapshotStarted; + releaseSnapshot(); + await new Promise(resolve => setImmediate(resolve)); + releaseBackoff(); + + await assert.rejects(refreshing, /changed before token persistence/); + assert.deepEqual(await resetting, { removed: [accountId], accounts: [] }); + assert.equal(f.saves, 1); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); +}); + test("deletion racing pending device login prevents a late authorization from restoring its cache", async t => { let complete; const f = await fixture(t, { completion: new Promise(resolve => { complete = resolve; }) }); From e627df19bfd8774a7a47947da3390258e79c10c2 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 05:24:08 +0000 Subject: [PATCH 047/102] GEN-2824 define portable proxy launch gates --- packages/api/src/proxy-launch-policy.ts | 50 +++++++++++++++++++ packages/api/test/proxy-launch-policy.test.ts | 36 +++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 packages/api/src/proxy-launch-policy.ts create mode 100644 packages/api/test/proxy-launch-policy.test.ts diff --git a/packages/api/src/proxy-launch-policy.ts b/packages/api/src/proxy-launch-policy.ts new file mode 100644 index 00000000..0987c6f3 --- /dev/null +++ b/packages/api/src/proxy-launch-policy.ts @@ -0,0 +1,50 @@ +/** Inputs required to decide whether a queued execution may create a guest VM. */ +export type ProxyLaunchState = { + enabled: boolean; + running: boolean; + unlocked: boolean; + pendingChanges: boolean; +}; + +export type ProxyLaunchJob = { + executionProvider: "exe" | "fly"; + executionType: "agent" | "code"; + portableIntegrationIds?: readonly string[]; + modelAccess?: "native" | "portable"; + modelAccountId?: string; +}; + +export type ProxyLaunchDecision = + | { kind: "allow" } + | { kind: "start_locked" } + | { kind: "blocked"; reason: "proxy_disabled" | "proxy_unavailable" | "proxy_locked" | "pending_changes"; retry: "manual" }; + +/** + * Encodes the GEN-2824 launch gate. This function is deliberately side-effect + * free: the scheduler must persist a blocked decision and wait for an explicit + * retry instead of treating a later unlock as a queued launch signal. + */ +export function proxyLaunchDecision(job: ProxyLaunchJob, proxy: ProxyLaunchState): ProxyLaunchDecision { + const needsUserCredentials = Boolean(job.portableIntegrationIds?.length) + || job.modelAccess === "portable" + || Boolean(job.modelAccountId); + + // exe.dev-native integrations keep their existing launch behavior. + if (job.executionProvider === "exe" && !needsUserCredentials) return { kind: "allow" }; + + if (!proxy.enabled) return { kind: "blocked", reason: "proxy_disabled", retry: "manual" }; + + // Only code jobs with Factorize access may automatically boot a stopped + // proxy, and that boot is always locked. + if (!needsUserCredentials && job.executionType === "code") { + if (!proxy.running) return { kind: "start_locked" }; + return { kind: "allow" }; + } + + // User credential grants require an already-running proxy with edits + // applied and a user unlock. Unlocking alone does not resume this run. + if (!proxy.running) return { kind: "blocked", reason: "proxy_unavailable", retry: "manual" }; + if (proxy.pendingChanges) return { kind: "blocked", reason: "pending_changes", retry: "manual" }; + if (!proxy.unlocked) return { kind: "blocked", reason: "proxy_locked", retry: "manual" }; + return { kind: "allow" }; +} diff --git a/packages/api/test/proxy-launch-policy.test.ts b/packages/api/test/proxy-launch-policy.test.ts new file mode 100644 index 00000000..81544828 --- /dev/null +++ b/packages/api/test/proxy-launch-policy.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { proxyLaunchDecision, type ProxyLaunchJob, type ProxyLaunchState } from "../src/proxy-launch-policy"; + +const stopped: ProxyLaunchState = { enabled: true, running: false, unlocked: false, pendingChanges: false }; +const locked: ProxyLaunchState = { enabled: true, running: true, unlocked: false, pendingChanges: false }; +const unlocked: ProxyLaunchState = { enabled: true, running: true, unlocked: true, pendingChanges: false }; +const flyCode: ProxyLaunchJob = { executionProvider: "fly", executionType: "code" }; + +describe("GEN-2824 proxy launch policy", () => { + it("starts an enabled proxy locked for Fly code jobs with no user credential grants", () => { + expect(proxyLaunchDecision(flyCode, stopped)).toEqual({ kind: "start_locked" }); + }); + + it("allows code-only Factorize access while the proxy is running and locked", () => { + expect(proxyLaunchDecision(flyCode, locked)).toEqual({ kind: "allow" }); + }); + + it.each([ + { label: "portable tools", job: { ...flyCode, portableIntegrationIds: ["integration"] } }, + { label: "portable model account", job: { ...flyCode, executionType: "agent" as const, modelAccess: "portable" as const, modelAccountId: "model" } }, + ])("requires an unlocked proxy with applied updates for $label", ({ job }) => { + expect(proxyLaunchDecision(job, stopped)).toEqual({ kind: "blocked", reason: "proxy_unavailable", retry: "manual" }); + expect(proxyLaunchDecision(job, locked)).toEqual({ kind: "blocked", reason: "proxy_locked", retry: "manual" }); + expect(proxyLaunchDecision(job, { ...unlocked, pendingChanges: true })).toEqual({ kind: "blocked", reason: "pending_changes", retry: "manual" }); + expect(proxyLaunchDecision(job, unlocked)).toEqual({ kind: "allow" }); + }); + + it("never auto-starts a disabled proxy and preserves native exe.dev behavior", () => { + expect(proxyLaunchDecision(flyCode, { ...stopped, enabled: false })).toEqual({ kind: "blocked", reason: "proxy_disabled", retry: "manual" }); + expect(proxyLaunchDecision({ executionProvider: "exe", executionType: "agent" }, { ...stopped, enabled: false })).toEqual({ kind: "allow" }); + }); + + it("does not auto-start a stopped proxy for credential-dependent jobs", () => { + expect(proxyLaunchDecision({ ...flyCode, portableIntegrationIds: ["integration"] }, stopped)).toMatchObject({ kind: "blocked", retry: "manual" }); + }); +}); From e85bf4acebfac9ebdc2d28bcd92a20745204a32a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 05:50:29 +0000 Subject: [PATCH 048/102] GEN-2824 disable Fly job machine restarts --- packages/api/src/fly-machines-client.ts | 4 ++++ packages/api/test/fly-machines-client.test.ts | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/api/src/fly-machines-client.ts b/packages/api/src/fly-machines-client.ts index a7139e37..f4742050 100644 --- a/packages/api/src/fly-machines-client.ts +++ b/packages/api/src/fly-machines-client.ts @@ -136,6 +136,10 @@ export function createFlyMachinesClient(options: FlyMachinesClientOptions) { const body = { config: { ...config, + // Job Machines are supervised by Factorize. A provider restart can + // replay a script after an ambiguous guest exit, duplicating side + // effects, so execution guests must stop with their process. + restart: { policy: "no" }, metadata: { ...config.metadata, [OPERATION_METADATA_KEY]: operationId }, }, }; diff --git a/packages/api/test/fly-machines-client.test.ts b/packages/api/test/fly-machines-client.test.ts index 9b5bf80c..60ab6d26 100644 --- a/packages/api/test/fly-machines-client.test.ts +++ b/packages/api/test/fly-machines-client.test.ts @@ -42,6 +42,7 @@ describe("bounded Fly Machines client", () => { let raw = ""; for await (const chunk of req) raw += chunk; const body = JSON.parse(raw); expect(body.config.metadata.factorize_operation_id).toBe("op-1"); + expect(body.config.restart).toEqual({ policy: "no" }); return json(res, 200, machine); } if (req.method === "GET" && req.url === "/v1/apps/app-a/machines/m-1") return json(res, 200, machine); @@ -53,7 +54,7 @@ describe("bounded Fly Machines client", () => { json(res, 404, { error: "not found" }); }); const client = createFlyMachinesClient({ token, apiOrigin: origin, wait: async () => undefined }); - const created = await client.createMachine("app-a", { image: "example/image", metadata: { keep: "yes" } }, "op-1"); + const created = await client.createMachine("app-a", { image: "example/image", restart: { policy: "always" }, metadata: { keep: "yes" } }, "op-1"); expect(created.id).toBe("m-1"); expect((await client.inspectMachine("app-a", "m-1")).id).toBe("m-1"); await client.startMachine("app-a", "m-1"); From 1f2553209ede5ddb49207508a8eabe92bed74ec2 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 06:03:33 +0000 Subject: [PATCH 049/102] GEN-2824 persist proxy launch blocks and manual retry --- packages/api-client/src/schema.d.ts | 61 +++++++++++++++++++ packages/api/src/api-contract.ts | 9 +++ packages/api/src/flow-service.ts | 6 ++ packages/api/src/postgres/run-repository.ts | 29 ++++++++- packages/api/src/postgres/run-scheduler.ts | 8 ++- packages/api/src/proxy-launch-policy.ts | 11 ++++ packages/api/test/proxy-launch-policy.test.ts | 8 ++- packages/api/test/run-launch-block.test.ts | 25 ++++++++ packages/docs/openapi.yaml | 61 +++++++++++++++++++ 9 files changed, 215 insertions(+), 3 deletions(-) create mode 100644 packages/api/test/run-launch-block.test.ts diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index 1e4bd5e7..858fd9ce 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -2068,6 +2068,30 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/runs/{runId}/retry-launch": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Manually retry a blocked launch + * @description Retries only a pre-launch proxy-gated run after its prerequisites have been resolved. The run remains blocked until an interactive owner explicitly retries it; unlock or status changes do not automatically launch it. + * + * Requires the runs:cancel scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + post: operations["post_api_v1_runs_runId_retry_launch"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/runs/{runId}/kill": { parameters: { query?: never; @@ -8206,6 +8230,43 @@ export interface operations { 503: components["responses"]["Error503"]; }; }; + post_api_v1_runs_runId_retry_launch: { + parameters: { + query?: never; + header?: never; + path: { + runId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Manual retry queued */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** Format: uuid */ + runId: string; + /** @constant */ + retryRequested: true; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + /** @description Interactive owner session required */ + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + /** @description Run is not waiting for a manual launch retry */ + 409: components["responses"]["Error409"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; post_api_v1_runs_runId_kill: { parameters: { query?: never; diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index 6ea38cdc..8d646991 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -1127,6 +1127,15 @@ export const API_ROUTES: RouteContract[] = [ { summary: "Stop run", responses: { "200": { description: "Stopped" } } }, (service, { params }) => service.stopRun(params.runId), ), + route("POST", "/api/v1/runs/{runId}/retry-launch", "runs:cancel", { + summary: "Manually retry a blocked launch", + description: "Retries only a pre-launch proxy-gated run after its prerequisites have been resolved. The run remains blocked until an interactive owner explicitly retries it; unlock or status changes do not automatically launch it.", + responses: { + "200": { description: "Manual retry queued", content: { "application/json": { schema: { type: "object", required: ["runId", "retryRequested"], properties: { runId: { type: "string", format: "uuid" }, retryRequested: { type: "boolean", const: true } }, additionalProperties: false } } } }, + "403": { description: "Interactive owner session required" }, + "409": { description: "Run is not waiting for a manual launch retry" }, + }, + }, (service, { params }) => service.retryBlockedLaunch(params.runId), { ownerSession: true, parameters: z.object({ runId: z.uuid() }) }), route( "POST", "/api/v1/runs/{runId}/kill", diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index 896dfd42..332f24ed 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -322,6 +322,12 @@ export class ApiService { catch (error) { if (error instanceof TraceReplayError) throw new ServiceError(error.status, error.code, error.message); throw error; } } async stopRun(runId: string) { await this.authorize("runs:cancel"); const stopped = await new RunQueryRepository(databaseFor(this.env), this.auth.tenantId).stop(runId); if (!stopped) throw new ServiceError(409, "operation_failed", "Run is not active"); if (this.env.SCHEDULER) await this.env.SCHEDULER.get(this.env.SCHEDULER.idFromName("global")).fetch("https://scheduler/wake", { method: "POST" }); return stopped; } + async retryBlockedLaunch(runId: string) { + await this.authorizeOwnerSession("runs:cancel"); + if (!await new RunRepository(databaseFor(this.env)).retryBlockedLaunch(this.auth.tenantId, runId)) throw new ServiceError(409, "operation_failed", "Run is not waiting for a manual launch retry"); + if (this.env.SCHEDULER) await this.env.SCHEDULER.get(this.env.SCHEDULER.idFromName("global")).fetch("https://scheduler/wake", { method: "POST" }); + return { runId, retryRequested: true }; + } killRun(runId: string) { return this.stopRun(runId); } async listJobSummaries(input: JobPageQuery, selector = false) { await this.authorize("jobs:read"); try { return await new JobSummaryRepository(databaseFor(this.env), this.auth.tenantId).page(input, selector); } catch (error) { if (error instanceof InvalidJobSummaryCursor) throw new ServiceError(400, "invalid_request", error.message); throw error; } } async listJobs() { await this.authorize("jobs:read"); const jobs = await this.jobs().list(), stats = await this.jobStatistics(jobs.map(job => job.id)); return Promise.all(jobs.map(job => this.presentJob(job, stats.get(job.id) ?? { running_count: "0", last_run_state: null }))); } diff --git a/packages/api/src/postgres/run-repository.ts b/packages/api/src/postgres/run-repository.ts index e881a0f1..85e83b03 100644 --- a/packages/api/src/postgres/run-repository.ts +++ b/packages/api/src/postgres/run-repository.ts @@ -51,10 +51,37 @@ export class RunRepository { async dueForPoll(limit = 40): Promise { const result = await this.database.pool.query(`SELECT r.tenant_id,r.id,r.job_id,r.invocation_id,r.state,r.encrypted_prompt,x.execution_config,x.trace_sources,x.execution_handle,x.execution_backend_kind,x.execution_capabilities,x.execution_diagnostics,x.failure_code,x.failure_reason,x.vm_cleanup_attempt,x.artifact_state,x.result_protocol_version,r.created_at,r.updated_at FROM app.job_runs r JOIN app.jobs j ON j.tenant_id=r.tenant_id AND j.id=r.job_id JOIN app.runs x ON x.tenant_id=r.tenant_id AND x.id=r.id - WHERE r.state IN ('running','blocked','stopping') AND (r.next_poll_at IS NULL OR r.next_poll_at<=now()) ORDER BY r.next_poll_at NULLS FIRST,r.updated_at LIMIT $1`, [limit]); + WHERE r.state IN ('running','blocked','stopping') AND x.execution_handle IS NOT NULL AND (r.next_poll_at IS NULL OR r.next_poll_at<=now()) ORDER BY r.next_poll_at NULLS FIRST,r.updated_at LIMIT $1`, [limit]); return result.rows.map(mapped); } + /** Persist a pre-provisioning gate without creating a VM or releasing the run's queue slot. */ + async blockLaunch(run: PersistedRun, reason: string): Promise { + const safeReason = reason.slice(0, 500); + await this.database.transaction(async client => { + const changed = await client.query(`UPDATE app.job_runs SET state='blocked',launch_lease_expires_at=NULL,next_poll_at=NULL,updated_at=now() + WHERE tenant_id=$1 AND id=$2 AND state='starting' RETURNING id`, [run.tenantId, run.id]); + if (!changed.rowCount) return; + await client.query(`UPDATE app.runs SET state='blocked',failure_source='system',failure_code='proxy_launch_blocked',failure_reason=$3,updated_at=now() + WHERE tenant_id=$1 AND id=$2 AND execution_handle IS NULL`, [run.tenantId, run.id, safeReason]); + await client.query("INSERT INTO app.run_activity(tenant_id,id,run_id,action,detail) VALUES ($1,$2,$3,'launch_blocked',$4)", [run.tenantId, crypto.randomUUID(), run.id, safeReason]); + }); + } + + /** A blocked pre-launch run resumes only after an explicit owner retry. */ + async retryBlockedLaunch(tenantId: string, runId: string): Promise { + return this.database.transaction(async client => { + const changed = await client.query(`UPDATE app.job_runs r SET state='reserved',launch_lease_expires_at=NULL,next_poll_at=NULL,updated_at=now() + FROM app.runs x WHERE r.tenant_id=$1 AND r.id=$2 AND r.tenant_id=x.tenant_id AND r.id=x.id + AND r.state='blocked' AND x.execution_handle IS NULL AND x.failure_code='proxy_launch_blocked' RETURNING r.id`, [tenantId, runId]); + if (!changed.rowCount) return false; + await client.query("UPDATE app.runs SET state='reserved',failure_source=NULL,failure_code=NULL,failure_reason=NULL,updated_at=now() WHERE tenant_id=$1 AND id=$2", [tenantId, runId]); + await client.query("INSERT INTO app.run_activity(tenant_id,id,run_id,action,detail) VALUES ($1,$2,$3,'launch_retry_requested','Owner requested a manual retry')", [tenantId, crypto.randomUUID(), runId]); + await client.query("INSERT INTO app.wake_hints(key,not_before) VALUES ('global',now()) ON CONFLICT (key) DO UPDATE SET not_before=least(app.wake_hints.not_before,excluded.not_before),updated_at=now()"); + return true; + }); + } + /** Persist cleanup ownership before provisioning any external resources. */ async recordLaunchHandle(run: PersistedRun, handle: RunHandle): Promise { const result = await this.database.pool.query(`UPDATE app.runs x SET execution_handle=$3,updated_at=now() diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index 7414b551..e1629ccc 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -20,6 +20,7 @@ import { TraceProjectionRepository, TRACE_PARSER_VERSION } from "./trace-project import { LiveTraceRepository } from "./live-trace-repository"; import { JOB_RESULT_INSTRUCTIONS, parseJobResult } from "../job-result"; import { issueRunCredential, runProxyInstructions } from "../run-credentials"; +import { ProxyLaunchBlockedError } from "../proxy-launch-policy"; export class RunScheduler { private runs: RunRepository; @@ -29,7 +30,12 @@ export class RunScheduler { await this.runs.consumeWakeHint(); await new AutomationScheduler(this.database, this.env).process(); for (const run of await this.runs.dueForPoll()) await this.poll(run).catch(error => this.fail(run, error, "poll")); - for (let count = 0; count < 20; count++) { const run = await this.runs.claimNext(); if (!run) break; await this.launch(run).catch(error => this.fail(run, error, "launch")); } + for (let count = 0; count < 20; count++) { + const run = await this.runs.claimNext(); if (!run) break; + await this.launch(run).catch(error => error instanceof ProxyLaunchBlockedError + ? this.runs.blockLaunch(run, `Credential proxy ${error.reason}; resolve the prerequisite and manually retry this run.`) + : this.fail(run, error, "launch")); + } } private connections(run: PersistedRun) { return new ConnectionRepository(this.database, run.tenantId, this.env.CREDENTIAL_ENCRYPTION_KEY); } diff --git a/packages/api/src/proxy-launch-policy.ts b/packages/api/src/proxy-launch-policy.ts index 0987c6f3..da8cb5ab 100644 --- a/packages/api/src/proxy-launch-policy.ts +++ b/packages/api/src/proxy-launch-policy.ts @@ -19,6 +19,17 @@ export type ProxyLaunchDecision = | { kind: "start_locked" } | { kind: "blocked"; reason: "proxy_disabled" | "proxy_unavailable" | "proxy_locked" | "pending_changes"; retry: "manual" }; +export class ProxyLaunchBlockedError extends Error { + constructor(readonly reason: Extract["reason"]) { + super(`Job launch is blocked until proxy prerequisite is resolved: ${reason}`); + this.name = "ProxyLaunchBlockedError"; + } +} + +export function requireProxyLaunch(decision: ProxyLaunchDecision): void { + if (decision.kind === "blocked") throw new ProxyLaunchBlockedError(decision.reason); +} + /** * Encodes the GEN-2824 launch gate. This function is deliberately side-effect * free: the scheduler must persist a blocked decision and wait for an explicit diff --git a/packages/api/test/proxy-launch-policy.test.ts b/packages/api/test/proxy-launch-policy.test.ts index 81544828..48952ca5 100644 --- a/packages/api/test/proxy-launch-policy.test.ts +++ b/packages/api/test/proxy-launch-policy.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { proxyLaunchDecision, type ProxyLaunchJob, type ProxyLaunchState } from "../src/proxy-launch-policy"; +import { proxyLaunchDecision, ProxyLaunchBlockedError, requireProxyLaunch, type ProxyLaunchJob, type ProxyLaunchState } from "../src/proxy-launch-policy"; const stopped: ProxyLaunchState = { enabled: true, running: false, unlocked: false, pendingChanges: false }; const locked: ProxyLaunchState = { enabled: true, running: true, unlocked: false, pendingChanges: false }; @@ -33,4 +33,10 @@ describe("GEN-2824 proxy launch policy", () => { it("does not auto-start a stopped proxy for credential-dependent jobs", () => { expect(proxyLaunchDecision({ ...flyCode, portableIntegrationIds: ["integration"] }, stopped)).toMatchObject({ kind: "blocked", retry: "manual" }); }); + + it("turns a blocked decision into a scheduler-recognized launch outcome", () => { + expect(() => requireProxyLaunch(proxyLaunchDecision({ ...flyCode, portableIntegrationIds: ["integration"] }, locked))) + .toThrow(new ProxyLaunchBlockedError("proxy_locked")); + expect(() => requireProxyLaunch(proxyLaunchDecision(flyCode, locked))).not.toThrow(); + }); }); diff --git a/packages/api/test/run-launch-block.test.ts b/packages/api/test/run-launch-block.test.ts new file mode 100644 index 00000000..5dc0670e --- /dev/null +++ b/packages/api/test/run-launch-block.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { RunRepository } from "../src/postgres/run-repository"; + +describe("durable proxy-gated launch retry", () => { + it("persists a blocked pre-launch run without polling it as a live execution", async () => { + const statements: string[] = []; + const client = { query: async (sql: string) => { statements.push(sql); return { rowCount: 1, rows: [{ id: "run" }] }; } }; + const database = { transaction: async (fn: (client: any) => Promise) => fn(client), pool: { query: client.query } }; + await new RunRepository(database as any).blockLaunch({ tenantId: "tenant", id: "run" } as any, "proxy_locked"); + expect(statements.some(sql => sql.includes("state='blocked'") && sql.includes("launch_lease_expires_at=NULL"))).toBe(true); + expect(statements.some(sql => sql.includes("execution_handle IS NULL"))).toBe(true); + expect(statements.some(sql => sql.includes("'launch_blocked'"))).toBe(true); + }); + + it("only manually requeues a proxy-gated run that has no execution handle", async () => { + const statements: string[] = []; + const client = { query: async (sql: string) => { statements.push(sql); return { rowCount: 1, rows: [{ id: "run" }] }; } }; + const database = { transaction: async (fn: (client: any) => Promise) => fn(client), pool: { query: client.query } }; + expect(await new RunRepository(database as any).retryBlockedLaunch("tenant", "run")).toBe(true); + expect(statements.some(sql => sql.includes("state='blocked'") && sql.includes("failure_code='proxy_launch_blocked'") && sql.includes("execution_handle IS NULL"))).toBe(true); + expect(statements.some(sql => sql.includes("state='reserved'"))).toBe(true); + expect(statements.some(sql => sql.includes("'launch_retry_requested'"))).toBe(true); + expect(statements.some(sql => sql.includes("wake_hints"))).toBe(true); + }); +}); diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index 32c63f8c..9e04c8a0 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -13417,6 +13417,67 @@ paths: schema: type: string minLength: 1 + /api/v1/runs/{runId}/retry-launch: + post: + summary: Manually retry a blocked launch + description: |- + Retries only a pre-launch proxy-gated run after its prerequisites have been resolved. The run remains blocked until an interactive owner explicitly retries it; unlock or status changes do not automatically launch it. + + Requires the runs:cancel scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "200": + description: Manual retry queued + content: + application/json: + schema: + type: object + required: + - runId + - retryRequested + properties: + runId: + type: string + format: uuid + retryRequested: + type: boolean + const: true + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + description: Interactive owner session required + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + description: Run is not waiting for a manual launch retry + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: post_api_v1_runs_runId_retry_launch + tags: + - runs + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: runs:cancel + x-owner-session-required: true + parameters: + - name: runId + in: path + required: true + schema: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ /api/v1/runs/{runId}/kill: post: summary: Kill run and immediately release its concurrency slot From eca2eca5557e1e26383d0d7d40794a2ba01db298 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 15:29:19 +0000 Subject: [PATCH 050/102] feat(credential-proxy): add bounded bearer forwarding --- .../credential-proxy/src/bearer-forwarder.mjs | 151 ++++++++++++++++++ .../src/destination-policy.mjs | 11 +- .../test/bearer-forwarder.test.mjs | 77 +++++++++ .../test/destination-policy.test.mjs | 2 +- 4 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 packages/credential-proxy/src/bearer-forwarder.mjs create mode 100644 packages/credential-proxy/test/bearer-forwarder.test.mjs diff --git a/packages/credential-proxy/src/bearer-forwarder.mjs b/packages/credential-proxy/src/bearer-forwarder.mjs new file mode 100644 index 00000000..5fc6ee80 --- /dev/null +++ b/packages/credential-proxy/src/bearer-forwarder.mjs @@ -0,0 +1,151 @@ +import https from "node:https"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createOriginConnector } from "./destination-policy.mjs"; + +export const DEFAULT_FORWARD_LIMITS = Object.freeze({ + connectMs: 10_000, + headersMs: 60_000, + idleMs: 120_000, + requestMs: 15 * 60_000, + requestBytes: 32 * 1024 * 1024, + responseBytes: 64 * 1024 * 1024, + headerBytes: 32 * 1024, +}); + +const HOP_BY_HOP = new Set([ + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailer", "transfer-encoding", "upgrade", "host", "authorization", + "content-length", "expect", +]); + +function safeHeaders(input, credential, maxBytes) { + const source = input instanceof Headers ? [...input.entries()] : Object.entries(input ?? {}); + const connectionTokens = new Set(); + for (const [name, value] of source) { + if (name.toLowerCase() === "connection") { + for (const token of String(value).split(",")) connectionTokens.add(token.trim().toLowerCase()); + } + } + const headers = Object.create(null); + let size = 0; + for (const [name, value] of source) { + const lower = name.toLowerCase(); + if (HOP_BY_HOP.has(lower) || lower.startsWith("x-forwarded-") || lower === "forwarded" || connectionTokens.has(lower)) continue; + const text = Array.isArray(value) ? value.join(", ") : String(value); + size += Buffer.byteLength(name) + Buffer.byteLength(text) + 4; + if (size > maxBytes) throw new RangeError("request headers exceed configured limit"); + headers[lower] = text; + } + headers.authorization = `Bearer ${credential}`; + return headers; +} + +function byteLimit(limit, label, onLimit) { + let bytes = 0; + return new Transform({ + transform(chunk, _encoding, callback) { + bytes += chunk.length; + if (bytes > limit) { + onLimit?.(); + callback(new RangeError(`${label} exceeds configured limit`)); + } else callback(null, chunk); + }, + }); +} + +function sendHttpsRequest({ socket, url, method, headers, body, limits, signal }) { + return new Promise((resolve, reject) => { + let settled = false; + let responseTimer; + let idleTimer; + const finishError = (error) => { + if (settled) return; + settled = true; + clearTimeout(responseTimer); + clearTimeout(idleTimer); + socket.destroy(error); + reject(error); + }; + const request = https.request(url, { + method, + headers, + agent: false, + createConnection: () => socket, + maxHeaderSize: limits.headerBytes, + signal, + }, (response) => { + clearTimeout(responseTimer); + let responseBytes = 0; + const chunks = []; + const resetIdle = () => { + clearTimeout(idleTimer); + idleTimer = setTimeout(() => finishError(new Error("upstream response idle deadline exceeded")), limits.idleMs); + idleTimer.unref?.(); + }; + response.on("data", (chunk) => { + responseBytes += chunk.length; + if (responseBytes > limits.responseBytes) return finishError(new RangeError("upstream response exceeds configured limit")); + chunks.push(chunk); + resetIdle(); + }); + response.once("error", finishError); + if (response.statusCode >= 300 && response.statusCode < 400) { + response.resume(); + return finishError(new Error("upstream redirects are not permitted")); + } + response.once("end", () => { + if (settled) return; + settled = true; + clearTimeout(idleTimer); + resolve({ status: response.statusCode, headers: response.headers, body: Buffer.concat(chunks) }); + }); + resetIdle(); + }); + request.once("error", finishError); + responseTimer = setTimeout(() => finishError(new Error("upstream response headers deadline exceeded")), limits.headersMs); + responseTimer.unref?.(); + if (!body) request.end(); + else { + const input = body[Symbol.asyncIterator] ? body : [body]; + pipeline(input, byteLimit(limits.requestBytes, "request body", () => request.destroy()), request, { signal }).catch(finishError); + } + }); +} + +/** + * Forward one request through a grant selected by authenticated connection state. + * The caller supplies the decrypted integration record; agent-controlled data + * cannot select either its credential or origin. + */ +export function createBearerForwarder({ connectorFactory = createOriginConnector, request = sendHttpsRequest, limits = DEFAULT_FORWARD_LIMITS } = {}) { + const active = new Set(); + return Object.freeze({ + async forward(integration, input, { signal } = {}) { + if (!integration || typeof integration.origin !== "string" || typeof integration.credential !== "string" || !integration.credential) { + throw new TypeError("integration grant is unavailable"); + } + if (active.size >= 16) throw new Error("concurrent request limit exceeded"); + const controller = new AbortController(); + const abort = () => controller.abort(signal?.reason); + signal?.addEventListener("abort", abort, { once: true }); + const timeout = setTimeout(() => controller.abort(new Error("request duration limit exceeded")), limits.requestMs); + timeout.unref?.(); + const key = {}; + active.add(key); + try { + const connect = connectorFactory(integration.origin, { timeoutMs: limits.connectMs }); + const { socket, url } = await connect(input.url); + const method = String(input.method ?? "GET").toUpperCase(); + if (!/^[A-Z!#$%&'*+.^_`|~-]+$/.test(method)) throw new TypeError("invalid HTTP method"); + const headers = safeHeaders(input.headers, integration.credential, limits.headerBytes); + return await request({ socket, url, method, headers, body: input.body, limits, signal: controller.signal }); + } finally { + clearTimeout(timeout); + signal?.removeEventListener("abort", abort); + active.delete(key); + } + }, + get activeRequests() { return active.size; }, + }); +} diff --git a/packages/credential-proxy/src/destination-policy.mjs b/packages/credential-proxy/src/destination-policy.mjs index 0b71dbbb..e0bbba9e 100644 --- a/packages/credential-proxy/src/destination-policy.mjs +++ b/packages/credential-proxy/src/destination-policy.mjs @@ -109,7 +109,7 @@ async function resolveAll(hostname) { return lookup(hostname, { all: true, verbatim: true }); } -function openTlsSocket({ hostname, port, address }) { +function openTlsSocket({ hostname, port, address, timeoutMs = 10_000 }) { return new Promise((resolve, reject) => { const socket = tlsConnect({ host: address, @@ -117,9 +117,12 @@ function openTlsSocket({ hostname, port, address }) { servername: isIP(hostname) ? undefined : hostname, checkServerIdentity: (_server, certificate) => checkServerIdentity(hostname, certificate), }); - const failed = (error) => { socket.destroy(); reject(error); }; + const timer = setTimeout(() => failed(new Error("upstream connect deadline exceeded")), timeoutMs); + timer.unref?.(); + const failed = (error) => { clearTimeout(timer); socket.destroy(); reject(error); }; socket.once("error", failed); socket.once("secureConnect", () => { + clearTimeout(timer); socket.removeListener("error", failed); resolve(socket); }); @@ -132,7 +135,7 @@ function openTlsSocket({ hostname, port, address }) { * so a second DNS lookup cannot change the destination between validation and * connection. Paths, queries, methods, and bodies remain available to callers. */ -export function createOriginConnector(configuredOrigin, { resolve = resolveAll, connect = openTlsSocket } = {}) { +export function createOriginConnector(configuredOrigin, { resolve = resolveAll, connect = openTlsSocket, timeoutMs = 10_000 } = {}) { const configured = parseHttpsUrl(configuredOrigin, "configured origin"); if (configured.url.pathname !== "/" || configured.url.search || configured.url.hash) { throw new TypeError("configured origin must contain only an HTTPS origin"); @@ -148,7 +151,7 @@ export function createOriginConnector(configuredOrigin, { resolve = resolveAll, if (!answer || !isPublicAddress(answer.address)) throw new Error("origin resolved to a non-public address"); } const address = answers[0].address; - const socket = await connect({ hostname: target.hostname, port: target.port, address, servername: target.hostname }); + const socket = await connect({ hostname: target.hostname, port: target.port, address, servername: target.hostname, timeoutMs }); const rawPeerAddress = socket?.remoteAddress ?? ""; const peerAddress = normalizeAddress(rawPeerAddress); if (!peerAddress || peerAddress !== normalizeAddress(address) || !isPublicAddress(rawPeerAddress)) { diff --git a/packages/credential-proxy/test/bearer-forwarder.test.mjs b/packages/credential-proxy/test/bearer-forwarder.test.mjs new file mode 100644 index 00000000..db1a2154 --- /dev/null +++ b/packages/credential-proxy/test/bearer-forwarder.test.mjs @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; + +const integration = { id: "integration-1", origin: "https://api.example.test", credential: "synthetic-token" }; + +test("forwards only through the granted origin and replaces unsafe routing and authorization headers", async () => { + const connections = []; + const calls = []; + const forwarder = createBearerForwarder({ + connectorFactory: origin => async url => { + connections.push({ origin, url }); + return { socket: { fixture: true }, url: new URL(url) }; + }, + request: async request => { calls.push(request); return { status: 201, body: Buffer.from("created") }; }, + }); + const result = await forwarder.forward(integration, { + method: "post", + url: "https://api.example.test/v1/items?source=job", + headers: { + Authorization: "Bearer attacker-token", + Host: "attacker.example", + Connection: "keep-alive, x-routing-hint", + "X-Routing-Hint": "attacker", + "X-Forwarded-Host": "attacker.example", + "Content-Type": "application/json", + }, + body: Buffer.from('{"ok":true}'), + }); + assert.deepEqual(result, { status: 201, body: Buffer.from("created") }); + assert.deepEqual(connections, [{ origin: integration.origin, url: "https://api.example.test/v1/items?source=job" }]); + assert.equal(calls[0].method, "POST"); + assert.equal(calls[0].headers.authorization, "Bearer synthetic-token"); + assert.equal(calls[0].headers.host, undefined); + assert.equal(calls[0].headers["x-routing-hint"], undefined); + assert.equal(calls[0].headers["x-forwarded-host"], undefined); + assert.equal(calls[0].headers["content-type"], "application/json"); +}); + +test("rejects requests outside the grant origin without sending them upstream", async () => { + let sent = false; + const forwarder = createBearerForwarder({ + connectorFactory: () => async () => { throw new TypeError("request URL does not match the configured HTTPS origin"); }, + request: async () => { sent = true; }, + }); + await assert.rejects(forwarder.forward(integration, { url: "https://169.254.169.254/latest/meta-data" }), /configured HTTPS origin/); + assert.equal(sent, false); + assert.equal(forwarder.activeRequests, 0); +}); + +test("applies an explicit per-proxy concurrent request bound", async () => { + let release; + const gate = new Promise(resolve => { release = resolve; }); + const forwarder = createBearerForwarder({ + connectorFactory: () => async url => ({ socket: {}, url: new URL(url) }), + request: async () => gate, + limits: { requestMs: 10_000 }, + }); + const started = []; + for (let i = 0; i < 16; i++) started.push(forwarder.forward(integration, { url: "https://api.example.test/" })); + await new Promise(resolve => setImmediate(resolve)); + await assert.rejects(forwarder.forward(integration, { url: "https://api.example.test/" }), /concurrent request limit/); + release({ status: 200 }); + await Promise.all(started); + assert.equal(forwarder.activeRequests, 0); +}); + +test("rejects oversized caller headers before upstream dispatch", async () => { + let sent = false; + const forwarder = createBearerForwarder({ + connectorFactory: () => async url => ({ socket: {}, url: new URL(url) }), + request: async () => { sent = true; }, + limits: { headerBytes: 20 }, + }); + await assert.rejects(forwarder.forward(integration, { url: "https://api.example.test/", headers: { "x-large": "x".repeat(50) } }), /headers exceed/); + assert.equal(sent, false); +}); diff --git a/packages/credential-proxy/test/destination-policy.test.mjs b/packages/credential-proxy/test/destination-policy.test.mjs index d6266632..264a3055 100644 --- a/packages/credential-proxy/test/destination-policy.test.mjs +++ b/packages/credential-proxy/test/destination-policy.test.mjs @@ -18,7 +18,7 @@ test("URL policy permits paths and queries only on the configured HTTPS origin", const result = await connect("https://api.example.test:8443/v1/items?limit=5"); assert.equal(result.url.pathname, "/v1/items"); assert.equal(result.url.search, "?limit=5"); - assert.deepEqual(calls, [{ hostname: "api.example.test", port: 8443, address: "93.184.216.34", servername: "api.example.test" }]); + assert.deepEqual(calls, [{ hostname: "api.example.test", port: 8443, address: "93.184.216.34", servername: "api.example.test", timeoutMs: 10_000 }]); }); test("rejects unsafe and malformed request URLs before resolution", async () => { From a6a282f3920910baeab1cda29de9edad5fc810cf Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 15:42:04 +0000 Subject: [PATCH 051/102] feat(credential-proxy): stream authorized tunnel requests --- .../credential-proxy/src/bearer-forwarder.mjs | 71 +++++++---- packages/credential-proxy/src/main.mjs | 8 +- .../credential-proxy/src/proxy-listener.mjs | 117 ++++++++++++++++++ packages/credential-proxy/src/runtime.mjs | 52 ++++++-- .../test/bearer-forwarder.test.mjs | 6 +- .../test/fixtures/upstream-cert.pem | 20 +++ .../test/fixtures/upstream-key.pem | 28 +++++ .../test/proxy-listener.e2e.test.mjs | 117 ++++++++++++++++++ 8 files changed, 379 insertions(+), 40 deletions(-) create mode 100644 packages/credential-proxy/src/proxy-listener.mjs create mode 100644 packages/credential-proxy/test/fixtures/upstream-cert.pem create mode 100644 packages/credential-proxy/test/fixtures/upstream-key.pem create mode 100644 packages/credential-proxy/test/proxy-listener.e2e.test.mjs diff --git a/packages/credential-proxy/src/bearer-forwarder.mjs b/packages/credential-proxy/src/bearer-forwarder.mjs index 5fc6ee80..37ea1854 100644 --- a/packages/credential-proxy/src/bearer-forwarder.mjs +++ b/packages/credential-proxy/src/bearer-forwarder.mjs @@ -59,48 +59,58 @@ function sendHttpsRequest({ socket, url, method, headers, body, limits, signal } let settled = false; let responseTimer; let idleTimer; + let responseStream; + const agent = new https.Agent({ keepAlive: false }); + agent.createConnection = () => socket; const finishError = (error) => { - if (settled) return; + if (settled) { + responseStream?.destroy(error); + return; + } settled = true; clearTimeout(responseTimer); clearTimeout(idleTimer); socket.destroy(error); + agent.destroy(); reject(error); }; const request = https.request(url, { method, headers, - agent: false, - createConnection: () => socket, + agent, maxHeaderSize: limits.headerBytes, signal, }, (response) => { clearTimeout(responseTimer); let responseBytes = 0; - const chunks = []; const resetIdle = () => { clearTimeout(idleTimer); - idleTimer = setTimeout(() => finishError(new Error("upstream response idle deadline exceeded")), limits.idleMs); + idleTimer = setTimeout(() => responseStream.destroy(new Error("upstream response idle deadline exceeded")), limits.idleMs); idleTimer.unref?.(); }; - response.on("data", (chunk) => { - responseBytes += chunk.length; - if (responseBytes > limits.responseBytes) return finishError(new RangeError("upstream response exceeds configured limit")); - chunks.push(chunk); - resetIdle(); + responseStream = new Transform({ + transform(chunk, _encoding, callback) { + responseBytes += chunk.length; + if (responseBytes > limits.responseBytes) return callback(new RangeError("upstream response exceeds configured limit")); + resetIdle(); + callback(null, chunk); + }, + }); + response.once("error", (error) => responseStream.destroy(error)); + responseStream.once("error", (error) => { + response.destroy(error); + request.destroy(error); }); - response.once("error", finishError); + responseStream.once("close", () => clearTimeout(idleTimer)); + responseStream.once("close", () => agent.destroy()); if (response.statusCode >= 300 && response.statusCode < 400) { response.resume(); return finishError(new Error("upstream redirects are not permitted")); } - response.once("end", () => { - if (settled) return; - settled = true; - clearTimeout(idleTimer); - resolve({ status: response.statusCode, headers: response.headers, body: Buffer.concat(chunks) }); - }); resetIdle(); + response.pipe(responseStream); + settled = true; + resolve({ status: response.statusCode, headers: response.headers, body: responseStream }); }); request.once("error", finishError); responseTimer = setTimeout(() => finishError(new Error("upstream response headers deadline exceeded")), limits.headersMs); @@ -133,17 +143,34 @@ export function createBearerForwarder({ connectorFactory = createOriginConnector timeout.unref?.(); const key = {}; active.add(key); + let released = false; + const release = () => { + if (released) return; + released = true; + clearTimeout(timeout); + signal?.removeEventListener("abort", abort); + active.delete(key); + }; try { const connect = connectorFactory(integration.origin, { timeoutMs: limits.connectMs }); const { socket, url } = await connect(input.url); + if (controller.signal.aborted) { + socket.destroy(controller.signal.reason); + throw controller.signal.reason ?? new Error("request cancelled"); + } const method = String(input.method ?? "GET").toUpperCase(); if (!/^[A-Z!#$%&'*+.^_`|~-]+$/.test(method)) throw new TypeError("invalid HTTP method"); const headers = safeHeaders(input.headers, integration.credential, limits.headerBytes); - return await request({ socket, url, method, headers, body: input.body, limits, signal: controller.signal }); - } finally { - clearTimeout(timeout); - signal?.removeEventListener("abort", abort); - active.delete(key); + const result = await request({ socket, url, method, headers, body: input.body, limits, signal: controller.signal }); + if (result?.body && typeof result.body.once === "function") { + result.body.once("end", release); + result.body.once("close", release); + result.body.once("error", release); + } else release(); + return result; + } catch (error) { + release(); + throw error; } }, get activeRequests() { return active.size; }, diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 20c32377..28cd1f65 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -10,10 +10,10 @@ function respond(value) { process.stdout.write(`${JSON.stringify(value)}\n`); } -function stop() { +async function stop() { if (closing) return; closing = true; - runtime.shutdown(); + await runtime.shutdown(); lines.close(); clearTimeout(shutdownDeadline); process.exit(0); @@ -25,7 +25,7 @@ shutdownDeadline = setTimeout(() => process.exit(1), 2_000); shutdownDeadline.unref(); respond({ type: "ready", state: runtime.state }); -lines.on("line", (line) => { +lines.on("line", async (line) => { if (closing) return; let command; try { @@ -35,7 +35,7 @@ lines.on("line", (line) => { runtime.unlock(command.key); respond({ type: "status", state: runtime.state }); } else if (command?.type === "lock") { - runtime.lock(); + await runtime.lock(); respond({ type: "status", state: runtime.state }); } else if (command?.type === "stop") stop(); else respond({ type: "error", error: "unsupported command" }); diff --git a/packages/credential-proxy/src/proxy-listener.mjs b/packages/credential-proxy/src/proxy-listener.mjs new file mode 100644 index 00000000..3def1524 --- /dev/null +++ b/packages/credential-proxy/src/proxy-listener.mjs @@ -0,0 +1,117 @@ +import { createServer } from "node:http"; +import { pipeline } from "node:stream/promises"; +import { createBearerForwarder } from "./bearer-forwarder.mjs"; + +const RESPONSE_HOP_BY_HOP = new Set([ + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", + "trailer", "transfer-encoding", "upgrade", +]); + +function grantTable(grants) { + if (!Array.isArray(grants)) throw new TypeError("authenticated connection grants must be an array"); + const table = new Map(); + for (const grant of grants) { + if (!grant || typeof grant.id !== "string" || !grant.id || table.has(grant.id) + || typeof grant.origin !== "string" || typeof grant.credential !== "string" || !grant.credential) { + throw new TypeError("authenticated connection contains an invalid integration grant"); + } + table.set(grant.id, Object.freeze({ id: grant.id, origin: grant.origin, credential: grant.credential })); + } + return table; +} + +function requestTarget(origin, suffix, search) { + const base = new URL(origin); + if (base.protocol !== "https:" || base.pathname !== "/" || base.search || base.hash) { + throw new TypeError("integration grant origin must be an HTTPS origin"); + } + const path = suffix || "/"; + if (!path.startsWith("/") || path.startsWith("//") || path.includes("\\")) { + throw new TypeError("invalid integration request path"); + } + return `${base.origin}${path}${search}`; +} + +/** + * Start one loopback HTTP endpoint for a single already-authenticated tunnel + * connection. The connection's consumed grants are held in this closure and + * can never be selected or supplied by an HTTP caller. + */ +export async function createAuthenticatedProxyListener({ connection, forwarder = createBearerForwarder(), host = "127.0.0.1", port = 0 } = {}) { + if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true + || !connection.connectionId || !connection.runId || !connection.vmId) { + throw new TypeError("proxy listener requires an authenticated consumed connection"); + } + const grants = grantTable(connection.grants); + const controllers = new Set(); + const server = createServer(async (request, response) => { + const controller = new AbortController(); + controllers.add(controller); + const cancel = () => { + if (!response.writableEnded) controller.abort(new Error("tunnel request cancelled")); + }; + request.once("aborted", cancel); + response.once("close", cancel); + try { + const parsed = new URL(request.url ?? "/", "http://proxy.loopback"); + const match = /^\/integrations\/([^/]+)(\/.*)?$/.exec(parsed.pathname); + if (!match) { + response.writeHead(404).end(); + return; + } + let integrationId; + try { integrationId = decodeURIComponent(match[1]); } catch { + response.writeHead(400).end(); + return; + } + if (!integrationId || integrationId.includes("/") || integrationId.includes("\\")) { + response.writeHead(400).end(); + return; + } + const integration = grants.get(integrationId); + if (!integration) { + response.writeHead(403).end(); + return; + } + const target = requestTarget(integration.origin, match[2] ?? "/", parsed.search); + const result = await forwarder.forward(integration, { + url: target, + method: request.method, + headers: request.headers, + body: request, + }, { signal: controller.signal }); + const headers = Object.create(null); + for (const [name, value] of Object.entries(result.headers ?? {})) { + if (value !== undefined && !RESPONSE_HOP_BY_HOP.has(name.toLowerCase())) headers[name] = value; + } + response.writeHead(result.status, headers); + await pipeline(result.body, response, { signal: controller.signal }); + } catch (error) { + if (!response.headersSent && !response.destroyed) { + response.writeHead(controller.signal.aborted ? 499 : 502, { "content-type": "text/plain; charset=utf-8" }); + response.end(controller.signal.aborted ? "request cancelled" : "upstream request failed"); + } else if (!response.destroyed) response.destroy(error); + } finally { + controllers.delete(controller); + request.removeListener("aborted", cancel); + response.removeListener("close", cancel); + } + }); + server.on("connect", (_request, socket) => { socket.end("HTTP/1.1 405 Method Not Allowed\r\nConnection: close\r\n\r\n"); }); + server.on("upgrade", (_request, socket) => socket.destroy()); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { + server.removeListener("error", reject); + resolve(); + }); + }); + return Object.freeze({ + connectionId: connection.connectionId, + address: server.address(), + close: () => { + for (const controller of controllers) controller.abort(new Error("authenticated tunnel disconnected")); + return new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + }, + }); +} diff --git a/packages/credential-proxy/src/runtime.mjs b/packages/credential-proxy/src/runtime.mjs index e85401c7..4627acb9 100644 --- a/packages/credential-proxy/src/runtime.mjs +++ b/packages/credential-proxy/src/runtime.mjs @@ -1,17 +1,23 @@ +import { createAuthenticatedProxyListener } from "./proxy-listener.mjs"; + /** - * Internal adapter contracts for the follow-up record, enrollment and forwarding - * slices. This runtime deliberately does not call or implement these adapters. - * - * @typedef {object} ProxyAdapters - * @property {(identity: object) => Promise} loadRecords - * @property {(request: object) => Promise} createEnrollment - * @property {(request: object) => Promise} forwardRequest + * Process-local runtime for decrypted records and authorized tunnel listeners. + * The transport adapter must pass a connection only after it has consumed and + * authenticated its enrollment against the current run and proxy boot epoch. */ -export function createProxyRuntime() { +export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyListener } = {}) { let unlockMaterial; let state = "locked"; let shutdown = false; + let locking = false; + const listeners = new Set(); + + async function closeListeners() { + const current = [...listeners]; + listeners.clear(); + await Promise.allSettled(current.map((listener) => listener.close())); + } return Object.freeze({ get state() { @@ -19,19 +25,41 @@ export function createProxyRuntime() { }, unlock(key) { if (shutdown) throw new Error("runtime is shutting down"); + if (locking) throw new Error("proxy is locking"); if (typeof key !== "string" || key.length === 0) throw new TypeError("unlock key must be a non-empty string"); unlockMaterial?.fill(0); unlockMaterial = Buffer.from(key, "utf8"); state = "unlocked"; }, - lock() { + async lock() { + locking = true; + state = "locked"; unlockMaterial?.fill(0); unlockMaterial = undefined; - if (!shutdown) state = "locked"; + try { await closeListeners(); } + finally { locking = false; } + }, + async acceptAuthenticatedConnection(connection) { + if (shutdown) throw new Error("runtime is shutting down"); + if (state !== "unlocked") throw new Error("proxy is locked"); + // The transport adapter calls this only after atomically consuming and + // authenticating enrollment. Grants are the consumed, current run state. + const listener = await listenerFactory({ connection }); + if (state !== "unlocked" || shutdown) { + await listener.close(); + throw new Error("proxy locked while accepting the connection"); + } + listeners.add(listener); + return listener; + }, + async disconnect(connectionId) { + const matching = [...listeners].filter((listener) => listener.connectionId === connectionId); + await Promise.allSettled(matching.map((listener) => listener.close())); + for (const listener of matching) listeners.delete(listener); }, - shutdown() { - this.lock(); + async shutdown() { shutdown = true; + await this.lock(); state = "stopped"; }, }); diff --git a/packages/credential-proxy/test/bearer-forwarder.test.mjs b/packages/credential-proxy/test/bearer-forwarder.test.mjs index db1a2154..9284b49a 100644 --- a/packages/credential-proxy/test/bearer-forwarder.test.mjs +++ b/packages/credential-proxy/test/bearer-forwarder.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import { Readable } from "node:stream"; import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; const integration = { id: "integration-1", origin: "https://api.example.test", credential: "synthetic-token" }; @@ -12,7 +13,7 @@ test("forwards only through the granted origin and replaces unsafe routing and a connections.push({ origin, url }); return { socket: { fixture: true }, url: new URL(url) }; }, - request: async request => { calls.push(request); return { status: 201, body: Buffer.from("created") }; }, + request: async request => { calls.push(request); return { status: 201, headers: {}, body: Readable.from([Buffer.from("created")]) }; }, }); const result = await forwarder.forward(integration, { method: "post", @@ -27,7 +28,8 @@ test("forwards only through the granted origin and replaces unsafe routing and a }, body: Buffer.from('{"ok":true}'), }); - assert.deepEqual(result, { status: 201, body: Buffer.from("created") }); + assert.equal(result.status, 201); + assert.deepEqual(Buffer.concat(await Array.fromAsync(result.body)), Buffer.from("created")); assert.deepEqual(connections, [{ origin: integration.origin, url: "https://api.example.test/v1/items?source=job" }]); assert.equal(calls[0].method, "POST"); assert.equal(calls[0].headers.authorization, "Bearer synthetic-token"); diff --git a/packages/credential-proxy/test/fixtures/upstream-cert.pem b/packages/credential-proxy/test/fixtures/upstream-cert.pem new file mode 100644 index 00000000..685d5670 --- /dev/null +++ b/packages/credential-proxy/test/fixtures/upstream-cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNDCCAhygAwIBAgIUWvCDWltyenSy441ZnfAvqkjgIwUwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQYXBpLmV4YW1wbGUudGVzdDAeFw0yNjExMTExNTQwNTha +Fw0zNjExMDgxNTQwNThaMBsxGTAXBgNVBAMMEGFwaS5leGFtcGxlLnRlc3QwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDClgGTbXnob9GHbVxMt1lSmsfW +6lSfuOEBEuE5n96bKcfSsVKMRKvw3j8PyGasF+DTQTwqFlcavHzmmVkOBpQYLSz3 +bvD0RnKJ8CXhZwcRSA77BENRJQJ3TRr9WIAVrz3dgkAEl5Tv4Y+0KILRqZMx+vg1 +R0CuAzt6JbhGTvFugRAgeSJootV/O9Uu+z4SqtEbOkpj7isRUJa1uirAmKoFucIU +sa/4wTBkWIL74Z2iEMdmNM0HQ0xTsoVmFVJb+yxCoCr5rzIdPuGDyQeqWaWtju7Z +kVJjUskyYgSrycUivxAElY9aedgmyZxoBIHiwdFsSt/Nu5Gj9mOahqAfAPdjAgMB +AAGjcDBuMB0GA1UdDgQWBBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAfBgNVHSMEGDAW +gBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAPBgNVHRMBAf8EBTADAQH/MBsGA1UdEQQU +MBKCEGFwaS5leGFtcGxlLnRlc3QwDQYJKoZIhvcNAQELBQADggEBAFOW5O4OKDe8 +6MWztml7Yt9Rvn6zcDTM4xsMKhMDITAlorXuAfu51JGgFOsqiqNP9L9cpN57k5JS +5wYA9Fy+cFiiXtkAqImSvjSf1gvlzKpHCh1s0dGjIYnoStoWbE0PzVweSQ2zCQy/ +aQC11E3/CyS2+toZR+i1zffaBLjbA6T4GToOKCrXdGl8Wv7vCyTbEMA6ouEspkJO +U/glcrkekGgIZVC10di70LI+LkgqsnuDpFIALb7qDMtZQuqnIEU/wyapORpQn6Lm +WjHPEcWSHe5dlHEQWO0P3XXEA2mbr+fboVWWqQ/5gt7aBTLOVfV9rSkQqGpvriPQ +RmX+kW6AbK4= +-----END CERTIFICATE----- diff --git a/packages/credential-proxy/test/fixtures/upstream-key.pem b/packages/credential-proxy/test/fixtures/upstream-key.pem new file mode 100644 index 00000000..69f0bcaa --- /dev/null +++ b/packages/credential-proxy/test/fixtures/upstream-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDClgGTbXnob9GH +bVxMt1lSmsfW6lSfuOEBEuE5n96bKcfSsVKMRKvw3j8PyGasF+DTQTwqFlcavHzm +mVkOBpQYLSz3bvD0RnKJ8CXhZwcRSA77BENRJQJ3TRr9WIAVrz3dgkAEl5Tv4Y+0 +KILRqZMx+vg1R0CuAzt6JbhGTvFugRAgeSJootV/O9Uu+z4SqtEbOkpj7isRUJa1 +uirAmKoFucIUsa/4wTBkWIL74Z2iEMdmNM0HQ0xTsoVmFVJb+yxCoCr5rzIdPuGD +yQeqWaWtju7ZkVJjUskyYgSrycUivxAElY9aedgmyZxoBIHiwdFsSt/Nu5Gj9mOa +hqAfAPdjAgMBAAECggEAHCixGbCiD6z+2WZOERwHZRjZCpTmoiaTCSC+TyR/hlzJ +eggHNh1J9JV08sfBc+4QrwGlv0F0YX4dZRGT/Z+eFtGSzEvVusPnF2cVVEUglLOk +11aKl1ePv8v8t+TyxDCT91Y+BWwaNYyiyyOBqu2rS3RSLwOcrJHzLX3/6MzLy33I +1++WK9wGUKEyzfQ8k363yUv1CmhN/4kJs9a00OLc9pvBKO/LLxMxGZ3lMUsVhVUc +3h0yjlRvhvYDvZeLJKGhWVhn1wPfa6BNPwMNatJ6O5ubdA1qegOOxi+VDheSg05H +MVILqoFWt6B7AY+iMHbzbR1YJxSYWNVA9X9KWqtWoQKBgQD7EnZ1137IBFeC0UwK +bvjnh92lPCg3Ek8iBaH+tHga08KZr4xR10tsbqXIKxoGBeE0m3uOQbnZ0cDYSlSP +MsEXk3hthnpSNT1Ln/K1sc5ht7wb0xhDzbVsRjhtH0Ae64LrExhn/wlTirrtwDcx +6Dw1zymbr7a6LflUtHNB7V28VwKBgQDGZ7kWt7w7yDZlLQPkEmJ3BIXTvDKPsfCv +1R4Xwn/ZHaXA8J6CHZ3tEzOSWSPU7TOCSwu3mrMS+CGK5GINOYJ54YfNcgbQJ+7G +Yzw7yUFidbhJ6TBulhmCO8gdR743CRweC0K28bVe5suPnNNdlJDyowwtboua9Vn5 +H0sxcNz11QKBgQDmqxDAtYqaUzrfSlE1aL0vYqklLP65kg238Tri2oVaCGHKdvlR +G58E8SdInyjajb5Zadv7V7FbSJWHYcsMgRHcCjXjUmDBdH5rI3mGquQ07OxbF/c8 +HBtpbZ5byxFLFw8B0Fpk+XBlG5kf47ASHw5qVXkhuy8DIEfpJZya3mx3IwKBgFNN +IU0Ef6ep/AymxcdwYJ8P6xBnJ+NlFmT82dGn+/2wkPNdE3AV90SelzyO6Jhxrvq5 +f4XrEdyZCZxDH1fmGX4KzvXXNj0XRw1MhPGlIiqwultt6MPFiZhAeVix0o9Q59lF +aRFekoPx6u21LplP0zvMcURx8XNyKACMvwlXFa6xAoGAK5qaim0rJzwrtlr8T1z/ +HxzM4ZAZJchb8fN7LExj2ZrzCmZY2pnpS8oH3ae9iopV9LKUR+b6OwgnV8eWv1qx +Ldc3l7R/8AD9MOmuFwU36VozB/esox6r6MKaBl8m009wceouZG7uV4KvKr2E2Kry +dfnJ6oBHEnM8GcAQjZ3tqJA= +-----END PRIVATE KEY----- diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs new file mode 100644 index 00000000..34bfae01 --- /dev/null +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -0,0 +1,117 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { createServer as createHttpsServer } from "node:https"; +import { once } from "node:events"; +import { connect as tlsConnect } from "node:tls"; +import { test } from "node:test"; +import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; +import { createAuthenticatedProxyListener } from "../src/proxy-listener.mjs"; +import { createProxyRuntime } from "../src/runtime.mjs"; + +const fixtures = new URL("./fixtures/", import.meta.url); +const key = await readFile(new URL("upstream-key.pem", fixtures)); +const cert = await readFile(new URL("upstream-cert.pem", fixtures)); + +async function waitForEvent(emitter, event) { + const pending = once(emitter, event); + return pending; +} + +test("authenticated tunnel listener streams over TLS, confines grants and cancels upstream work", { timeout: 5_000 }, async () => { + let observedAuthorization; + let upstreamEnded = false; + let cancelUpstream; + let finishUpstream; + let cancelStartedResolve; + const cancelStarted = new Promise(resolve => { cancelStartedResolve = resolve; }); + const upstream = createHttpsServer({ key, cert }, async (request, response) => { + observedAuthorization = request.headers.authorization; + if (request.url === "/cancel") { + const closed = new Promise(resolve => request.socket.once("close", resolve)); + response.writeHead(200, { "content-type": "text/plain" }); + response.write("first chunk"); + cancelUpstream = closed; + cancelStartedResolve(); + return; + } + const received = []; + for await (const chunk of request) received.push(chunk); + assert.equal(Buffer.concat(received).toString(), "request-body"); + response.writeHead(200, { "content-type": "text/plain" }); + response.write("first chunk"); + response.once("close", () => { upstreamEnded = true; }); + await new Promise(resolve => { finishUpstream = resolve; }); + response.end(" second chunk"); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + const forwarder = createBearerForwarder({ + connectorFactory: origin => async requestUrl => { + assert.equal(origin, `https://api.example.test:${upstreamPort}`); + const url = new URL(requestUrl); + const socket = tlsConnect({ host: "127.0.0.1", port: upstreamPort, servername: url.hostname, ca: cert }); + await once(socket, "secureConnect"); + return { socket, url }; + }, + }); + const runtime = createProxyRuntime({ + listenerFactory: ({ connection }) => createAuthenticatedProxyListener({ connection, forwarder }), + }); + let listener; + try { + runtime.unlock("synthetic-unlock-key"); + listener = await runtime.acceptAuthenticatedConnection({ + authenticated: true, + enrollmentConsumed: true, + connectionId: "consumed-enrollment-1", + runId: "run-1", + vmId: "vm-1", + grants: [{ id: "tool-1", origin: `https://api.example.test:${upstreamPort}`, credential: "synthetic-bearer" }], + }); + const address = listener.address; + assert.equal(address.address, "127.0.0.1"); + const base = `http://127.0.0.1:${address.port}`; + const response = await fetch(`${base}/integrations/tool-1/echo?source=vm`, { + method: "POST", + headers: { authorization: "Bearer caller-token", "content-type": "text/plain" }, + body: "request-body", + }); + assert.equal(response.status, 200); + assert.equal(observedAuthorization, "Bearer synthetic-bearer"); + const reader = response.body.getReader(); + const first = await reader.read(); + assert.equal(Buffer.from(first.value).toString(), "first chunk"); + assert.equal(upstreamEnded, false, "response was delivered before the upstream finished"); + assert.equal(forwarder.activeRequests, 1, "streaming response retains its concurrency slot"); + finishUpstream(); + const second = await reader.read(); + assert.equal(Buffer.from(second.value).toString(), " second chunk"); + assert.equal((await reader.read()).done, true); + assert.equal(forwarder.activeRequests, 0); + + const denied = await fetch(`${base}/integrations/other-tool/echo`); + assert.equal(denied.status, 403); + assert.equal(observedAuthorization, "Bearer synthetic-bearer", "ungranted request never reached the upstream"); + + const abort = new AbortController(); + const cancelled = fetch(`${base}/integrations/tool-1/cancel`, { signal: abort.signal }); + await cancelStarted; + abort.abort(); + await assert.rejects(cancelled, error => error.name === "AbortError"); + assert.ok(cancelUpstream, "upstream cancellation hook was installed"); + await Promise.race([cancelUpstream, new Promise((_, reject) => setTimeout(() => reject(new Error("upstream was not cancelled")), 1_000))]); + assert.equal(forwarder.activeRequests, 0); + } finally { + finishUpstream?.(); + if (listener) await runtime.disconnect("consumed-enrollment-1"); + await runtime.shutdown(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); + +test("listener refuses an unconsumed or unauthenticated connection", async () => { + await assert.rejects(createAuthenticatedProxyListener({ connection: { + authenticated: true, connectionId: "connection", runId: "run", vmId: "vm", grants: [], + } }), /authenticated consumed connection/); +}); From f1d2c84ad316a235119107abf5b51b89a51c0f2a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 16:00:46 +0000 Subject: [PATCH 052/102] feat(credential-proxy): connect authenticated VM tunnel --- .../credential-proxy/src/tunnel-transport.mjs | 131 ++++++++++++++++++ .../test/proxy-listener.e2e.test.mjs | 66 +++++++++ 2 files changed, 197 insertions(+) create mode 100644 packages/credential-proxy/src/tunnel-transport.mjs diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs new file mode 100644 index 00000000..b3a68363 --- /dev/null +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -0,0 +1,131 @@ +import { connect as connectTls, createServer as createTlsServer } from "node:tls"; +import { connect as connectTcp, createServer as createTcpServer } from "node:net"; +import { once } from "node:events"; + +const HANDSHAKE_LIMIT = 4096; +const HANDSHAKE_TIMEOUT_MS = 10_000; + +function peerIdentity(socket) { + const certificate = socket.getPeerCertificate(); + if (!socket.authorized || !certificate?.subject?.CN) throw new Error("tunnel client certificate is not authorized"); + return certificate.subject.CN; +} + +/** + * Accept only a TLS client with a trusted VM certificate, consume its one-use + * enrollment, and bridge that stream to the connection-scoped HTTP listener. + * The bridge does not expose a shell or arbitrary destination forwarding. + */ +export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnrollment, host = "127.0.0.1", port = 0 } = {}) { + if (!tls || !runtime || typeof authenticateEnrollment !== "function") throw new TypeError("tunnel server requires TLS, runtime and enrollment authentication"); + const connections = new Set(); + const server = createTlsServer({ ...tls, requestCert: true, rejectUnauthorized: true }, (socket) => { + connections.add(socket); + let handshake = Buffer.alloc(0); + let settled = false; + const timeout = setTimeout(() => socket.destroy(new Error("tunnel enrollment timed out")), HANDSHAKE_TIMEOUT_MS); + timeout.unref(); + const reject = () => { + if (settled) return; + settled = true; + clearTimeout(timeout); + socket.destroy(); + }; + socket.on("data", async (chunk) => { + if (settled) return; + handshake = Buffer.concat([handshake, chunk]); + const end = handshake.indexOf("\n"); + if (end < 0) return handshake.length > HANDSHAKE_LIMIT ? reject() : undefined; + if (end > HANDSHAKE_LIMIT) return reject(); + settled = true; + clearTimeout(timeout); + socket.pause(); + const line = handshake.subarray(0, end).toString("utf8").replace(/\r$/, ""); + const rest = handshake.subarray(end + 1); + const match = /^FACTORIZE-TUNNEL\/1 ([A-Za-z0-9_-]{32,128})$/.exec(line); + if (!match) return reject(); + let listener; + let target; + try { + const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket) }); + if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true) throw new Error("enrollment was not consumed"); + listener = await runtime.acceptAuthenticatedConnection(connection); + const address = listener.address; + target = connectTcp(address.port, address.address); + await once(target, "connect"); + if (rest.length) target.write(rest); + socket.pipe(target); + target.pipe(socket); + socket.resume(); + const close = () => { + socket.unpipe(target); + target.unpipe(socket); + target.destroy(); + void runtime.disconnect(connection.connectionId); + }; + socket.once("close", close); + target.once("close", close); + } catch { + listener && await runtime.disconnect(listener.connectionId); + socket.destroy(); + } + }); + socket.once("close", () => { clearTimeout(timeout); connections.delete(socket); }); + socket.once("error", () => {}); + }); + server.listen(port, host); + return { + server, + listening: once(server, "listening"), + address: () => server.address(), + close: async () => { + for (const socket of connections) socket.destroy(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + }, + }; +} + +/** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ +export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPort, host = "127.0.0.1", port = 0 } = {}) { + if (!tls || typeof enrollment !== "function" || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { + throw new TypeError("VM tunnel helper requires TLS, enrollment and proxy address"); + } + const sockets = new Set(); + const server = createTcpServer((local) => { + sockets.add(local); + void (async () => { + let remote; + try { + const grant = await enrollment(); + if (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token)) throw new Error("fresh enrollment unavailable"); + remote = connectTls({ ...tls, host: proxyHost, port: proxyPort, servername: tls.servername ?? proxyHost, rejectUnauthorized: true }); + sockets.add(remote); + const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); + try { await once(remote, "secureConnect"); } + finally { clearTimeout(connectTimer); } + remote.write(`FACTORIZE-TUNNEL/1 ${grant.token}\n`); + local.pipe(remote); + remote.pipe(local); + local.resume(); + remote.once("close", () => { sockets.delete(remote); local.destroy(); }); + local.once("close", () => remote.destroy()); + } catch { + local.destroy(); + remote?.destroy(); + } + })(); + const remove = () => { sockets.delete(local); }; + local.once("close", remove); + local.once("error", remove); + }); + server.listen(port, host); + await once(server, "listening"); + const address = server.address(); + return Object.freeze({ + address, + close: async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + }, + }); +} diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 34bfae01..fadcf5bb 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -7,6 +7,7 @@ import { test } from "node:test"; import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; import { createAuthenticatedProxyListener } from "../src/proxy-listener.mjs"; import { createProxyRuntime } from "../src/runtime.mjs"; +import { createAuthenticatedTunnelServer, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; const fixtures = new URL("./fixtures/", import.meta.url); const key = await readFile(new URL("upstream-key.pem", fixtures)); @@ -115,3 +116,68 @@ test("listener refuses an unconsumed or unauthenticated connection", async () => authenticated: true, connectionId: "connection", runId: "run", vmId: "vm", grants: [], } }), /authenticated consumed connection/); }); + +test("VM helper uses mutual TLS and consumed enrollment to reach only its granted integration listener", { timeout: 5_000 }, async () => { + let observedAuthorization; + let enrollmentCount = 0; + let authenticationCount = 0; + const upstream = createHttpsServer({ key, cert }, (request, response) => { + observedAuthorization = request.headers.authorization; + response.end("transport-ok"); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + const forwarder = createBearerForwarder({ + connectorFactory: origin => async requestUrl => { + assert.equal(origin, `https://api.example.test:${upstreamPort}`); + const url = new URL(requestUrl); + const socket = tlsConnect({ host: "127.0.0.1", port: upstreamPort, servername: url.hostname, ca: cert }); + await once(socket, "secureConnect"); + return { socket, url }; + }, + }); + const runtime = createProxyRuntime({ listenerFactory: ({ connection }) => createAuthenticatedProxyListener({ connection, forwarder }) }); + const tunnel = createAuthenticatedTunnelServer({ + tls: { key, cert, ca: cert }, + runtime, + authenticateEnrollment: async ({ token, vmIdentity }) => { + authenticationCount++; + assert.equal(token, (authenticationCount === 1 ? "A" : "B").repeat(43)); + assert.equal(vmIdentity, "api.example.test"); + return { + authenticated: true, + enrollmentConsumed: true, + connectionId: "transport-enrollment-1", + runId: "run-transport-1", + vmId: vmIdentity, + grants: [{ id: "tool-1", origin: `https://api.example.test:${upstreamPort}`, credential: "synthetic-bearer" }], + }; + }, + }); + await tunnel.listening; + runtime.unlock("synthetic-unlock-key"); + const local = await startVmTunnelHelper({ + tls: { key, cert, ca: cert, servername: "api.example.test" }, + enrollment: async () => ({ token: (++enrollmentCount === 1 ? "A" : "B").repeat(43) }), + proxyHost: "127.0.0.1", + proxyPort: tunnel.address().port, + }); + try { + const response = await fetch(`http://127.0.0.1:${local.address.port}/integrations/tool-1/echo`, { + headers: { authorization: "Bearer caller-token" }, + }); + assert.equal(response.status, 200); + assert.equal(await response.text(), "transport-ok"); + assert.equal(observedAuthorization, "Bearer synthetic-bearer"); + const denied = await fetch(`http://127.0.0.1:${local.address.port}/integrations/other-tool/echo`); + assert.equal(denied.status, 403); + assert.equal(enrollmentCount, 2, "each local connection gets fresh one-use enrollment"); + assert.equal(authenticationCount, 2, "each enrollment is independently consumed by the proxy transport"); + } finally { + await local.close(); + await tunnel.close(); + await runtime.shutdown(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); From 8c6be50401318f33cdfc164a1e2ea7e1b8cdd635 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 16:13:28 +0000 Subject: [PATCH 053/102] Bind enrollment acceptance to proxy boot epoch --- packages/credential-proxy/src/main.mjs | 2 +- packages/credential-proxy/src/runtime.mjs | 9 +++++++- .../credential-proxy/src/tunnel-transport.mjs | 2 +- .../test/proxy-listener.e2e.test.mjs | 23 ++++++++++++++++++- .../test/runtime-process.test.mjs | 7 +++++- 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 28cd1f65..d74a19c2 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -23,7 +23,7 @@ process.once("SIGTERM", stop); process.once("SIGINT", stop); shutdownDeadline = setTimeout(() => process.exit(1), 2_000); shutdownDeadline.unref(); -respond({ type: "ready", state: runtime.state }); +respond({ type: "ready", state: runtime.state, bootEpoch: runtime.bootEpoch }); lines.on("line", async (line) => { if (closing) return; diff --git a/packages/credential-proxy/src/runtime.mjs b/packages/credential-proxy/src/runtime.mjs index 4627acb9..06cf340c 100644 --- a/packages/credential-proxy/src/runtime.mjs +++ b/packages/credential-proxy/src/runtime.mjs @@ -1,4 +1,5 @@ import { createAuthenticatedProxyListener } from "./proxy-listener.mjs"; +import { randomUUID } from "node:crypto"; /** * Process-local runtime for decrypted records and authorized tunnel listeners. @@ -6,7 +7,10 @@ import { createAuthenticatedProxyListener } from "./proxy-listener.mjs"; * authenticated its enrollment against the current run and proxy boot epoch. */ -export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyListener } = {}) { +export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyListener, bootEpoch = randomUUID() } = {}) { + if (typeof bootEpoch !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(bootEpoch)) { + throw new TypeError("proxy boot epoch must be a UUID v4"); + } let unlockMaterial; let state = "locked"; let shutdown = false; @@ -20,6 +24,8 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL } return Object.freeze({ + // A fresh process identity invalidates enrollments from every prior boot. + bootEpoch, get state() { return state; }, @@ -42,6 +48,7 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL async acceptAuthenticatedConnection(connection) { if (shutdown) throw new Error("runtime is shutting down"); if (state !== "unlocked") throw new Error("proxy is locked"); + if (connection?.bootEpoch !== bootEpoch) throw new Error("enrollment belongs to a different proxy boot"); // The transport adapter calls this only after atomically consuming and // authenticating enrollment. Grants are the consumed, current run state. const listener = await listenerFactory({ connection }); diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index b3a68363..5777b185 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -47,7 +47,7 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro let listener; let target; try { - const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket) }); + const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket), bootEpoch: runtime.bootEpoch }); if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true) throw new Error("enrollment was not consumed"); listener = await runtime.acceptAuthenticatedConnection(connection); const address = listener.address; diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index fadcf5bb..7ba9eef0 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -65,6 +65,7 @@ test("authenticated tunnel listener streams over TLS, confines grants and cancel listener = await runtime.acceptAuthenticatedConnection({ authenticated: true, enrollmentConsumed: true, + bootEpoch: runtime.bootEpoch, connectionId: "consumed-enrollment-1", runId: "run-1", vmId: "vm-1", @@ -117,6 +118,24 @@ test("listener refuses an unconsumed or unauthenticated connection", async () => } }), /authenticated consumed connection/); }); +test("runtime rejects a consumed enrollment from an earlier proxy boot", async () => { + const runtime = createProxyRuntime({ bootEpoch: "11111111-1111-4111-8111-111111111111" }); + runtime.unlock("synthetic-unlock-key"); + try { + await assert.rejects(runtime.acceptAuthenticatedConnection({ + authenticated: true, + enrollmentConsumed: true, + bootEpoch: "22222222-2222-4222-8222-222222222222", + connectionId: "stale-enrollment", + runId: "run-1", + vmId: "vm-1", + grants: [], + }), /different proxy boot/); + } finally { + await runtime.shutdown(); + } +}); + test("VM helper uses mutual TLS and consumed enrollment to reach only its granted integration listener", { timeout: 5_000 }, async () => { let observedAuthorization; let enrollmentCount = 0; @@ -141,13 +160,15 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante const tunnel = createAuthenticatedTunnelServer({ tls: { key, cert, ca: cert }, runtime, - authenticateEnrollment: async ({ token, vmIdentity }) => { + authenticateEnrollment: async ({ token, vmIdentity, bootEpoch }) => { authenticationCount++; assert.equal(token, (authenticationCount === 1 ? "A" : "B").repeat(43)); assert.equal(vmIdentity, "api.example.test"); + assert.equal(bootEpoch, runtime.bootEpoch); return { authenticated: true, enrollmentConsumed: true, + bootEpoch, connectionId: "transport-enrollment-1", runId: "run-transport-1", vmId: vmIdentity, diff --git a/packages/credential-proxy/test/runtime-process.test.mjs b/packages/credential-proxy/test/runtime-process.test.mjs index 64fb7fe7..03944bf9 100644 --- a/packages/credential-proxy/test/runtime-process.test.mjs +++ b/packages/credential-proxy/test/runtime-process.test.mjs @@ -38,9 +38,13 @@ async function startProxy() { }); child.stderr.setEncoding("utf8").on("data", (chunk) => { stderr += chunk; }); await waitFor(() => messages.length > 0, "proxy startup"); - assert.deepEqual(messages.shift(), { type: "ready", state: "locked" }); + const ready = messages.shift(); + assert.equal(ready.type, "ready"); + assert.equal(ready.state, "locked"); + assert.match(ready.bootEpoch, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); return { child, + bootEpoch: ready.bootEpoch, get logs() { return `${log}${stderr}`; }, command(command) { child.stdin.write(`${JSON.stringify(command)}\n`); @@ -65,6 +69,7 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts const restarted = await startProxy(); processes.push(restarted); + assert.notEqual(restarted.bootEpoch, first.bootEpoch, "every process restart gets a fresh enrollment epoch"); assert.deepEqual(await restarted.command({ type: "status" }), { type: "status", state: "locked" }); assert.deepEqual(await restarted.command({ type: "unlock", key: tokenMarker }), { type: "status", state: "unlocked" }); restarted.child.stdin.write('{"type":"stop"}\n'); From 1ec03454f805c12cb03c55a6ab95222f4bc35031 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 16:52:22 +0000 Subject: [PATCH 054/102] Bind tunnel enrollment to authenticated VM peer --- .../credential-proxy/src/tunnel-transport.mjs | 10 +++++++++- .../test/proxy-listener.e2e.test.mjs | 20 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index 5777b185..2ad21a67 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -48,7 +48,15 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro let target; try { const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket), bootEpoch: runtime.bootEpoch }); - if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true) throw new Error("enrollment was not consumed"); + // The production control plane returns the consumed enrollment binding. + // Compare its VM and boot identities with the authenticated TLS peer and + // this process so an adapter cannot accidentally attach another run's + // grants to the tunnel. + if (!connection || connection.authenticated !== true || connection.enrollmentConsumed !== true + || connection.vmIdentity !== peerIdentity(socket) || connection.bootEpoch !== runtime.bootEpoch + || typeof connection.connectionId !== "string" || connection.connectionId.length === 0) { + throw new Error("enrollment binding does not match the authenticated tunnel"); + } listener = await runtime.acceptAuthenticatedConnection(connection); const address = listener.address; target = connectTcp(address.port, address.address); diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 7ba9eef0..c8c6a940 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -162,12 +162,13 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante runtime, authenticateEnrollment: async ({ token, vmIdentity, bootEpoch }) => { authenticationCount++; - assert.equal(token, (authenticationCount === 1 ? "A" : "B").repeat(43)); + assert.equal(token, (authenticationCount === 1 ? "A" : authenticationCount === 2 ? "B" : "C").repeat(43)); assert.equal(vmIdentity, "api.example.test"); assert.equal(bootEpoch, runtime.bootEpoch); return { authenticated: true, enrollmentConsumed: true, + vmIdentity: token === "C".repeat(43) ? "other-vm" : vmIdentity, bootEpoch, connectionId: "transport-enrollment-1", runId: "run-transport-1", @@ -180,10 +181,11 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante runtime.unlock("synthetic-unlock-key"); const local = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, - enrollment: async () => ({ token: (++enrollmentCount === 1 ? "A" : "B").repeat(43) }), + enrollment: async () => ({ token: (++enrollmentCount === 1 ? "A" : enrollmentCount === 2 ? "B" : "C").repeat(43) }), proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, }); + let mismatchLocal; try { const response = await fetch(`http://127.0.0.1:${local.address.port}/integrations/tool-1/echo`, { headers: { authorization: "Bearer caller-token" }, @@ -195,7 +197,21 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante assert.equal(denied.status, 403); assert.equal(enrollmentCount, 2, "each local connection gets fresh one-use enrollment"); assert.equal(authenticationCount, 2, "each enrollment is independently consumed by the proxy transport"); + mismatchLocal = await startVmTunnelHelper({ + tls: { key, cert, ca: cert, servername: "api.example.test" }, + enrollment: async () => ({ token: (++enrollmentCount === 3 ? "C" : "D").repeat(43) }), + proxyHost: "127.0.0.1", + proxyPort: tunnel.address().port, + }); + await assert.rejects( + fetch(`http://127.0.0.1:${mismatchLocal.address.port}/integrations/tool-1/echo`), + /fetch failed/, + "mismatched VM binding is rejected", + ); + assert.equal(enrollmentCount, 3); + assert.equal(authenticationCount, 3); } finally { + await mismatchLocal?.close(); await local.close(); await tunnel.close(); await runtime.shutdown(); From 4ae0b74860e91f02180084a58fc5b2dc9e5ae02f Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 17:53:46 +0000 Subject: [PATCH 055/102] Add authenticated proxy boot registration --- .../0036_proxy_control_credentials.sql | 10 ++++ .../src/postgres/proxy-control-repository.ts | 56 ++++++++++++++++++ packages/api/src/postgres/schema.ts | 3 +- packages/api/src/proxy-control.ts | 38 ++++++++++++ packages/api/src/worker.ts | 2 + .../api/test/proxy-control-repository.test.ts | 51 ++++++++++++++++ packages/api/test/proxy-control.test.ts | 58 +++++++++++++++++++ .../credential-proxy/src/control-client.mjs | 25 ++++++++ packages/credential-proxy/src/main.mjs | 10 ++++ .../test/control-client.test.mjs | 25 ++++++++ 10 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 packages/api/migrations/0036_proxy_control_credentials.sql create mode 100644 packages/api/src/postgres/proxy-control-repository.ts create mode 100644 packages/api/src/proxy-control.ts create mode 100644 packages/api/test/proxy-control-repository.test.ts create mode 100644 packages/api/test/proxy-control.test.ts create mode 100644 packages/credential-proxy/src/control-client.mjs create mode 100644 packages/credential-proxy/test/control-client.test.mjs diff --git a/packages/api/migrations/0036_proxy_control_credentials.sql b/packages/api/migrations/0036_proxy_control_credentials.sql new file mode 100644 index 00000000..6c7d8b38 --- /dev/null +++ b/packages/api/migrations/0036_proxy_control_credentials.sql @@ -0,0 +1,10 @@ +CREATE TABLE app.credential_proxy_control_credentials ( + tenant_id uuid NOT NULL REFERENCES app.tenants(id) ON DELETE CASCADE, + proxy_identity text NOT NULL, + boot_epoch uuid NOT NULL, + token_digest bytea NOT NULL UNIQUE, + revoked_at timestamptz, + consumed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (tenant_id, proxy_identity) +); diff --git a/packages/api/src/postgres/proxy-control-repository.ts b/packages/api/src/postgres/proxy-control-repository.ts new file mode 100644 index 00000000..2dee38ae --- /dev/null +++ b/packages/api/src/postgres/proxy-control-repository.ts @@ -0,0 +1,56 @@ +import type { Database } from "./database"; + +export type RegisteredProxy = { tenantId: string; proxyIdentity: string; bootEpoch: string }; +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const digest = async (token: string) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(token))); + +/** Credentials for the Factorize-to-proxy control plane. These are never job run credentials. */ +export class ProxyControlRepository { + constructor(private database: Database) {} + + /** Called only from the authenticated infrastructure provisioning path; the token is returned once. */ + async provisionCredential(tenantId: string, proxyIdentity: string, bootEpoch: string): Promise { + if (!uuid.test(tenantId) || !proxyIdentity.trim() || proxyIdentity.length > 255 || !uuid.test(bootEpoch)) throw new TypeError("Invalid proxy identity or boot epoch"); + const token = `fzc_${Buffer.from(crypto.getRandomValues(new Uint8Array(32))).toString("base64url")}`; + await this.database.pool.query(`INSERT INTO app.credential_proxy_control_credentials(tenant_id,proxy_identity,boot_epoch,token_digest,revoked_at,consumed_at) + VALUES ($1,$2,$3,$4,NULL,NULL) ON CONFLICT (tenant_id,proxy_identity) DO UPDATE SET boot_epoch=excluded.boot_epoch,token_digest=excluded.token_digest,revoked_at=NULL,consumed_at=NULL,created_at=now()`, + [tenantId, proxyIdentity, bootEpoch, Buffer.from(await digest(token))]); + return token; + } + + async authenticate(token: string): Promise | null> { + if (typeof token !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return null; + const result = await this.database.pool.query<{ tenant_id: string; proxy_identity: string }>(`SELECT tenant_id,proxy_identity + FROM app.credential_proxy_control_credentials WHERE token_digest=$1 AND revoked_at IS NULL`, [Buffer.from(await digest(token))]); + const row = result.rows[0]; + return row ? { tenantId: row.tenant_id, proxyIdentity: row.proxy_identity } : null; + } + + async registerBoot(token: string, bootEpoch: string): Promise { + if (!uuid.test(bootEpoch)) throw new TypeError("bootEpoch must be a UUID"); + const tokenDigest = Buffer.from(await digest(token)); + return this.database.transaction(async client => { + const credential = await client.query<{ tenant_id: string; proxy_identity: string }>(`SELECT tenant_id,proxy_identity + FROM app.credential_proxy_control_credentials WHERE token_digest=$1 AND boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL FOR UPDATE`, [tokenDigest, bootEpoch]); + const identity = credential.rows[0]; + if (!identity) return null; + const saved = await client.query(`INSERT INTO app.credential_proxy_states(tenant_id,proxy_identity,boot_epoch) + VALUES ($1,$2,$3) ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity,boot_epoch=excluded.boot_epoch,updated_at=now() + WHERE app.credential_proxy_states.proxy_identity=excluded.proxy_identity + AND app.credential_proxy_states.disabled_at IS NULL AND app.credential_proxy_states.revoked_at IS NULL RETURNING tenant_id`, + [identity.tenant_id, identity.proxy_identity, bootEpoch]); + if (!saved.rowCount) return null; + await client.query(`UPDATE app.credential_proxy_control_credentials SET consumed_at=now() + WHERE token_digest=$1 AND boot_epoch=$2 AND consumed_at IS NULL`, [tokenDigest, bootEpoch]); + await client.query(`UPDATE app.credential_run_grants SET revoked_at=now() + WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch<>$3 AND revoked_at IS NULL`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); + return { tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch }; + }); + } + + async revokeCredential(tenantId: string, proxyIdentity: string): Promise { + const result = await this.database.pool.query(`UPDATE app.credential_proxy_control_credentials SET revoked_at=now() + WHERE tenant_id=$1 AND proxy_identity=$2 AND revoked_at IS NULL`, [tenantId, proxyIdentity]); + return Boolean(result.rowCount); + } +} diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 554df98f..57b492ba 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -53,6 +53,7 @@ export const integrationChanges = app.table("integration_changes", { export const credentialProxyStates = app.table("credential_proxy_states", { tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), disabledAt: timestamp("disabled_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), updatedAt: updatedAt(), }); +export const credentialProxyControlCredentials = app.table("credential_proxy_control_credentials", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), tokenDigest: bytea("token_digest").notNull().unique(), revokedAt: timestamp("revoked_at", { withTimezone: true }), consumedAt: timestamp("consumed_at", { withTimezone: true }), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.proxyIdentity] })]); export const credentialRunGrants = app.table("credential_run_grants", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), vmIdentity: text("vm_identity").notNull(), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), grantRevision: bigint("grant_revision", { mode: "number" }).notNull(), revokedAt: timestamp("revoked_at", { withTimezone: true }), createdAt: createdAt(), }, t => [primaryKey({ columns: [t.tenantId, t.runId] }), foreignKey({ columns: [t.tenantId, t.runId], foreignColumns: [jobRuns.tenantId, jobRuns.id] }).onDelete("cascade"), check("credential_run_grants_revision_check", sql`${t.grantRevision} > 0`)]); @@ -162,7 +163,7 @@ export const runRelations = relations(runs, ({ one, many }) => ({ export const databaseSchema = { schemaMigrations, tenants, oauthConnections, oauthConsentPreviews, authUsers, authAccounts, authResetTokens, authSessions, oauthDeviceAuthorizations, members, accessTokens, - connections, integrationCollections, integrationRecords, integrationChanges, credentialProxyStates, credentialRunGrants, credentialRunEnrollments, linearWorkspaces, githubInstallations, githubSetupStates, projects, jobs, vmImageVerifications, skills, jobSkills, triggers, webhookConditionsCutover, invocations, manualInvocationPermissions, jobRuns, runCredentials, runs, runArtifacts, runRetentionCleanup, + connections, integrationCollections, integrationRecords, integrationChanges, credentialProxyStates, credentialProxyControlCredentials, credentialRunGrants, credentialRunEnrollments, linearWorkspaces, githubInstallations, githubSetupStates, projects, jobs, vmImageVerifications, skills, jobSkills, triggers, webhookConditionsCutover, invocations, manualInvocationPermissions, jobRuns, runCredentials, runs, runArtifacts, runRetentionCleanup, runTraceEvents, runActivity, activeClaims, scheduleState, automaticWakes, lifecycleDeliveries, lifecycleEvents, jobEditDeliveries, jobEvents, webhookDeliveries, webhookDeliveryEvents, pendingVerifications, tailFingerprints, wakeHints, runTraceCursors, runTraceChunks, runTraceProjections, traceReplayLimits, traceReplayOperations, diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts new file mode 100644 index 00000000..3cd1632b --- /dev/null +++ b/packages/api/src/proxy-control.ts @@ -0,0 +1,38 @@ +import { databaseFor } from "./postgres/database"; +import { ProxyControlRepository } from "./postgres/proxy-control-repository"; + +const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "Cache-Control": "no-store" } }); + +/** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ +export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { + const url = new URL(request.url); + if (url.pathname !== "/internal/credential-proxy/register-boot") return json({ error: "not_found" }, 404); + if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); + if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); + const authorization = request.headers.get("authorization") ?? ""; + const token = authorization.startsWith("Bearer ") ? authorization.slice(7) : ""; + if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return json({ error: "unauthorized" }, 401); + const declaredLength = Number(request.headers.get("content-length") ?? "0"); + if (declaredLength > 1024) return json({ error: "invalid_request" }, 413); + let body: unknown; + try { + const reader = request.body?.getReader(); + const chunks: Uint8Array[] = []; let length = 0; + if (reader) for (;;) { + const { done, value } = await reader.read(); if (done) break; + length += value.byteLength; + if (length > 1024) { await reader.cancel(); return json({ error: "invalid_request" }, 413); } + chunks.push(value); + } + const bytes = new Uint8Array(length); let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + body = JSON.parse(new TextDecoder().decode(bytes)); + } catch { return json({ error: "invalid_request" }, 400); } + if (!body || typeof body !== "object" || Array.isArray(body) + || Object.keys(body).some(key => key !== "bootEpoch") + || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); + const repository = new ProxyControlRepository(databaseFor(env)); + const registration = await repository.registerBoot(token, (body as any).bootEpoch); + if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); + return json({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); +} diff --git a/packages/api/src/worker.ts b/packages/api/src/worker.ts index d7868347..e32c339b 100644 --- a/packages/api/src/worker.ts +++ b/packages/api/src/worker.ts @@ -14,6 +14,7 @@ import { artifactUpload } from "./artifact-upload"; import { traceChunkUpload } from "./trace-chunk-upload"; import { databaseFor } from "./postgres/database"; import { IdentityRepository } from "./postgres/identity-repository"; +import { proxyControlFetch } from "./proxy-control"; const scopes = [...ACCESS_SCOPES]; // Keep accepting grants issued by the pre-resource-scope protocol. New @@ -85,6 +86,7 @@ function provider(env: Env) { return new OAuthProvider(providerOptions(env) export { AlarmCoordinator } from "./alarm-coordinator"; const backend = { async fetch(request: Request, env: Env, ctx: ExecutionContext) { + if (new URL(request.url).pathname.startsWith("/internal/credential-proxy/")) return proxyControlFetch(request, env); const staticResponse = !["/authorize", "/device"].includes(new URL(request.url).pathname) ? await staticApp(request, env) : null; if (staticResponse) return staticResponse; const oauth = provider(env); diff --git a/packages/api/test/proxy-control-repository.test.ts b/packages/api/test/proxy-control-repository.test.ts new file mode 100644 index 00000000..20f1934c --- /dev/null +++ b/packages/api/test/proxy-control-repository.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from "vitest"; +import { ProxyControlRepository } from "../src/postgres/proxy-control-repository"; + +function database(query: ReturnType, transactionQuery = query) { + return { pool: { query }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) } as any; +} + +describe("dedicated credential proxy control identity", () => { + it("issues a scoped one-time secret and stores only its digest", async () => { + const query = vi.fn().mockResolvedValue({ rows: [], rowCount: 1 }); + const repository = new ProxyControlRepository(database(query)); + const token = await repository.provisionCredential("11111111-1111-4111-8111-111111111111", "proxy-a", "22222222-2222-4222-8222-222222222222"); + expect(token).toMatch(/^fzc_[A-Za-z0-9_-]{43}$/); + const stored = query.mock.calls[0][1][3] as Buffer; + expect(stored).toBeInstanceOf(Buffer); + expect(stored.toString()).not.toContain(token); + expect(stored.byteLength).toBe(32); + }); + + it("authenticates only a live credential and never accepts a guest run token", async () => { + const query = vi.fn().mockResolvedValue({ rows: [{ tenant_id: "tenant-a", proxy_identity: "proxy-a" }], rowCount: 1 }); + const repository = new ProxyControlRepository(database(query)); + expect(await repository.authenticate("fzt_run_" + "a".repeat(43))).toBeNull(); + expect(query).not.toHaveBeenCalled(); + expect(await repository.authenticate("fzc_" + "a".repeat(43))).toEqual({ tenantId: "tenant-a", proxyIdentity: "proxy-a" }); + }); + + it("registers a fresh boot for its credential identity and rejects revoked, disabled, or mismatched identities", async () => { + const identity = { tenant_id: "tenant-a", proxy_identity: "proxy-a" }; + const credentialQuery = vi.fn().mockResolvedValue({ rows: [identity], rowCount: 1 }); + const stateQuery = vi.fn().mockResolvedValue({ rows: [{ tenant_id: identity.tenant_id }], rowCount: 1 }); + const repository = new ProxyControlRepository(database(vi.fn(), vi.fn())); + const tx = { query: vi.fn().mockImplementation((sql: string) => sql.includes("credential_proxy_control_credentials") ? credentialQuery() : sql.includes("INSERT INTO app.credential_proxy_states") ? stateQuery() : Promise.resolve({ rows: [], rowCount: 1 })) }; + (repository as any).database = { transaction: (work: (client: any) => Promise) => work(tx) }; + const epoch = "22222222-2222-4222-8222-222222222222"; + expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch)).toEqual({ tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch: epoch }); + expect(tx.query.mock.calls.map(call => call[0])).toHaveLength(4); + expect(tx.query.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); + expect(tx.query.mock.calls[2][0]).toContain("consumed_at=now()"); + expect(tx.query.mock.calls[3][0]).toContain("boot_epoch<>$3"); + credentialQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 }); + expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch)).toBeNull(); + }); + + it("rejects malformed boot epochs before opening a transaction", async () => { + const transaction = vi.fn(); + const repository = new ProxyControlRepository({ transaction, pool: { query: vi.fn() } } as any); + await expect(repository.registerBoot("fzc_" + "a".repeat(43), "old-boot")).rejects.toThrow("bootEpoch must be a UUID"); + expect(transaction).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts new file mode 100644 index 00000000..734cfff4 --- /dev/null +++ b/packages/api/test/proxy-control.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi } from "vitest"; +import { proxyControlFetch } from "../src/proxy-control"; + +const token = `fzc_${"A".repeat(43)}`; +const bootEpoch = "22222222-2222-4222-8222-222222222222"; + +function envFor({ credentialFound = true, stateSaved = true } = {}) { + const transactionQuery = vi.fn(async (sql: string) => { + if (sql.includes("credential_proxy_control_credentials")) return { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture" }] : [], rowCount: credentialFound ? 1 : 0 }; + if (sql.includes("INSERT INTO app.credential_proxy_states")) return { rows: stateSaved ? [{ tenant_id: "11111111-1111-4111-8111-111111111111" }] : [], rowCount: stateSaved ? 1 : 0 }; + return { rows: [], rowCount: 1 }; + }); + return { DATABASE: { pool: { query: vi.fn() }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery } as any; +} + +function request({ authorization, body = { bootEpoch }, method = "POST" }: { authorization?: string; body?: unknown; method?: string } = {}) { + return new Request("https://factorize.example/internal/credential-proxy/register-boot", { method, headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}) }, body: method === "POST" ? JSON.stringify(body) : undefined }); +} + +describe("dedicated proxy boot registration endpoint", () => { + it("binds registration to the proxy identity owned by its distinct control credential", async () => { + const env = envFor(); + const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch }); + expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(4); + expect(env.transactionQuery.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); + expect(env.transactionQuery.mock.calls[2][0]).toContain("consumed_at=now()"); + }); + + it("rejects guest job tokens before database access", async () => { + const env = envFor(); + const response = await proxyControlFetch(request({ authorization: `Bearer fzt_run_${"B".repeat(32)}` }), env); + expect(response.status).toBe(401); + expect(env.transactionQuery).not.toHaveBeenCalled(); + }); + + it("rejects revoked credentials, disabled proxy identities and tenant or proxy substitution", async () => { + for (const env of [envFor({ credentialFound: false }), envFor({ stateSaved: false })]) { + const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); + expect(response.status).toBe(403); + } + }); + + it("validates the endpoint, method and exact boot payload", async () => { + const env = envFor(); + expect((await proxyControlFetch(new Request("https://factorize.example/internal/credential-proxy/register-boot", { method: "GET" }), env)).status).toBe(405); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, body: { bootEpoch, tenantId: "attacker-tenant" } }), env)).status).toBe(400); + expect(env.transactionQuery).not.toHaveBeenCalled(); + }); + + it("bounds registration payloads before parsing", async () => { + const env = envFor(); + const response = await proxyControlFetch(request({ authorization: `Bearer ${token}`, body: { bootEpoch, padding: "x".repeat(2_000) } }), env); + expect(response.status).toBe(413); + expect(env.transactionQuery).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs new file mode 100644 index 00000000..77a430e2 --- /dev/null +++ b/packages/credential-proxy/src/control-client.mjs @@ -0,0 +1,25 @@ +const bootEpochPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +/** Register one proxy process boot using a provisioning-scoped credential. The token is only held in this call's memory. */ +export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetchImpl = fetch }) { + const origin = new URL(controlOrigin); + if (origin.protocol !== "https:" || origin.pathname !== "/" || origin.username || origin.password || origin.search || origin.hash) throw new TypeError("proxy control origin must be an HTTPS origin"); + if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token ?? "")) throw new TypeError("invalid proxy control credential"); + if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); + let credential = token; + try { + const response = await fetchImpl(new URL("/internal/credential-proxy/register-boot", origin), { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${credential}` }, + body: JSON.stringify({ bootEpoch }), + signal: AbortSignal.timeout(10_000), + redirect: "error", + }); + if (!response.ok) throw new Error("proxy boot registration rejected"); + const result = await response.json(); + if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string") throw new Error("invalid proxy boot registration response"); + return { proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch }; + } finally { + credential = ""; + } +} diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index d74a19c2..f5409b16 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -1,5 +1,6 @@ import { createInterface } from "node:readline"; import { createProxyRuntime } from "./runtime.mjs"; +import { registerProxyBoot } from "./control-client.mjs"; const runtime = createProxyRuntime(); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); @@ -37,6 +38,15 @@ lines.on("line", async (line) => { } else if (command?.type === "lock") { await runtime.lock(); respond({ type: "status", state: runtime.state }); + } else if (command?.type === "register-boot") { + let credential = command.credential; + command.credential = undefined; + try { + const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + respond({ type: "boot-registered", ...registration }); + } catch { + respond({ type: "error", error: "proxy boot registration failed" }); + } finally { credential = ""; } } else if (command?.type === "stop") stop(); else respond({ type: "error", error: "unsupported command" }); } catch { diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs new file mode 100644 index 00000000..7f59aba7 --- /dev/null +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { registerProxyBoot } from "../src/control-client.mjs"; + +const token = `fzc_${"A".repeat(43)}`; +const bootEpoch = "22222222-2222-4222-8222-222222222222"; + +test("boot registration uses a one-call control credential, HTTPS, exact path and fresh process epoch", async () => { + let request; + const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async (url, init) => { + request = { url: String(url), init }; + return Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch }); + } }); + assert.deepEqual(result, { proxyIdentity: "proxy-fixture", bootEpoch }); + assert.equal(request.url, "https://factorize.example/internal/credential-proxy/register-boot"); + assert.equal(request.init.headers.Authorization, `Bearer ${token}`); + assert.equal(request.init.redirect, "error"); + assert.deepEqual(JSON.parse(request.init.body), { bootEpoch }); +}); + +test("boot registration rejects unsafe origins, invalid credentials and mismatched response bindings", async () => { + await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch }), /HTTPS origin/); + await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); + await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async () => Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch: "33333333-3333-4333-8333-333333333333" }) }), /invalid proxy boot registration response/); +}); From 56fd6c2ebb0de361b2d8d6d4bc9311b7f5397177 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 18:15:56 +0000 Subject: [PATCH 056/102] Add secure proxy bootstrap provisioning boundary --- packages/api/src/proxy-provisioner.ts | 37 +++++++++++++++++ packages/api/test/proxy-provisioner.test.ts | 45 +++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 packages/api/src/proxy-provisioner.ts create mode 100644 packages/api/test/proxy-provisioner.test.ts diff --git a/packages/api/src/proxy-provisioner.ts b/packages/api/src/proxy-provisioner.ts new file mode 100644 index 00000000..f6d4957a --- /dev/null +++ b/packages/api/src/proxy-provisioner.ts @@ -0,0 +1,37 @@ +import type { ProxyControlRepository } from "./postgres/proxy-control-repository"; + +export type ProxyBootstrapProvisioning = { + tenantId: string; + proxyIdentity: string; + /** A new UUID for every process boot. */ + bootEpoch: string; + controlOrigin: string; +}; + +/** + * Connects one-use bootstrap credential issuance to the infrastructure + * provisioner's private control channel. Implementations must deliver the + * credential directly to proxy stdin (or an equally short-lived memory-only + * channel), never through Machine config, environment, metadata, disk, or logs. + */ +export async function provisionCredentialProxy( + input: ProxyBootstrapProvisioning, + dependencies: { + control: Pick; + startProxy: (request: ProxyBootstrapProvisioning & { credential: string }) => Promise; + }, +): Promise<{ proxyIdentity: string; bootEpoch: string }> { + const origin = new URL(input.controlOrigin); + if (origin.protocol !== "https:" || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) { + throw new TypeError("controlOrigin must be an HTTPS origin"); + } + const credential = await dependencies.control.provisionCredential(input.tenantId, input.proxyIdentity, input.bootEpoch); + try { + await dependencies.startProxy({ ...input, controlOrigin: origin.origin, credential }); + } catch (error) { + // A failed/ambiguous start must not leave a valid bootstrap credential. + await dependencies.control.revokeCredential(input.tenantId, input.proxyIdentity).catch(() => false); + throw error; + } + return { proxyIdentity: input.proxyIdentity, bootEpoch: input.bootEpoch }; +} diff --git a/packages/api/test/proxy-provisioner.test.ts b/packages/api/test/proxy-provisioner.test.ts new file mode 100644 index 00000000..1a107b53 --- /dev/null +++ b/packages/api/test/proxy-provisioner.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it, vi } from "vitest"; +import { provisionCredentialProxy } from "../src/proxy-provisioner"; + +const input = { + tenantId: "11111111-1111-4111-8111-111111111111", + proxyIdentity: "fly-app/proxy-machine", + bootEpoch: "22222222-2222-4222-8222-222222222222", + controlOrigin: "https://factorize.example/", +}; + +describe("credential proxy bootstrap provisioning", () => { + it("issues an epoch-bound credential only to the provisioner's private start channel", async () => { + const control = { + provisionCredential: vi.fn().mockResolvedValue("fzc_one-use-secret"), + revokeCredential: vi.fn().mockResolvedValue(true), + }; + const startProxy = vi.fn().mockResolvedValue(undefined); + + await expect(provisionCredentialProxy(input, { control, startProxy })).resolves.toEqual({ + proxyIdentity: input.proxyIdentity, + bootEpoch: input.bootEpoch, + }); + + expect(control.provisionCredential).toHaveBeenCalledWith(input.tenantId, input.proxyIdentity, input.bootEpoch); + expect(startProxy).toHaveBeenCalledWith({ ...input, controlOrigin: "https://factorize.example", credential: "fzc_one-use-secret" }); + expect(control.revokeCredential).not.toHaveBeenCalled(); + }); + + it("revokes the one-use credential if proxy startup fails", async () => { + const control = { + provisionCredential: vi.fn().mockResolvedValue("fzc_one-use-secret"), + revokeCredential: vi.fn().mockResolvedValue(true), + }; + const failure = new Error("startup failed"); + await expect(provisionCredentialProxy(input, { control, startProxy: vi.fn().mockRejectedValue(failure) })).rejects.toBe(failure); + expect(control.revokeCredential).toHaveBeenCalledWith(input.tenantId, input.proxyIdentity); + }); + + it("rejects non-HTTPS or non-origin control URLs before issuing a credential", async () => { + const control = { provisionCredential: vi.fn(), revokeCredential: vi.fn() }; + await expect(provisionCredentialProxy({ ...input, controlOrigin: "http://factorize.example" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); + await expect(provisionCredentialProxy({ ...input, controlOrigin: "https://factorize.example/path" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); + expect(control.provisionCredential).not.toHaveBeenCalled(); + }); +}); From 6df5360018e5bdb57cbadf1dd5ccdd844c4cfdad Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 18:37:32 +0000 Subject: [PATCH 057/102] Provision credential proxy with isolated bootstrap --- .../src/fly-credential-proxy-provisioner.ts | 134 ++++++++++++++++++ .../fly-credential-proxy-provisioner.test.ts | 93 ++++++++++++ .../src/bootstrap-control.mjs | 67 +++++++++ packages/credential-proxy/src/main.mjs | 8 +- packages/credential-proxy/src/startup.mjs | 46 ++++++ .../test/bootstrap-control.test.mjs | 56 ++++++++ .../test/runtime-process.test.mjs | 12 +- 7 files changed, 409 insertions(+), 7 deletions(-) create mode 100644 packages/api/src/fly-credential-proxy-provisioner.ts create mode 100644 packages/api/test/fly-credential-proxy-provisioner.test.ts create mode 100644 packages/credential-proxy/src/bootstrap-control.mjs create mode 100644 packages/credential-proxy/src/startup.mjs create mode 100644 packages/credential-proxy/test/bootstrap-control.test.mjs diff --git a/packages/api/src/fly-credential-proxy-provisioner.ts b/packages/api/src/fly-credential-proxy-provisioner.ts new file mode 100644 index 00000000..ad8751d4 --- /dev/null +++ b/packages/api/src/fly-credential-proxy-provisioner.ts @@ -0,0 +1,134 @@ +import { provisionCredentialProxy, type ProxyBootstrapProvisioning } from "./proxy-provisioner"; +import type { ProxyControlRepository } from "./postgres/proxy-control-repository"; +import type { FlyMachine, FlyMachineConfig } from "./fly-machines-client"; +import { createFlyGuestTransport } from "./fly-guest-transport"; + +const IMAGE_DIGEST = /@sha256:[a-f0-9]{64}$/i; +const UUID_V4 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +export type FlyCredentialProxyConfig = { + tenantId: string; + app: string; + /** Only reviewed credential-proxy images pinned by digest are accepted. */ + image: string; + region: string; + size: string; + memoryMb?: number; + controlOrigin: string; + /** Stable across retries of one provisioning operation. */ + operationId: string; +}; + +export type FlyCredentialProxyMachineClient = { + createMachine(app: string, config: FlyMachineConfig, operationId: string): Promise; + startMachine(app: string, machine: string): Promise; + destroyMachine(app: string, machine: string): Promise; +}; + +export type FlyCredentialProxyGuest = { + /** The reviewed image startup command consumes this payload from stdin only. */ + startProxy(input: ProxyBootstrapProvisioning & { credential: string }): Promise; +}; + +/** Provider exec adapter: the bootstrap record is the command's stdin payload. */ +export function createFlyCredentialProxyGuest(options: { + app: string; + machine: string; + token: string; + cliPath?: string; + spawnProcess?: typeof import("node:child_process").spawn; +}): FlyCredentialProxyGuest { + const transport = createFlyGuestTransport({ ...options, timeoutMs: 30_000, outputLimitBytes: 16 * 1024 }); + return { + async startProxy(input) { + const result = await transport.run(["/usr/local/bin/factorize-proxy-startup"], `${JSON.stringify({ + controlOrigin: input.controlOrigin, + bootEpoch: input.bootEpoch, + credential: input.credential, + })}\n`); + if (!result.ok) throw new Error("Credential proxy startup command failed"); + if (result.stdout.trim() !== "Credential proxy started") throw new Error("Credential proxy startup was not confirmed"); + }, + }; +} + +export type FlyCredentialProxyProvisionerOptions = { + control: Pick; + machines: FlyCredentialProxyMachineClient; + guest: (machine: FlyMachine) => FlyCredentialProxyGuest; + /** Injectable to keep boot identity deterministic in boundary tests. */ + newBootEpoch?: () => string; +}; + +function validate(input: FlyCredentialProxyConfig): void { + if (!input.tenantId || !input.app || !input.operationId || input.operationId.length > 200) throw new TypeError("Invalid credential proxy provisioning identity"); + if (!IMAGE_DIGEST.test(input.image)) throw new TypeError("Credential proxy image must be pinned by sha256 digest"); + if (!/^[a-z0-9]{3}$/.test(input.region)) throw new TypeError("Invalid credential proxy region"); + if (!/^[a-z0-9-]{1,64}$/.test(input.size)) throw new TypeError("Invalid credential proxy size"); + if (input.memoryMb !== undefined && (!Number.isInteger(input.memoryMb) || input.memoryMb < 256 || input.memoryMb > 8192)) throw new TypeError("Credential proxy memory must be between 256 and 8192 MiB"); +} + +/** + * Provisions one dedicated Fly Machine for the credential proxy and wires its + * startup through the one-use bootstrap boundary. The bootstrap credential is + * never part of the Machine config; the guest adapter must deliver it over its + * private stdin startup channel. + */ +export function createFlyCredentialProxyProvisioner(options: FlyCredentialProxyProvisionerOptions) { + return { + async start(input: FlyCredentialProxyConfig): Promise<{ proxyIdentity: string; bootEpoch: string; machine: FlyMachine }> { + validate(input); + const origin = new URL(input.controlOrigin); + if (origin.protocol !== "https:" || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) { + throw new TypeError("controlOrigin must be an HTTPS origin"); + } + const candidateBootEpoch = (options.newBootEpoch ?? (() => crypto.randomUUID()))(); + if (!UUID_V4.test(candidateBootEpoch)) throw new TypeError("boot epoch must be a UUID v4"); + const config: FlyMachineConfig = { + image: input.image, + region: input.region, + cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", candidateBootEpoch], + guest: { cpu_kind: "shared", cpus: 1, memory_mb: input.memoryMb ?? 512 }, + metadata: { + factorize_role: "credential-proxy", + factorize_tenant_id: input.tenantId, + factorize_operation_id: input.operationId, + factorize_boot_epoch: candidateBootEpoch, + }, + // The proxy must be explicitly started and must not be restarted after + // process failure with a stale one-use bootstrap credential. + auto_start: false, + auto_stop: false, + restart: { policy: "no" }, + }; + const machine = await options.machines.createMachine(input.app, config, input.operationId); + if (!machine.id) throw new Error("Fly returned a credential proxy Machine without an id"); + const bootEpoch = machine.config?.metadata?.factorize_boot_epoch ?? candidateBootEpoch; + if (!UUID_V4.test(bootEpoch)) throw new Error("Fly returned a credential proxy Machine with an invalid boot epoch"); + const proxyIdentity = `fly:${input.app}:${machine.id}`; + const bootstrap = { + tenantId: input.tenantId, + proxyIdentity, + bootEpoch, + controlOrigin: origin.origin, + }; + try { + await provisionCredentialProxy(bootstrap, { + control: options.control, + startProxy: async (request) => { + // Fly Machine startup is explicit. Machine config and metadata only + // hold public identity/configuration; the token crosses stdin. + await options.machines.startMachine(input.app, machine.id); + await options.guest(machine).startProxy(request); + }, + }); + } catch (error) { + // Preserve the sanitized original provisioning failure while making a + // best-effort cleanup of a machine that never completed startup. + await options.machines.destroyMachine(input.app, machine.id).catch(() => undefined); + throw error; + } + return { proxyIdentity, bootEpoch, machine }; + }, + }; +} diff --git a/packages/api/test/fly-credential-proxy-provisioner.test.ts b/packages/api/test/fly-credential-proxy-provisioner.test.ts new file mode 100644 index 00000000..aeecf1ca --- /dev/null +++ b/packages/api/test/fly-credential-proxy-provisioner.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; +import { createFlyCredentialProxyProvisioner, createFlyCredentialProxyGuest } from "../src/fly-credential-proxy-provisioner"; + +const epoch = "11111111-1111-4111-8111-111111111111"; +const machine = { id: "machine-1", state: "created" }; +const input = { + tenantId: "11111111-1111-4111-8111-111111111112", + app: "factorize-proxies", + image: `ghcr.io/generationsoftware/credential-proxy@sha256:${"a".repeat(64)}`, + region: "iad", + size: "shared-cpu-1x", + controlOrigin: "https://factorize.example", + operationId: "tenant-proxy-start-1", +}; + +function fixture() { + const control = { + provisionCredential: vi.fn(async () => `fzc_${"b".repeat(43)}`), + revokeCredential: vi.fn(async () => true), + }; + const machines = { + createMachine: vi.fn(async () => machine), + startMachine: vi.fn(async () => undefined), + destroyMachine: vi.fn(async () => undefined), + }; + const guest = vi.fn(() => ({ startProxy: vi.fn(async () => undefined) })); + const provisioner = createFlyCredentialProxyProvisioner({ control, machines, guest, newBootEpoch: () => epoch }); + return { control, machines, guest, provisioner }; +} + +describe("Fly credential proxy startup", () => { + it("creates a digest-pinned dedicated Machine and delivers bootstrap only through startup", async () => { + const { control, machines, guest, provisioner } = fixture(); + const started = await provisioner.start(input); + expect(started).toMatchObject({ proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, machine }); + expect(machines.createMachine).toHaveBeenCalledWith(input.app, expect.objectContaining({ + image: input.image, + region: input.region, + cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", epoch], + restart: { policy: "no" }, + auto_start: false, + metadata: expect.objectContaining({ factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId }), + }), input.operationId); + expect(JSON.stringify(machines.createMachine.mock.calls[0])).not.toContain("fzc_"); + expect(machines.startMachine).toHaveBeenCalledWith(input.app, machine.id); + expect(control.provisionCredential).toHaveBeenCalledWith(input.tenantId, "fly:factorize-proxies:machine-1", epoch); + expect(guest).toHaveBeenCalledWith(machine); + const guestAdapter = guest.mock.results[0]!.value; + expect(guestAdapter.startProxy).toHaveBeenCalledWith(expect.objectContaining({ + tenantId: input.tenantId, + proxyIdentity: "fly:factorize-proxies:machine-1", + bootEpoch: epoch, + controlOrigin: input.controlOrigin, + credential: `fzc_${"b".repeat(43)}`, + })); + }); + + it("revokes bootstrap authority and removes a Machine when startup fails", async () => { + const { control, machines, guest } = fixture(); + guest.mockReturnValue({ startProxy: vi.fn().mockRejectedValue(new Error("private startup channel failed")) }); + const provisioner = createFlyCredentialProxyProvisioner({ control, machines, guest, newBootEpoch: () => epoch }); + await expect(provisioner.start(input)).rejects.toThrow("private startup channel failed"); + expect(control.revokeCredential).toHaveBeenCalledWith(input.tenantId, "fly:factorize-proxies:machine-1"); + expect(machines.destroyMachine).toHaveBeenCalledWith(input.app, machine.id); + }); + + it("rejects mutable images and unsafe control origins before provisioning", async () => { + const { machines, provisioner } = fixture(); + await expect(provisioner.start({ ...input, image: "ghcr.io/example/proxy:latest" })).rejects.toThrow("pinned by sha256 digest"); + await expect(provisioner.start({ ...input, controlOrigin: "http://factorize.example" })).rejects.toThrow("HTTPS origin"); + expect(machines.createMachine).not.toHaveBeenCalled(); + }); +}); + +describe("Fly credential proxy guest adapter", () => { + it("sends the bootstrap credential as stdin to the reviewed startup command", async () => { + const calls: unknown[][] = []; + const spawnProcess = ((command: string, args: string[], options: { stdio: string[] }) => { + calls.push([command, args, options]); + const stdoutHandlers: Array<(chunk: Buffer) => void> = []; + const child: any = { pid: undefined, stdin: { end: (value: string) => { calls.push(["stdin", value]); } }, + stdout: { on: (_event: string, callback: (chunk: Buffer) => void) => stdoutHandlers.push(callback) }, + stderr: { on: () => undefined }, on: (event: string, callback: (code: number) => void) => { + if (event === "close") queueMicrotask(() => { stdoutHandlers.forEach(handler => handler(Buffer.from("Credential proxy started\n"))); callback(0); }); + }, kill: () => undefined }; + return child; + }) as any; + const guest = createFlyCredentialProxyGuest({ app: input.app, machine: machine.id, token: "server-token-fixture", spawnProcess }); + await guest.startProxy({ tenantId: input.tenantId, proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, credential: `fzc_${"c".repeat(43)}` }); + expect(calls[0]?.[1]).toContain("/usr/local/bin/factorize-proxy-startup"); + expect(String(calls[1]?.[1])).toContain(`"credential":"fzc_${"c".repeat(43)}"`); + }); +}); diff --git a/packages/credential-proxy/src/bootstrap-control.mjs b/packages/credential-proxy/src/bootstrap-control.mjs new file mode 100644 index 00000000..80e02fec --- /dev/null +++ b/packages/credential-proxy/src/bootstrap-control.mjs @@ -0,0 +1,67 @@ +import { createServer } from "node:net"; +import { chmod, mkdir, rm } from "node:fs/promises"; +import { dirname } from "node:path"; +import { registerProxyBoot } from "./control-client.mjs"; + +const MAX_BOOTSTRAP_BYTES = 4096; + +/** Local-only receiver for a provider exec channel; credentials never enter env or files. */ +export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot } = {}) { + if (!runtime || typeof register !== "function") throw new TypeError("proxy bootstrap control requires a runtime and registrar"); + await mkdir(dirname(socketPath), { recursive: true, mode: 0o700 }); + await chmod(dirname(socketPath), 0o700); + await rm(socketPath, { force: true }); + const server = createServer((socket) => { + let data = Buffer.alloc(0); + let settled = false; + let processing = false; + const finish = (value) => { + if (settled) return; + settled = true; + socket.end(`${JSON.stringify(value)}\n`); + }; + socket.setTimeout(5000, () => finish({ error: "invalid bootstrap request" })); + socket.on("data", async (chunk) => { + if (settled || processing) return; + if (data.byteLength + chunk.byteLength > MAX_BOOTSTRAP_BYTES) return finish({ error: "invalid bootstrap request" }); + data = Buffer.concat([data, chunk]); + const end = data.indexOf(10); + if (end < 0) return; + if (end !== data.byteLength - 1) return finish({ error: "invalid bootstrap request" }); + let request; + try { request = JSON.parse(data.subarray(0, end).toString("utf8")); } + catch { return finish({ error: "invalid bootstrap request" }); } + if (!request || typeof request !== "object" || Array.isArray(request) + || Object.keys(request).sort().join(",") !== "bootEpoch,controlOrigin,credential" + || request.bootEpoch !== runtime.bootEpoch || typeof request.credential !== "string" + || !/^fzc_[A-Za-z0-9_-]{43}$/.test(request.credential)) { + return finish({ error: "invalid bootstrap request" }); + } + processing = true; + socket.pause(); + socket.removeAllListeners("data"); + let credential = request.credential; + try { + const registration = await register({ controlOrigin: request.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + finish({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); + } catch { + finish({ error: "proxy boot registration failed" }); + } finally { + credential = ""; + request.credential = ""; + } + }); + socket.on("error", () => {}); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, () => { server.removeListener("error", reject); resolve(); }); + }); + await chmod(socketPath, 0o600); + return Object.freeze({ + close: async () => { + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + await rm(socketPath, { force: true }); + }, + }); +} diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index f5409b16..01fce4c6 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -1,8 +1,13 @@ import { createInterface } from "node:readline"; +import { randomUUID } from "node:crypto"; import { createProxyRuntime } from "./runtime.mjs"; import { registerProxyBoot } from "./control-client.mjs"; +import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; -const runtime = createProxyRuntime(); +const epochIndex = process.argv.indexOf("--boot-epoch"); +const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; +const runtime = createProxyRuntime({ bootEpoch }); +const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET }); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); let closing = false; let shutdownDeadline; @@ -15,6 +20,7 @@ async function stop() { if (closing) return; closing = true; await runtime.shutdown(); + await bootstrapControl.close(); lines.close(); clearTimeout(shutdownDeadline); process.exit(0); diff --git a/packages/credential-proxy/src/startup.mjs b/packages/credential-proxy/src/startup.mjs new file mode 100644 index 00000000..1eb7d163 --- /dev/null +++ b/packages/credential-proxy/src/startup.mjs @@ -0,0 +1,46 @@ +import { connect } from "node:net"; +import { once } from "node:events"; + +const SOCKET_PATH = "/run/factorize-proxy/bootstrap.sock"; +const MAX_BYTES = 4096; + +async function readRequest() { + let bytes = 0; + const chunks = []; + for await (const chunk of process.stdin) { + bytes += chunk.byteLength; + if (bytes > MAX_BYTES) throw new Error("invalid bootstrap request"); + chunks.push(chunk); + } + const data = Buffer.concat(chunks); + if (!data.byteLength || data[data.byteLength - 1] !== 10) throw new Error("invalid bootstrap request"); + return data; +} + +async function main() { + const request = await readRequest(); + const socket = connect(SOCKET_PATH); + const result = new Promise((resolve, reject) => { + let data = Buffer.alloc(0); + socket.on("data", chunk => { + data = Buffer.concat([data, chunk]); + const end = data.indexOf(10); + if (end >= 0) { + try { resolve(JSON.parse(data.subarray(0, end).toString("utf8"))); } + catch { reject(new Error("invalid bootstrap response")); } + socket.end(); + } + }); + socket.once("error", () => reject(new Error("proxy startup control unavailable"))); + socket.setTimeout(7000, () => reject(new Error("proxy startup control timed out"))); + }); + socket.write(request); + const response = await result; + if (response.registered !== true) throw new Error("proxy boot registration failed"); + process.stdout.write("Credential proxy started\n"); +} + +main().catch(() => { + process.stderr.write("Credential proxy startup failed\n"); + process.exitCode = 1; +}); diff --git a/packages/credential-proxy/test/bootstrap-control.test.mjs b/packages/credential-proxy/test/bootstrap-control.test.mjs new file mode 100644 index 00000000..2cae7dfb --- /dev/null +++ b/packages/credential-proxy/test/bootstrap-control.test.mjs @@ -0,0 +1,56 @@ +import { strict as assert } from "node:assert"; +import { afterEach, test } from "node:test"; +import { connect } from "node:net"; +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { once } from "node:events"; +import { startProxyBootstrapControl } from "../src/bootstrap-control.mjs"; + +const epoch = "11111111-1111-4111-8111-111111111111"; +const token = `fzc_${"a".repeat(43)}`; +let directory; +let service; +afterEach(async () => { + await service?.close(); service = undefined; + if (directory) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +async function fixture(register) { + directory = await mkdtemp(join(tmpdir(), "factorize-proxy-startup-")); + const socketPath = join(directory, "control.sock"); + const runtime = { bootEpoch: epoch }; + service = await startProxyBootstrapControl({ runtime, socketPath, register }); + return socketPath; +} + +async function request(socketPath, payload) { + const socket = connect(socketPath); + let response = ""; + socket.on("data", chunk => { response += chunk.toString(); }); + await once(socket, "connect"); + socket.end(`${JSON.stringify(payload)}\n`); + await once(socket, "close"); + return JSON.parse(response); +} + +test("local startup control redeems the one-use credential and leaves a private socket", async () => { + const calls = []; + const socketPath = await fixture(async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch }; }); + const result = await request(socketPath, { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + assert.deepEqual(result, { registered: true, proxyIdentity: "fly:app:machine", bootEpoch: epoch }); + assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch }]); + assert.equal((await stat(socketPath)).mode & 0o777, 0o600); +}); + +test("local startup control rejects run credentials, epoch substitution, and extra fields", async () => { + const calls = []; + const socketPath = await fixture(async value => { calls.push(value); return {}; }); + for (const payload of [ + { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, + { controlOrigin: "https://factorize.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, + { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, + ]) assert.deepEqual(await request(socketPath, payload), { error: "invalid bootstrap request" }); + assert.deepEqual(calls, []); +}); diff --git a/packages/credential-proxy/test/runtime-process.test.mjs b/packages/credential-proxy/test/runtime-process.test.mjs index 03944bf9..87ebeb61 100644 --- a/packages/credential-proxy/test/runtime-process.test.mjs +++ b/packages/credential-proxy/test/runtime-process.test.mjs @@ -19,8 +19,8 @@ async function waitFor(predicate, label) { } } -async function startProxy() { - const child = spawn(process.execPath, [entry], { stdio: ["pipe", "pipe", "pipe"] }); +async function startProxy(directory) { + const child = spawn(process.execPath, [entry], { stdio: ["pipe", "pipe", "pipe"], env: { ...process.env, FACTORIZE_PROXY_BOOTSTRAP_SOCKET: join(directory, "bootstrap.sock") } }); let pending = ""; let log = ""; let stderr = ""; @@ -59,7 +59,7 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts try { await writeFile(join(directory, "identity.json"), JSON.stringify({ identity: "public-fixture", generation: 1 })); await writeFile(join(directory, "public-metadata.json"), JSON.stringify({ displayName: "fixture" })); - const first = await startProxy(); + const first = await startProxy(directory); processes.push(first); assert.deepEqual(await first.command({ type: "status" }), { type: "status", state: "locked" }); assert.deepEqual(await first.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked" }); @@ -67,7 +67,7 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts await waitFor(() => first.child.exitCode !== null || first.child.signalCode !== null, "first process stop"); assert.equal(first.child.exitCode, 0); - const restarted = await startProxy(); + const restarted = await startProxy(directory); processes.push(restarted); assert.notEqual(restarted.bootEpoch, first.bootEpoch, "every process restart gets a fresh enrollment epoch"); assert.deepEqual(await restarted.command({ type: "status" }), { type: "status", state: "locked" }); @@ -76,14 +76,14 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts await waitFor(() => restarted.child.exitCode !== null || restarted.child.signalCode !== null, "command shutdown"); assert.equal(restarted.child.exitCode, 0); - const crashed = await startProxy(); + const crashed = await startProxy(directory); processes.push(crashed); assert.deepEqual(await crashed.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked" }); crashed.child.kill("SIGKILL"); await waitFor(() => crashed.child.exitCode !== null || crashed.child.signalCode !== null, "crash stop"); assert.equal(crashed.child.signalCode, "SIGKILL"); - const afterCrash = await startProxy(); + const afterCrash = await startProxy(directory); processes.push(afterCrash); assert.deepEqual(await afterCrash.command({ type: "status" }), { type: "status", state: "locked" }); afterCrash.child.kill("SIGTERM"); From 207d68f47a93d61bd96a59e6f687c3902ee9d6dd Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 19:35:26 +0000 Subject: [PATCH 058/102] Retry proxy tunnel enrollment before forwarding --- .../credential-proxy/src/tunnel-transport.mjs | 93 +++++++++++++++---- .../test/proxy-listener.e2e.test.mjs | 30 +++--- 2 files changed, 91 insertions(+), 32 deletions(-) diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index 2ad21a67..8fe4bfa2 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -4,6 +4,9 @@ import { once } from "node:events"; const HANDSHAKE_LIMIT = 4096; const HANDSHAKE_TIMEOUT_MS = 10_000; +const CONTROL_RETRY_WINDOW_MS = 5 * 60_000; +const CONTROL_RETRY_MAX_MS = 30_000; +const CONTROL_RETRY_SLOW_MS = 60_000; function peerIdentity(socket) { const certificate = socket.getPeerCertificate(); @@ -62,6 +65,7 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro target = connectTcp(address.port, address.address); await once(target, "connect"); if (rest.length) target.write(rest); + socket.write("FACTORIZE-TUNNEL/1 ACCEPTED\n"); socket.pipe(target); target.pipe(socket); socket.resume(); @@ -94,32 +98,50 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro } /** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ -export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPort, host = "127.0.0.1", port = 0 } = {}) { +export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPort, host = "127.0.0.1", port = 0, + now = Date.now, random = Math.random, wait = delay } = {}) { if (!tls || typeof enrollment !== "function" || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { throw new TypeError("VM tunnel helper requires TLS, enrollment and proxy address"); } const sockets = new Set(); const server = createTcpServer((local) => { sockets.add(local); + local.pause(); void (async () => { - let remote; - try { - const grant = await enrollment(); - if (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token)) throw new Error("fresh enrollment unavailable"); - remote = connectTls({ ...tls, host: proxyHost, port: proxyPort, servername: tls.servername ?? proxyHost, rejectUnauthorized: true }); - sockets.add(remote); - const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); - try { await once(remote, "secureConnect"); } - finally { clearTimeout(connectTimer); } - remote.write(`FACTORIZE-TUNNEL/1 ${grant.token}\n`); - local.pipe(remote); - remote.pipe(local); - local.resume(); - remote.once("close", () => { sockets.delete(remote); local.destroy(); }); - local.once("close", () => remote.destroy()); - } catch { - local.destroy(); - remote?.destroy(); + const startedAt = now(); + let attempt = 0; + while (!local.destroyed) { + let remote; + try { + const grant = await enrollment(); + if (local.destroyed) return; + if (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token)) throw new Error("fresh enrollment unavailable"); + remote = connectTls({ ...tls, host: proxyHost, port: proxyPort, servername: tls.servername ?? proxyHost, rejectUnauthorized: true }); + sockets.add(remote); + const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); + try { await once(remote, "secureConnect"); } + finally { clearTimeout(connectTimer); } + remote.write(`FACTORIZE-TUNNEL/1 ${grant.token}\n`); + await waitForTunnelAcceptance(remote); + local.pipe(remote); + remote.pipe(local); + local.resume(); + remote.once("close", () => { sockets.delete(remote); local.destroy(); }); + local.once("close", () => remote.destroy()); + return; + } catch { + remote?.destroy(); + if (remote) sockets.delete(remote); + if (local.destroyed) return; + const elapsed = now() - startedAt; + const base = elapsed < CONTROL_RETRY_WINDOW_MS + ? Math.min(1000 * (2 ** Math.min(attempt, 5)), CONTROL_RETRY_MAX_MS) + : CONTROL_RETRY_SLOW_MS; + const jittered = Math.round(base * (0.5 + random())); + const delayMs = Math.max(1, elapsed < CONTROL_RETRY_WINDOW_MS ? Math.min(CONTROL_RETRY_MAX_MS, jittered) : jittered); + attempt++; + try { await wait(delayMs); } catch { local.destroy(); return; } + } } })(); const remove = () => { sockets.delete(local); }; @@ -137,3 +159,36 @@ export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPor }, }); } + +function delay(milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)); +} + +function waitForTunnelAcceptance(socket) { + return new Promise((resolve, reject) => { + let response = Buffer.alloc(0); + const cleanup = () => { + clearTimeout(timer); + socket.removeListener("data", onData); + socket.removeListener("error", onError); + socket.removeListener("close", onClose); + }; + const onError = error => { cleanup(); reject(error); }; + const onClose = () => { cleanup(); reject(new Error("proxy closed before accepting the tunnel")); }; + const onData = chunk => { + response = Buffer.concat([response, chunk]); + if (response.length > HANDSHAKE_LIMIT) { cleanup(); reject(new Error("invalid proxy tunnel response")); return; } + const end = response.indexOf(10); + if (end < 0) return; + const accepted = response.subarray(0, end).toString("utf8") === "FACTORIZE-TUNNEL/1 ACCEPTED"; + const trailingData = response.subarray(end + 1); + cleanup(); + if (!accepted || trailingData.length) reject(new Error("proxy tunnel was rejected")); + else resolve(); + }; + const timer = setTimeout(() => { cleanup(); socket.destroy(); reject(new Error("proxy tunnel acceptance timed out")); }, HANDSHAKE_TIMEOUT_MS); + socket.on("data", onData); + socket.once("error", onError); + socket.once("close", onClose); + }); +} diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index c8c6a940..70a862df 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -140,6 +140,8 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante let observedAuthorization; let enrollmentCount = 0; let authenticationCount = 0; + let mismatchReachedResolve; + const mismatchReached = new Promise(resolve => { mismatchReachedResolve = resolve; }); const upstream = createHttpsServer({ key, cert }, (request, response) => { observedAuthorization = request.headers.authorization; response.end("transport-ok"); @@ -162,13 +164,15 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante runtime, authenticateEnrollment: async ({ token, vmIdentity, bootEpoch }) => { authenticationCount++; - assert.equal(token, (authenticationCount === 1 ? "A" : authenticationCount === 2 ? "B" : "C").repeat(43)); + assert.ok(["A", "B", "C", "X"].some(value => token === value.repeat(43))); assert.equal(vmIdentity, "api.example.test"); assert.equal(bootEpoch, runtime.bootEpoch); + if (token === "A".repeat(43)) throw new Error("temporary control-plane outage"); + if (token === "X".repeat(43)) mismatchReachedResolve(); return { authenticated: true, enrollmentConsumed: true, - vmIdentity: token === "C".repeat(43) ? "other-vm" : vmIdentity, + vmIdentity: token === "X".repeat(43) ? "other-vm" : vmIdentity, bootEpoch, connectionId: "transport-enrollment-1", runId: "run-transport-1", @@ -181,9 +185,10 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante runtime.unlock("synthetic-unlock-key"); const local = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, - enrollment: async () => ({ token: (++enrollmentCount === 1 ? "A" : enrollmentCount === 2 ? "B" : "C").repeat(43) }), + enrollment: async () => ({ token: ["A", "B", "C"][enrollmentCount++ % 3].repeat(43) }), proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, + wait: async () => {}, }); let mismatchLocal; try { @@ -195,21 +200,20 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante assert.equal(observedAuthorization, "Bearer synthetic-bearer"); const denied = await fetch(`http://127.0.0.1:${local.address.port}/integrations/other-tool/echo`); assert.equal(denied.status, 403); - assert.equal(enrollmentCount, 2, "each local connection gets fresh one-use enrollment"); - assert.equal(authenticationCount, 2, "each enrollment is independently consumed by the proxy transport"); + assert.ok(enrollmentCount >= 3, "failed setup and local connections get fresh one-use enrollment"); + assert.equal(enrollmentCount, authenticationCount, "each enrollment is independently consumed by the proxy transport"); mismatchLocal = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, - enrollment: async () => ({ token: (++enrollmentCount === 3 ? "C" : "D").repeat(43) }), + enrollment: async () => ({ token: (enrollmentCount++, "X".repeat(43)) }), proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, }); - await assert.rejects( - fetch(`http://127.0.0.1:${mismatchLocal.address.port}/integrations/tool-1/echo`), - /fetch failed/, - "mismatched VM binding is rejected", - ); - assert.equal(enrollmentCount, 3); - assert.equal(authenticationCount, 3); + const rejectedRequest = fetch(`http://127.0.0.1:${mismatchLocal.address.port}/integrations/tool-1/echo`); + await mismatchReached; + await mismatchLocal.close(); + mismatchLocal = undefined; + await assert.rejects(rejectedRequest, /fetch failed/, "mismatched VM binding is rejected"); + assert.equal(authenticationCount, enrollmentCount); } finally { await mismatchLocal?.close(); await local.close(); From 497f4bd6e6ea74a0f46becbf89a34fd61a1b18fd Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 20:04:51 +0000 Subject: [PATCH 059/102] Add authenticated proxy enrollment control API --- .../0037_proxy_control_credentials.sql | 2 + .../src/postgres/proxy-control-repository.ts | 22 +++++++--- packages/api/src/postgres/schema.ts | 2 +- packages/api/src/proxy-control.ts | 30 ++++++++++--- .../api/test/proxy-control-repository.test.ts | 4 +- packages/api/test/proxy-control.test.ts | 43 +++++++++++++++++-- .../credential-proxy/src/control-client.mjs | 34 +++++++++++++-- packages/credential-proxy/src/main.mjs | 4 +- .../test/control-client.test.mjs | 27 ++++++++++-- 9 files changed, 143 insertions(+), 25 deletions(-) create mode 100644 packages/api/migrations/0037_proxy_control_credentials.sql diff --git a/packages/api/migrations/0037_proxy_control_credentials.sql b/packages/api/migrations/0037_proxy_control_credentials.sql new file mode 100644 index 00000000..b3c69e79 --- /dev/null +++ b/packages/api/migrations/0037_proxy_control_credentials.sql @@ -0,0 +1,2 @@ +ALTER TABLE app.credential_proxy_control_credentials + ADD COLUMN proxy_token_digest bytea UNIQUE; diff --git a/packages/api/src/postgres/proxy-control-repository.ts b/packages/api/src/postgres/proxy-control-repository.ts index 2dee38ae..09fd28c9 100644 --- a/packages/api/src/postgres/proxy-control-repository.ts +++ b/packages/api/src/postgres/proxy-control-repository.ts @@ -1,6 +1,6 @@ import type { Database } from "./database"; -export type RegisteredProxy = { tenantId: string; proxyIdentity: string; bootEpoch: string }; +export type RegisteredProxy = { tenantId: string; proxyIdentity: string; bootEpoch: string; controlCredential: string }; const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; const digest = async (token: string) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(token))); @@ -13,7 +13,7 @@ export class ProxyControlRepository { if (!uuid.test(tenantId) || !proxyIdentity.trim() || proxyIdentity.length > 255 || !uuid.test(bootEpoch)) throw new TypeError("Invalid proxy identity or boot epoch"); const token = `fzc_${Buffer.from(crypto.getRandomValues(new Uint8Array(32))).toString("base64url")}`; await this.database.pool.query(`INSERT INTO app.credential_proxy_control_credentials(tenant_id,proxy_identity,boot_epoch,token_digest,revoked_at,consumed_at) - VALUES ($1,$2,$3,$4,NULL,NULL) ON CONFLICT (tenant_id,proxy_identity) DO UPDATE SET boot_epoch=excluded.boot_epoch,token_digest=excluded.token_digest,revoked_at=NULL,consumed_at=NULL,created_at=now()`, + VALUES ($1,$2,$3,$4,NULL,NULL) ON CONFLICT (tenant_id,proxy_identity) DO UPDATE SET boot_epoch=excluded.boot_epoch,token_digest=excluded.token_digest,proxy_token_digest=NULL,revoked_at=NULL,consumed_at=NULL,created_at=now()`, [tenantId, proxyIdentity, bootEpoch, Buffer.from(await digest(token))]); return token; } @@ -40,14 +40,26 @@ export class ProxyControlRepository { AND app.credential_proxy_states.disabled_at IS NULL AND app.credential_proxy_states.revoked_at IS NULL RETURNING tenant_id`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); if (!saved.rowCount) return null; - await client.query(`UPDATE app.credential_proxy_control_credentials SET consumed_at=now() - WHERE token_digest=$1 AND boot_epoch=$2 AND consumed_at IS NULL`, [tokenDigest, bootEpoch]); + const proxyToken = `fzc_${Buffer.from(crypto.getRandomValues(new Uint8Array(32))).toString("base64url")}`; + await client.query(`UPDATE app.credential_proxy_control_credentials SET consumed_at=now(),proxy_token_digest=$3 + WHERE token_digest=$1 AND boot_epoch=$2 AND consumed_at IS NULL`, [tokenDigest, bootEpoch, Buffer.from(await digest(proxyToken))]); await client.query(`UPDATE app.credential_run_grants SET revoked_at=now() WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch<>$3 AND revoked_at IS NULL`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); - return { tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch }; + return { tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch, controlCredential: proxyToken }; }); } + /** Authenticates the proxy's memory-held control credential against its current registered boot. */ + async authenticateProxy(token: string): Promise | null> { + if (typeof token !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return null; + const result = await this.database.pool.query<{ tenant_id: string; proxy_identity: string; boot_epoch: string }>(`SELECT c.tenant_id,c.proxy_identity,c.boot_epoch + FROM app.credential_proxy_control_credentials c JOIN app.credential_proxy_states p + ON p.tenant_id=c.tenant_id AND p.proxy_identity=c.proxy_identity AND p.boot_epoch=c.boot_epoch + WHERE c.proxy_token_digest=$1 AND c.revoked_at IS NULL AND p.disabled_at IS NULL AND p.revoked_at IS NULL`, [Buffer.from(await digest(token))]); + const row = result.rows[0]; + return row ? { tenantId: row.tenant_id, proxyIdentity: row.proxy_identity, bootEpoch: row.boot_epoch } : null; + } + async revokeCredential(tenantId: string, proxyIdentity: string): Promise { const result = await this.database.pool.query(`UPDATE app.credential_proxy_control_credentials SET revoked_at=now() WHERE tenant_id=$1 AND proxy_identity=$2 AND revoked_at IS NULL`, [tenantId, proxyIdentity]); diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 57b492ba..f64f2d72 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -53,7 +53,7 @@ export const integrationChanges = app.table("integration_changes", { export const credentialProxyStates = app.table("credential_proxy_states", { tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), disabledAt: timestamp("disabled_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), updatedAt: updatedAt(), }); -export const credentialProxyControlCredentials = app.table("credential_proxy_control_credentials", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), tokenDigest: bytea("token_digest").notNull().unique(), revokedAt: timestamp("revoked_at", { withTimezone: true }), consumedAt: timestamp("consumed_at", { withTimezone: true }), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.proxyIdentity] })]); +export const credentialProxyControlCredentials = app.table("credential_proxy_control_credentials", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), tokenDigest: bytea("token_digest").notNull().unique(), proxyTokenDigest: bytea("proxy_token_digest").unique(), revokedAt: timestamp("revoked_at", { withTimezone: true }), consumedAt: timestamp("consumed_at", { withTimezone: true }), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.proxyIdentity] })]); export const credentialRunGrants = app.table("credential_run_grants", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), vmIdentity: text("vm_identity").notNull(), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), grantRevision: bigint("grant_revision", { mode: "number" }).notNull(), revokedAt: timestamp("revoked_at", { withTimezone: true }), createdAt: createdAt(), }, t => [primaryKey({ columns: [t.tenantId, t.runId] }), foreignKey({ columns: [t.tenantId, t.runId], foreignColumns: [jobRuns.tenantId, jobRuns.id] }).onDelete("cascade"), check("credential_run_grants_revision_check", sql`${t.grantRevision} > 0`)]); diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index 3cd1632b..695779cc 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -1,12 +1,13 @@ import { databaseFor } from "./postgres/database"; import { ProxyControlRepository } from "./postgres/proxy-control-repository"; +import { RunEnrollmentRepository } from "./postgres/run-enrollment-repository"; const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "Cache-Control": "no-store" } }); /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (url.pathname !== "/internal/credential-proxy/register-boot") return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; @@ -28,11 +29,26 @@ export async function proxyControlFetch(request: Request, env: Parameters key !== "bootEpoch") - || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); const repository = new ProxyControlRepository(databaseFor(env)); - const registration = await repository.registerBoot(token, (body as any).bootEpoch); - if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); - return json({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); + if (url.pathname === "/internal/credential-proxy/register-boot") { + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => key !== "bootEpoch") || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); + const registration = await repository.registerBoot(token, (body as any).bootEpoch); + if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); + return json({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch, controlCredential: registration.controlCredential }); + } + + const proxy = await repository.authenticateProxy(token); + if (!proxy) return json({ error: "unauthorized" }, 401); + const enrollments = new RunEnrollmentRepository(databaseFor(env)); + if (url.pathname === "/internal/credential-proxy/enroll") { + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity"].includes(key)) + || typeof (body as any).runId !== "string" || typeof (body as any).vmIdentity !== "string") return json({ error: "invalid_request" }, 400); + const enrollment = await enrollments.issue({ tenantId: proxy.tenantId, runId: (body as any).runId, vmIdentity: (body as any).vmIdentity, proxyIdentity: proxy.proxyIdentity, bootEpoch: proxy.bootEpoch }, new Date(Date.now() + 30_000)); + return enrollment ? json(enrollment) : json({ error: "enrollment_unavailable" }, 403); + } + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity", "enrollmentToken"].includes(key)) + || typeof (body as any).runId !== "string" || typeof (body as any).vmIdentity !== "string" || typeof (body as any).enrollmentToken !== "string") return json({ error: "invalid_request" }, 400); + const binding = { tenantId: proxy.tenantId, runId: (body as any).runId, vmIdentity: (body as any).vmIdentity, proxyIdentity: proxy.proxyIdentity, bootEpoch: proxy.bootEpoch }; + const consumed = await enrollments.consume((body as any).enrollmentToken, binding); + return consumed ? json({ consumed: true, binding: consumed }) : json({ error: "enrollment_unavailable" }, 403); } diff --git a/packages/api/test/proxy-control-repository.test.ts b/packages/api/test/proxy-control-repository.test.ts index 20f1934c..9508486a 100644 --- a/packages/api/test/proxy-control-repository.test.ts +++ b/packages/api/test/proxy-control-repository.test.ts @@ -33,7 +33,9 @@ describe("dedicated credential proxy control identity", () => { const tx = { query: vi.fn().mockImplementation((sql: string) => sql.includes("credential_proxy_control_credentials") ? credentialQuery() : sql.includes("INSERT INTO app.credential_proxy_states") ? stateQuery() : Promise.resolve({ rows: [], rowCount: 1 })) }; (repository as any).database = { transaction: (work: (client: any) => Promise) => work(tx) }; const epoch = "22222222-2222-4222-8222-222222222222"; - expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch)).toEqual({ tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch: epoch }); + const registration = await repository.registerBoot("fzc_" + "a".repeat(43), epoch); + expect(registration).toMatchObject({ tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch: epoch }); + expect(registration?.controlCredential).toMatch(/^fzc_[A-Za-z0-9_-]{43}$/); expect(tx.query.mock.calls.map(call => call[0])).toHaveLength(4); expect(tx.query.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(tx.query.mock.calls[2][0]).toContain("consumed_at=now()"); diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 734cfff4..c0a33b1e 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -10,11 +10,14 @@ function envFor({ credentialFound = true, stateSaved = true } = {}) { if (sql.includes("INSERT INTO app.credential_proxy_states")) return { rows: stateSaved ? [{ tenant_id: "11111111-1111-4111-8111-111111111111" }] : [], rowCount: stateSaved ? 1 : 0 }; return { rows: [], rowCount: 1 }; }); - return { DATABASE: { pool: { query: vi.fn() }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery } as any; + const query = vi.fn(async (sql: string) => sql.includes("proxy_token_digest") + ? { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }] : [], rowCount: credentialFound ? 1 : 0 } + : { rows: [], rowCount: 0 }); + return { DATABASE: { pool: { query }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery, query } as any; } -function request({ authorization, body = { bootEpoch }, method = "POST" }: { authorization?: string; body?: unknown; method?: string } = {}) { - return new Request("https://factorize.example/internal/credential-proxy/register-boot", { method, headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}) }, body: method === "POST" ? JSON.stringify(body) : undefined }); +function request({ authorization, body = { bootEpoch }, method = "POST", path = "/internal/credential-proxy/register-boot" }: { authorization?: string; body?: unknown; method?: string; path?: string } = {}) { + return new Request(`https://factorize.example${path}`, { method, headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}) }, body: method === "POST" ? JSON.stringify(body) : undefined }); } describe("dedicated proxy boot registration endpoint", () => { @@ -22,7 +25,7 @@ describe("dedicated proxy boot registration endpoint", () => { const env = envFor(); const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); expect(response.status).toBe(200); - expect(await response.json()).toEqual({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch }); + expect(await response.json()).toMatchObject({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: expect.stringMatching(/^fzc_[A-Za-z0-9_-]{43}$/) }); expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(4); expect(env.transactionQuery.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(env.transactionQuery.mock.calls[2][0]).toContain("consumed_at=now()"); @@ -55,4 +58,36 @@ describe("dedicated proxy boot registration endpoint", () => { expect(response.status).toBe(413); expect(env.transactionQuery).not.toHaveBeenCalled(); }); + + it("issues a short-lived enrollment only through the registered proxy identity", async () => { + const env = envFor(); + env.transactionQuery.mockImplementation(async (sql: string) => sql.includes("SELECT g.grant_revision") + ? { rows: [{ grant_revision: 3 }], rowCount: 1 } + : { rows: [], rowCount: 1 }); + const response = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/enroll", + body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123" }, + }), env); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ grantRevision: 3, enrollmentId: expect.any(String), token: expect.any(String) }); + expect(env.query).toHaveBeenCalled(); + expect(env.transactionQuery.mock.calls[0][0]).toContain("JOIN app.credential_proxy_states"); + }); + + it("consumes enrollment against the authenticated proxy boot and rejects untrusted extra bindings", async () => { + const env = envFor(); + env.transactionQuery.mockImplementation(async (sql: string) => sql.includes("UPDATE app.credential_run_enrollments") + ? { rows: [{ id: "44444444-4444-4444-8444-444444444444", grant_revision: 3 }], rowCount: 1 } + : { rows: [], rowCount: 1 }); + const accepted = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/consume-enrollment", + body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123", enrollmentToken: "e".repeat(43) }, + }), env); + expect(accepted.status).toBe(200); + expect(await accepted.json()).toMatchObject({ consumed: true, binding: { bootEpoch, proxyIdentity: "proxy-fixture", vmIdentity: "fly:vm-123" } }); + const rejected = await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123", tenantId: "other" } }), env); + expect(rejected.status).toBe(400); + }); }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 77a430e2..08b0358f 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -1,9 +1,14 @@ const bootEpochPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; /** Register one proxy process boot using a provisioning-scoped credential. The token is only held in this call's memory. */ -export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetchImpl = fetch }) { +function validateOrigin(controlOrigin) { const origin = new URL(controlOrigin); if (origin.protocol !== "https:" || origin.pathname !== "/" || origin.username || origin.password || origin.search || origin.hash) throw new TypeError("proxy control origin must be an HTTPS origin"); + return origin; +} + +export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetchImpl = fetch }) { + const origin = validateOrigin(controlOrigin); if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token ?? "")) throw new TypeError("invalid proxy control credential"); if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); let credential = token; @@ -17,9 +22,32 @@ export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetch }); if (!response.ok) throw new Error("proxy boot registration rejected"); const result = await response.json(); - if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string") throw new Error("invalid proxy boot registration response"); - return { proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch }; + if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(result.controlCredential ?? "")) throw new Error("invalid proxy boot registration response"); + return { proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch, controlCredential: result.controlCredential }; } finally { credential = ""; } } + +async function proxyControlRequest({ controlOrigin, token, path, body, fetchImpl }) { + const origin = validateOrigin(controlOrigin); + if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token ?? "")) throw new TypeError("invalid proxy control credential"); + const response = await fetchImpl(new URL(path, origin), { + method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, + body: JSON.stringify(body), signal: AbortSignal.timeout(10_000), redirect: "error", + }); + if (!response.ok) throw new Error("proxy control request rejected"); + return response.json(); +} + +export async function issueProxyEnrollment(options) { + const result = await proxyControlRequest({ ...options, path: "/internal/credential-proxy/enroll" }); + if (typeof result?.token !== "string" || typeof result?.enrollmentId !== "string" || !Number.isSafeInteger(result?.grantRevision)) throw new Error("invalid proxy enrollment response"); + return result; +} + +export async function consumeProxyEnrollment(options) { + const result = await proxyControlRequest({ ...options, path: "/internal/credential-proxy/consume-enrollment" }); + if (result?.consumed !== true || typeof result?.binding?.enrollmentId !== "string") throw new Error("invalid consumed enrollment response"); + return result.binding; +} diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 01fce4c6..3a7750ae 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -11,6 +11,7 @@ const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); let closing = false; let shutdownDeadline; +let controlCredential; function respond(value) { process.stdout.write(`${JSON.stringify(value)}\n`); @@ -49,7 +50,8 @@ lines.on("line", async (line) => { command.credential = undefined; try { const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); - respond({ type: "boot-registered", ...registration }); + controlCredential = registration.controlCredential; + respond({ type: "boot-registered", proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { respond({ type: "error", error: "proxy boot registration failed" }); } finally { credential = ""; } diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 7f59aba7..56cc2b26 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { registerProxyBoot } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, issueProxyEnrollment, registerProxyBoot } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -9,9 +9,9 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an let request; const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async (url, init) => { request = { url: String(url), init }; - return Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch }); + return Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); } }); - assert.deepEqual(result, { proxyIdentity: "proxy-fixture", bootEpoch }); + assert.deepEqual(result, { proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); assert.equal(request.url, "https://factorize.example/internal/credential-proxy/register-boot"); assert.equal(request.init.headers.Authorization, `Bearer ${token}`); assert.equal(request.init.redirect, "error"); @@ -23,3 +23,24 @@ test("boot registration rejects unsafe origins, invalid credentials and mismatch await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async () => Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch: "33333333-3333-4333-8333-333333333333" }) }), /invalid proxy boot registration response/); }); + +test("proxy control client requests and consumes a one-use enrollment over authenticated HTTPS", async () => { + const controlToken = `fzc_${"c".repeat(43)}`; + const requests = []; + const fetchImpl = async (url, init) => { + requests.push({ url: String(url), init }); + return requests.length === 1 + ? Response.json({ enrollmentId: "enrollment-1", token: "e".repeat(43), grantRevision: 2 }) + : Response.json({ consumed: true, binding: { enrollmentId: "enrollment-1" } }); + }; + const common = { controlOrigin: "https://factorize.example", token: controlToken, fetchImpl }; + const enrollment = await issueProxyEnrollment({ ...common, body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1" } }); + assert.equal(enrollment.token, "e".repeat(43)); + const consumed = await consumeProxyEnrollment({ ...common, body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1", enrollmentToken: enrollment.token } }); + assert.equal(consumed.enrollmentId, "enrollment-1"); + assert.deepEqual(requests.map(request => request.url), [ + "https://factorize.example/internal/credential-proxy/enroll", + "https://factorize.example/internal/credential-proxy/consume-enrollment", + ]); + assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${controlToken}` && request.init.redirect === "error")); +}); From 516938c09977055a7a517d48b132ce43ccfaf11d Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 20:26:19 +0000 Subject: [PATCH 060/102] Wire proxy-side run enrollment into tunnel auth --- .../credential-proxy/src/control-client.mjs | 45 ++++++++++++++ .../credential-proxy/src/tunnel-transport.mjs | 23 +++++--- .../test/control-client.test.mjs | 58 ++++++++++++++++++- .../test/proxy-listener.e2e.test.mjs | 44 ++++++++++++++ 4 files changed, 160 insertions(+), 10 deletions(-) diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 08b0358f..b5216f10 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -51,3 +51,48 @@ export async function consumeProxyEnrollment(options) { if (result?.consumed !== true || typeof result?.binding?.enrollmentId !== "string") throw new Error("invalid consumed enrollment response"); return result.binding; } + +const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * Create the proxy-side half of enrollment. The job VM supplies only its run + * ID over the already authenticated TLS connection; the proxy derives VM and + * boot identity locally, then issues and consumes a fresh one-use enrollment + * using its private control credential. Integration secrets stay in the + * proxy's local grant resolver and never cross this control API. + */ +export function createProxyEnrollmentAuthenticator({ controlOrigin, controlCredential, bootEpoch, resolveGrants, fetchImpl = fetch } = {}) { + if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(controlCredential ?? "")) throw new TypeError("invalid proxy control credential"); + if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); + if (typeof resolveGrants !== "function") throw new TypeError("proxy enrollment requires a local grant resolver"); + validateOrigin(controlOrigin); + let credential = controlCredential; + return async function authenticateRun({ runId, vmIdentity, bootEpoch: requestedBootEpoch }) { + if (!uuidPattern.test(runId ?? "") || typeof vmIdentity !== "string" || !vmIdentity.trim() || vmIdentity.length > 255 + || requestedBootEpoch !== bootEpoch) throw new Error("invalid proxy enrollment identity"); + const options = { controlOrigin, token: credential, fetchImpl }; + const requested = { runId, vmIdentity }; + const enrollment = await issueProxyEnrollment({ ...options, body: requested }); + const binding = await consumeProxyEnrollment({ ...options, body: { ...requested, enrollmentToken: enrollment.token } }); + if (binding.runId !== runId || binding.vmIdentity !== vmIdentity || binding.bootEpoch !== bootEpoch + || typeof binding.tenantId !== "string" || typeof binding.proxyIdentity !== "string" + || binding.enrollmentId !== enrollment.enrollmentId || binding.grantRevision !== enrollment.grantRevision) { + throw new Error("consumed enrollment binding did not match the authenticated connection"); + } + const grants = await resolveGrants(binding); + if (!Array.isArray(grants)) throw new Error("proxy grants are unavailable"); + return { + authenticated: true, + enrollmentConsumed: true, + connectionId: binding.enrollmentId, + runId: binding.runId, + vmId: binding.vmIdentity, + vmIdentity: binding.vmIdentity, + bootEpoch: binding.bootEpoch, + tenantId: binding.tenantId, + proxyIdentity: binding.proxyIdentity, + grantRevision: binding.grantRevision, + grants, + }; + }; +} diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index 8fe4bfa2..c68f5857 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -19,8 +19,8 @@ function peerIdentity(socket) { * enrollment, and bridge that stream to the connection-scoped HTTP listener. * The bridge does not expose a shell or arbitrary destination forwarding. */ -export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnrollment, host = "127.0.0.1", port = 0 } = {}) { - if (!tls || !runtime || typeof authenticateEnrollment !== "function") throw new TypeError("tunnel server requires TLS, runtime and enrollment authentication"); +export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnrollment, authenticateRun, host = "127.0.0.1", port = 0 } = {}) { + if (!tls || !runtime || (typeof authenticateEnrollment !== "function" && typeof authenticateRun !== "function")) throw new TypeError("tunnel server requires TLS, runtime and enrollment authentication"); const connections = new Set(); const server = createTlsServer({ ...tls, requestCert: true, rejectUnauthorized: true }, (socket) => { connections.add(socket); @@ -46,11 +46,15 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro const line = handshake.subarray(0, end).toString("utf8").replace(/\r$/, ""); const rest = handshake.subarray(end + 1); const match = /^FACTORIZE-TUNNEL\/1 ([A-Za-z0-9_-]{32,128})$/.exec(line); - if (!match) return reject(); + const runMatch = /^FACTORIZE-TUNNEL\/2 ([0-9a-f-]{36})$/i.exec(line); + if ((!match || typeof authenticateEnrollment !== "function") && (!runMatch || typeof authenticateRun !== "function")) return reject(); let listener; let target; try { - const connection = await authenticateEnrollment({ token: match[1], vmIdentity: peerIdentity(socket), bootEpoch: runtime.bootEpoch }); + const vmIdentity = peerIdentity(socket); + const connection = runMatch + ? await authenticateRun({ runId: runMatch[1], vmIdentity, bootEpoch: runtime.bootEpoch }) + : await authenticateEnrollment({ token: match[1], vmIdentity, bootEpoch: runtime.bootEpoch }); // The production control plane returns the consumed enrollment binding. // Compare its VM and boot identities with the authenticated TLS peer and // this process so an adapter cannot accidentally attach another run's @@ -98,9 +102,10 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro } /** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ -export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPort, host = "127.0.0.1", port = 0, +export async function startVmTunnelHelper({ tls, enrollment, runId, proxyHost, proxyPort, host = "127.0.0.1", port = 0, now = Date.now, random = Math.random, wait = delay } = {}) { - if (!tls || typeof enrollment !== "function" || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { + const runBasedEnrollment = typeof runId === "string" && /^[0-9a-f-]{36}$/i.test(runId); + if (!tls || (!runBasedEnrollment && typeof enrollment !== "function") || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { throw new TypeError("VM tunnel helper requires TLS, enrollment and proxy address"); } const sockets = new Set(); @@ -113,15 +118,15 @@ export async function startVmTunnelHelper({ tls, enrollment, proxyHost, proxyPor while (!local.destroyed) { let remote; try { - const grant = await enrollment(); + const grant = runBasedEnrollment ? undefined : await enrollment(); if (local.destroyed) return; - if (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token)) throw new Error("fresh enrollment unavailable"); + if (!runBasedEnrollment && (!grant || typeof grant.token !== "string" || !/^[A-Za-z0-9_-]{32,128}$/.test(grant.token))) throw new Error("fresh enrollment unavailable"); remote = connectTls({ ...tls, host: proxyHost, port: proxyPort, servername: tls.servername ?? proxyHost, rejectUnauthorized: true }); sockets.add(remote); const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); try { await once(remote, "secureConnect"); } finally { clearTimeout(connectTimer); } - remote.write(`FACTORIZE-TUNNEL/1 ${grant.token}\n`); + remote.write(runBasedEnrollment ? `FACTORIZE-TUNNEL/2 ${runId}\n` : `FACTORIZE-TUNNEL/1 ${grant.token}\n`); await waitForTunnelAcceptance(remote); local.pipe(remote); remote.pipe(local); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 56cc2b26..16e0ab93 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, issueProxyEnrollment, registerProxyBoot } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, registerProxyBoot } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -44,3 +44,59 @@ test("proxy control client requests and consumes a one-use enrollment over authe ]); assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${controlToken}` && request.init.redirect === "error")); }); + +test("proxy enrollment authenticator binds issued and consumed state to the mTLS VM and current boot", async () => { + const controlToken = `fzc_${"d".repeat(43)}`; + const requests = []; + const grants = [{ id: "tool-1", origin: "https://api.example.test", credential: "synthetic-secret" }]; + const fetchImpl = async (url, init) => { + const request = { url: String(url), init, body: JSON.parse(init.body) }; + requests.push(request); + if (requests.length === 1) return Response.json({ enrollmentId: "enrollment-1", token: "e".repeat(43), grantRevision: 7 }); + return Response.json({ consumed: true, binding: { + tenantId: "11111111-1111-4111-8111-111111111111", + runId: request.body.runId, + vmIdentity: request.body.vmIdentity, + proxyIdentity: "fly:proxy-1", + bootEpoch, + enrollmentId: "enrollment-1", + grantRevision: 7, + } }); + }; + const authenticateRun = createProxyEnrollmentAuthenticator({ + controlOrigin: "https://factorize.example", + controlCredential: controlToken, + bootEpoch, + fetchImpl, + resolveGrants: async binding => { + assert.equal(binding.runId, "33333333-3333-4333-8333-333333333333"); + return grants; + }, + }); + const connection = await authenticateRun({ + runId: "33333333-3333-4333-8333-333333333333", + vmIdentity: "fly:vm-1", + bootEpoch, + }); + assert.deepEqual(requests.map(request => request.url), [ + "https://factorize.example/internal/credential-proxy/enroll", + "https://factorize.example/internal/credential-proxy/consume-enrollment", + ]); + assert.deepEqual(requests.map(request => request.body), [ + { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1" }, + { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1", enrollmentToken: "e".repeat(43) }, + ]); + assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${controlToken}`)); + assert.equal(connection.authenticated, true); + assert.equal(connection.enrollmentConsumed, true); + assert.equal(connection.vmId, "fly:vm-1"); + assert.equal(connection.bootEpoch, bootEpoch); + assert.equal(connection.connectionId, "enrollment-1"); + assert.deepEqual(connection.grants, grants); + await assert.rejects(authenticateRun({ + runId: "33333333-3333-4333-8333-333333333333", + vmIdentity: "fly:vm-1", + bootEpoch: "44444444-4444-4444-8444-444444444444", + }), /invalid proxy enrollment identity/); + assert.equal(requests.length, 2, "a stale boot cannot issue another enrollment"); +}); diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 70a862df..0236b0b7 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -222,3 +222,47 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); } }); + +test("VM helper sends only run identity while the proxy performs authenticated enrollment", { timeout: 5_000 }, async () => { + const runtime = createProxyRuntime(); + const authenticatedRuns = []; + const tunnel = createAuthenticatedTunnelServer({ + tls: { key, cert, ca: cert }, + runtime, + authenticateRun: async ({ runId, vmIdentity, bootEpoch }) => { + authenticatedRuns.push({ runId, vmIdentity, bootEpoch }); + return { + authenticated: true, + enrollmentConsumed: true, + connectionId: "enrollment-from-control-api", + runId, + vmId: vmIdentity, + vmIdentity, + bootEpoch, + grants: [], + }; + }, + }); + await tunnel.listening; + runtime.unlock("synthetic-unlock-key"); + const local = await startVmTunnelHelper({ + tls: { key, cert, ca: cert, servername: "api.example.test" }, + runId: "33333333-3333-4333-8333-333333333333", + proxyHost: "127.0.0.1", + proxyPort: tunnel.address().port, + }); + try { + const response = await fetch(`http://127.0.0.1:${local.address.port}/unrecognized`); + assert.equal(response.status, 404); + assert.equal(authenticatedRuns.length, 1); + assert.deepEqual(authenticatedRuns[0], { + runId: "33333333-3333-4333-8333-333333333333", + vmIdentity: "api.example.test", + bootEpoch: runtime.bootEpoch, + }); + } finally { + await local.close(); + await tunnel.close(); + await runtime.shutdown(); + } +}); From 8c35922a76cb41417b69c4e94ab2ffa5eb35ab27 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 20:42:29 +0000 Subject: [PATCH 061/102] Wire proxy control enrollment into tunnel startup --- .../credential-proxy/src/tunnel-transport.mjs | 19 +++++++ .../test/proxy-listener.e2e.test.mjs | 55 +++++++++++-------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index c68f5857..e670f554 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -1,6 +1,7 @@ import { connect as connectTls, createServer as createTlsServer } from "node:tls"; import { connect as connectTcp, createServer as createTcpServer } from "node:net"; import { once } from "node:events"; +import { createProxyEnrollmentAuthenticator } from "./control-client.mjs"; const HANDSHAKE_LIMIT = 4096; const HANDSHAKE_TIMEOUT_MS = 10_000; @@ -101,6 +102,24 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro }; } +/** Start the proxy's production-shaped mTLS listener using its boot-registered + * control credential and process-local decrypted integration grants. */ +export async function startAuthenticatedProxyTunnel({ + tls, runtime, controlOrigin, controlCredential, resolveGrants, fetchImpl = fetch, host = "127.0.0.1", port = 0, +} = {}) { + const authenticateRun = createProxyEnrollmentAuthenticator({ + controlOrigin, controlCredential, bootEpoch: runtime?.bootEpoch, resolveGrants, fetchImpl, + }); + const tunnel = createAuthenticatedTunnelServer({ tls, runtime, authenticateRun, host, port }); + try { + await tunnel.listening; + return tunnel; + } catch (error) { + await tunnel.close().catch(() => {}); + throw error; + } +} + /** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ export async function startVmTunnelHelper({ tls, enrollment, runId, proxyHost, proxyPort, host = "127.0.0.1", port = 0, now = Date.now, random = Math.random, wait = delay } = {}) { diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 0236b0b7..59785b0c 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -7,7 +7,7 @@ import { test } from "node:test"; import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; import { createAuthenticatedProxyListener } from "../src/proxy-listener.mjs"; import { createProxyRuntime } from "../src/runtime.mjs"; -import { createAuthenticatedTunnelServer, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; +import { createAuthenticatedTunnelServer, startAuthenticatedProxyTunnel, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; const fixtures = new URL("./fixtures/", import.meta.url); const key = await readFile(new URL("upstream-key.pem", fixtures)); @@ -225,41 +225,52 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante test("VM helper sends only run identity while the proxy performs authenticated enrollment", { timeout: 5_000 }, async () => { const runtime = createProxyRuntime(); - const authenticatedRuns = []; - const tunnel = createAuthenticatedTunnelServer({ + const requests = []; + const enrollmentId = "11111111-1111-4111-8111-111111111111"; + const runId = "33333333-3333-4333-8333-333333333333"; + const tunnel = await startAuthenticatedProxyTunnel({ tls: { key, cert, ca: cert }, runtime, - authenticateRun: async ({ runId, vmIdentity, bootEpoch }) => { - authenticatedRuns.push({ runId, vmIdentity, bootEpoch }); - return { - authenticated: true, - enrollmentConsumed: true, - connectionId: "enrollment-from-control-api", + controlOrigin: "https://factorize.example", + controlCredential: `fzc_${"c".repeat(43)}`, + fetchImpl: async (url, init) => { + const body = JSON.parse(init.body); + requests.push({ url: String(url), authorization: init.headers.Authorization, body }); + if (String(url).endsWith("/enroll")) { + return Response.json({ enrollmentId, token: "e".repeat(43), grantRevision: 7 }); + } + return Response.json({ consumed: true, binding: { + tenantId: "22222222-2222-4222-8222-222222222222", runId, - vmId: vmIdentity, - vmIdentity, - bootEpoch, - grants: [], - }; + vmIdentity: "api.example.test", + proxyIdentity: "proxy-fixture", + bootEpoch: runtime.bootEpoch, + enrollmentId, + grantRevision: 7, + } }); + }, + resolveGrants: async binding => { + assert.equal(binding.runId, runId); + assert.equal(binding.bootEpoch, runtime.bootEpoch); + return []; }, }); - await tunnel.listening; runtime.unlock("synthetic-unlock-key"); const local = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, - runId: "33333333-3333-4333-8333-333333333333", + runId, proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, }); try { const response = await fetch(`http://127.0.0.1:${local.address.port}/unrecognized`); assert.equal(response.status, 404); - assert.equal(authenticatedRuns.length, 1); - assert.deepEqual(authenticatedRuns[0], { - runId: "33333333-3333-4333-8333-333333333333", - vmIdentity: "api.example.test", - bootEpoch: runtime.bootEpoch, - }); + assert.deepEqual(requests.map(({ url, authorization, body }) => ({ + path: new URL(url).pathname, authorization, body, + })), [ + { path: "/internal/credential-proxy/enroll", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test" } }, + { path: "/internal/credential-proxy/consume-enrollment", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test", enrollmentToken: "e".repeat(43) } }, + ]); } finally { await local.close(); await tunnel.close(); From 712f6fe9ca113406f22b62ef80b198c4b7c4ddd9 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:06:25 +0000 Subject: [PATCH 062/102] Retain proxy control credential after boot registration --- .../src/bootstrap-control.mjs | 7 ++++-- packages/credential-proxy/src/main.mjs | 25 +++++++++++++++---- .../test/bootstrap-control.test.mjs | 15 ++++++++--- .../test/runtime-process.test.mjs | 12 ++++----- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/packages/credential-proxy/src/bootstrap-control.mjs b/packages/credential-proxy/src/bootstrap-control.mjs index 80e02fec..332bdcc0 100644 --- a/packages/credential-proxy/src/bootstrap-control.mjs +++ b/packages/credential-proxy/src/bootstrap-control.mjs @@ -6,8 +6,8 @@ import { registerProxyBoot } from "./control-client.mjs"; const MAX_BOOTSTRAP_BYTES = 4096; /** Local-only receiver for a provider exec channel; credentials never enter env or files. */ -export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot } = {}) { - if (!runtime || typeof register !== "function") throw new TypeError("proxy bootstrap control requires a runtime and registrar"); +export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot, onRegistered = () => {} } = {}) { + if (!runtime || typeof register !== "function" || typeof onRegistered !== "function") throw new TypeError("proxy bootstrap control requires a runtime, registrar, and registration callback"); await mkdir(dirname(socketPath), { recursive: true, mode: 0o700 }); await chmod(dirname(socketPath), 0o700); await rm(socketPath, { force: true }); @@ -43,6 +43,9 @@ export async function startProxyBootstrapControl({ runtime, socketPath = "/run/f let credential = request.credential; try { const registration = await register({ controlOrigin: request.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + // Keep the exchanged proxy credential inside the proxy process. The + // one-shot startup command receives only the public registration result. + await onRegistered(registration, { controlOrigin: request.controlOrigin, bootEpoch: runtime.bootEpoch }); finish({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { finish({ error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 3a7750ae..d63c3454 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -7,19 +7,33 @@ import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; const epochIndex = process.argv.indexOf("--boot-epoch"); const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; const runtime = createProxyRuntime({ bootEpoch }); -const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET }); +let controlCredential; +let controlOrigin; +const bootstrapControl = await startProxyBootstrapControl({ + runtime, + socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET, + onRegistered(registration, request) { + controlCredential = registration.controlCredential; + controlOrigin = request.controlOrigin; + }, +}); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); let closing = false; let shutdownDeadline; -let controlCredential; function respond(value) { process.stdout.write(`${JSON.stringify(value)}\n`); } +function respondStatus() { + respond({ type: "status", state: runtime.state, proxyControlRegistered: Boolean(controlCredential && controlOrigin) }); +} + async function stop() { if (closing) return; closing = true; + controlCredential = undefined; + controlOrigin = undefined; await runtime.shutdown(); await bootstrapControl.close(); lines.close(); @@ -38,19 +52,20 @@ lines.on("line", async (line) => { let command; try { command = JSON.parse(line); - if (command?.type === "status") respond({ type: "status", state: runtime.state }); + if (command?.type === "status") respondStatus(); else if (command?.type === "unlock") { runtime.unlock(command.key); - respond({ type: "status", state: runtime.state }); + respondStatus(); } else if (command?.type === "lock") { await runtime.lock(); - respond({ type: "status", state: runtime.state }); + respondStatus(); } else if (command?.type === "register-boot") { let credential = command.credential; command.credential = undefined; try { const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); controlCredential = registration.controlCredential; + controlOrigin = command.controlOrigin; respond({ type: "boot-registered", proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { respond({ type: "error", error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/test/bootstrap-control.test.mjs b/packages/credential-proxy/test/bootstrap-control.test.mjs index 2cae7dfb..ece8a05e 100644 --- a/packages/credential-proxy/test/bootstrap-control.test.mjs +++ b/packages/credential-proxy/test/bootstrap-control.test.mjs @@ -17,11 +17,11 @@ afterEach(async () => { directory = undefined; }); -async function fixture(register) { +async function fixture(register, onRegistered) { directory = await mkdtemp(join(tmpdir(), "factorize-proxy-startup-")); const socketPath = join(directory, "control.sock"); const runtime = { bootEpoch: epoch }; - service = await startProxyBootstrapControl({ runtime, socketPath, register }); + service = await startProxyBootstrapControl({ runtime, socketPath, register, onRegistered }); return socketPath; } @@ -37,10 +37,19 @@ async function request(socketPath, payload) { test("local startup control redeems the one-use credential and leaves a private socket", async () => { const calls = []; - const socketPath = await fixture(async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch }; }); + const registered = []; + const socketPath = await fixture( + async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }; }, + (value, request) => registered.push({ value, request }), + ); const result = await request(socketPath, { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); assert.deepEqual(result, { registered: true, proxyIdentity: "fly:app:machine", bootEpoch: epoch }); assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch }]); + assert.deepEqual(registered, [{ + value: { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }, + request: { controlOrigin: "https://factorize.example", bootEpoch: epoch }, + }]); + assert.equal(JSON.stringify(result).includes(token), false, "startup response never returns the proxy control credential"); assert.equal((await stat(socketPath)).mode & 0o777, 0o600); }); diff --git a/packages/credential-proxy/test/runtime-process.test.mjs b/packages/credential-proxy/test/runtime-process.test.mjs index 87ebeb61..1e4d5d90 100644 --- a/packages/credential-proxy/test/runtime-process.test.mjs +++ b/packages/credential-proxy/test/runtime-process.test.mjs @@ -61,8 +61,8 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts await writeFile(join(directory, "public-metadata.json"), JSON.stringify({ displayName: "fixture" })); const first = await startProxy(directory); processes.push(first); - assert.deepEqual(await first.command({ type: "status" }), { type: "status", state: "locked" }); - assert.deepEqual(await first.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked" }); + assert.deepEqual(await first.command({ type: "status" }), { type: "status", state: "locked", proxyControlRegistered: false }); + assert.deepEqual(await first.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked", proxyControlRegistered: false }); first.child.kill("SIGTERM"); await waitFor(() => first.child.exitCode !== null || first.child.signalCode !== null, "first process stop"); assert.equal(first.child.exitCode, 0); @@ -70,22 +70,22 @@ test("process starts locked, unlocks in memory, shuts down bounded, and restarts const restarted = await startProxy(directory); processes.push(restarted); assert.notEqual(restarted.bootEpoch, first.bootEpoch, "every process restart gets a fresh enrollment epoch"); - assert.deepEqual(await restarted.command({ type: "status" }), { type: "status", state: "locked" }); - assert.deepEqual(await restarted.command({ type: "unlock", key: tokenMarker }), { type: "status", state: "unlocked" }); + assert.deepEqual(await restarted.command({ type: "status" }), { type: "status", state: "locked", proxyControlRegistered: false }); + assert.deepEqual(await restarted.command({ type: "unlock", key: tokenMarker }), { type: "status", state: "unlocked", proxyControlRegistered: false }); restarted.child.stdin.write('{"type":"stop"}\n'); await waitFor(() => restarted.child.exitCode !== null || restarted.child.signalCode !== null, "command shutdown"); assert.equal(restarted.child.exitCode, 0); const crashed = await startProxy(directory); processes.push(crashed); - assert.deepEqual(await crashed.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked" }); + assert.deepEqual(await crashed.command({ type: "unlock", key: keyMarker }), { type: "status", state: "unlocked", proxyControlRegistered: false }); crashed.child.kill("SIGKILL"); await waitFor(() => crashed.child.exitCode !== null || crashed.child.signalCode !== null, "crash stop"); assert.equal(crashed.child.signalCode, "SIGKILL"); const afterCrash = await startProxy(directory); processes.push(afterCrash); - assert.deepEqual(await afterCrash.command({ type: "status" }), { type: "status", state: "locked" }); + assert.deepEqual(await afterCrash.command({ type: "status" }), { type: "status", state: "locked", proxyControlRegistered: false }); afterCrash.child.kill("SIGTERM"); await waitFor(() => afterCrash.child.exitCode !== null || afterCrash.child.signalCode !== null, "post-crash shutdown"); assert.equal(afterCrash.child.exitCode, 0); From f5fcd5144b4f224a2fa04aa17de1f033ccfa0965 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:29:12 +0000 Subject: [PATCH 063/102] Authenticate tunnel enrollment with run credential --- packages/api/src/proxy-control.ts | 16 +++++++-- packages/api/test/proxy-control.test.ts | 34 +++++++++++++++++-- .../credential-proxy/src/control-client.mjs | 6 ++-- .../credential-proxy/src/tunnel-transport.mjs | 11 +++--- .../test/control-client.test.mjs | 14 +++++--- .../test/proxy-listener.e2e.test.mjs | 6 ++-- 6 files changed, 68 insertions(+), 19 deletions(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index 695779cc..fbc1d125 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -1,6 +1,7 @@ import { databaseFor } from "./postgres/database"; import { ProxyControlRepository } from "./postgres/proxy-control-repository"; import { RunEnrollmentRepository } from "./postgres/run-enrollment-repository"; +import { authenticateRunCredential } from "./run-credentials"; const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "Cache-Control": "no-store" } }); @@ -41,9 +42,18 @@ export async function proxyControlFetch(request: Request, env: Parameters !["runId", "vmIdentity"].includes(key)) - || typeof (body as any).runId !== "string" || typeof (body as any).vmIdentity !== "string") return json({ error: "invalid_request" }, 400); - const enrollment = await enrollments.issue({ tenantId: proxy.tenantId, runId: (body as any).runId, vmIdentity: (body as any).vmIdentity, proxyIdentity: proxy.proxyIdentity, bootEpoch: proxy.bootEpoch }, new Date(Date.now() + 30_000)); + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity", "runCredential"].includes(key)) + || typeof (body as any).runId !== "string" || typeof (body as any).vmIdentity !== "string" || typeof (body as any).runCredential !== "string" + || (body as any).vmIdentity.length === 0 || (body as any).vmIdentity.length > 255) return json({ error: "invalid_request" }, 400); + // A proxy-authenticated request alone must not let a peer that can reach + // the proxy claim another active run. The VM's distinct run credential is + // validated against the current run and tenant before issuing enrollment. + const runAuth = await authenticateRunCredential(new Request(request.url, { + headers: { Authorization: `Bearer ${(body as any).runCredential}` }, + }), env); + if (!runAuth || runAuth.runId !== (body as any).runId || runAuth.tenantId !== proxy.tenantId + || !runAuth.scopes.includes("integrations:read")) return json({ error: "enrollment_unavailable" }, 403); + const enrollment = await enrollments.issue({ tenantId: proxy.tenantId, runId: runAuth.runId, vmIdentity: (body as any).vmIdentity, proxyIdentity: proxy.proxyIdentity, bootEpoch: proxy.bootEpoch }, new Date(Date.now() + 30_000)); return enrollment ? json(enrollment) : json({ error: "enrollment_unavailable" }, 403); } if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity", "enrollmentToken"].includes(key)) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index c0a33b1e..2c8d91dc 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -3,6 +3,8 @@ import { proxyControlFetch } from "../src/proxy-control"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; +const runId = "33333333-3333-4333-8333-333333333333"; +const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R".repeat(43)}`; function envFor({ credentialFound = true, stateSaved = true } = {}) { const transactionQuery = vi.fn(async (sql: string) => { @@ -12,7 +14,9 @@ function envFor({ credentialFound = true, stateSaved = true } = {}) { }); const query = vi.fn(async (sql: string) => sql.includes("proxy_token_digest") ? { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }] : [], rowCount: credentialFound ? 1 : 0 } - : { rows: [], rowCount: 0 }); + : sql.includes("FROM app.run_credentials") + ? { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", user_id: "owner-fixture", session_version: 1, job_id: "job-fixture", scopes: ["integrations:read"] }] : [], rowCount: credentialFound ? 1 : 0 } + : { rows: [], rowCount: 0 }); return { DATABASE: { pool: { query }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery, query } as any; } @@ -67,14 +71,40 @@ describe("dedicated proxy boot registration endpoint", () => { const response = await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", - body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123" }, + body: { runId, vmIdentity: "fly:vm-123", runCredential }, }), env); expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ grantRevision: 3, enrollmentId: expect.any(String), token: expect.any(String) }); expect(env.query).toHaveBeenCalled(); + expect(env.query.mock.calls.some((call: any[]) => call[0].includes("FROM app.run_credentials"))).toBe(true); expect(env.transactionQuery.mock.calls[0][0]).toContain("JOIN app.credential_proxy_states"); }); + it("refuses enrollment when the run credential is absent, invalid, cross-tenant, mismatched, or lacks integration scope", async () => { + for (const body of [ + { runId, vmIdentity: "fly:vm-123" }, + { runId, vmIdentity: "fly:vm-123", runCredential: "fzt_run_invalid" }, + { runId: "44444444-4444-4444-8444-444444444444", vmIdentity: "fly:vm-123", runCredential }, + ]) { + const env = envFor(); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body }), env)).status).toBe(403); + expect(env.transactionQuery).not.toHaveBeenCalled(); + } + const crossTenant = envFor(); + crossTenant.query.mockImplementation(async (sql: string) => sql.includes("proxy_token_digest") + ? { rows: [{ tenant_id: "other-tenant", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 } + : { rows: [], rowCount: 0 }); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body: { runId, vmIdentity: "fly:vm-123", runCredential } }), crossTenant)).status).toBe(403); + const noScope = envFor(); + noScope.query.mockImplementation(async (sql: string) => sql.includes("proxy_token_digest") + ? { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 } + : sql.includes("FROM app.run_credentials") + ? { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", user_id: "owner-fixture", session_version: 1, job_id: "job-fixture", scopes: ["runs:read"] }], rowCount: 1 } + : { rows: [], rowCount: 0 }); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body: { runId, vmIdentity: "fly:vm-123", runCredential } }), noScope)).status).toBe(403); + expect(noScope.transactionQuery).not.toHaveBeenCalled(); + }); + it("consumes enrollment against the authenticated proxy boot and rejects untrusted extra bindings", async () => { const env = envFor(); env.transactionQuery.mockImplementation(async (sql: string) => sql.includes("UPDATE app.credential_run_enrollments") diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index b5216f10..faf9ad3e 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -67,11 +67,11 @@ export function createProxyEnrollmentAuthenticator({ controlOrigin, controlCrede if (typeof resolveGrants !== "function") throw new TypeError("proxy enrollment requires a local grant resolver"); validateOrigin(controlOrigin); let credential = controlCredential; - return async function authenticateRun({ runId, vmIdentity, bootEpoch: requestedBootEpoch }) { + return async function authenticateRun({ runId, vmIdentity, runCredential, bootEpoch: requestedBootEpoch }) { if (!uuidPattern.test(runId ?? "") || typeof vmIdentity !== "string" || !vmIdentity.trim() || vmIdentity.length > 255 - || requestedBootEpoch !== bootEpoch) throw new Error("invalid proxy enrollment identity"); + || !/^fzt_run_[A-Za-z0-9_-]{43,200}$/.test(runCredential ?? "") || requestedBootEpoch !== bootEpoch) throw new Error("invalid proxy enrollment identity"); const options = { controlOrigin, token: credential, fetchImpl }; - const requested = { runId, vmIdentity }; + const requested = { runId, vmIdentity, runCredential }; const enrollment = await issueProxyEnrollment({ ...options, body: requested }); const binding = await consumeProxyEnrollment({ ...options, body: { ...requested, enrollmentToken: enrollment.token } }); if (binding.runId !== runId || binding.vmIdentity !== vmIdentity || binding.bootEpoch !== bootEpoch diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index e670f554..28d9d9a9 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -47,14 +47,14 @@ export function createAuthenticatedTunnelServer({ tls, runtime, authenticateEnro const line = handshake.subarray(0, end).toString("utf8").replace(/\r$/, ""); const rest = handshake.subarray(end + 1); const match = /^FACTORIZE-TUNNEL\/1 ([A-Za-z0-9_-]{32,128})$/.exec(line); - const runMatch = /^FACTORIZE-TUNNEL\/2 ([0-9a-f-]{36})$/i.exec(line); + const runMatch = /^FACTORIZE-TUNNEL\/2 ([0-9a-f-]{36}) (fzt_run_[A-Za-z0-9_-]{43,200})$/i.exec(line); if ((!match || typeof authenticateEnrollment !== "function") && (!runMatch || typeof authenticateRun !== "function")) return reject(); let listener; let target; try { const vmIdentity = peerIdentity(socket); const connection = runMatch - ? await authenticateRun({ runId: runMatch[1], vmIdentity, bootEpoch: runtime.bootEpoch }) + ? await authenticateRun({ runId: runMatch[1], runCredential: runMatch[2], vmIdentity, bootEpoch: runtime.bootEpoch }) : await authenticateEnrollment({ token: match[1], vmIdentity, bootEpoch: runtime.bootEpoch }); // The production control plane returns the consumed enrollment binding. // Compare its VM and boot identities with the authenticated TLS peer and @@ -121,10 +121,11 @@ export async function startAuthenticatedProxyTunnel({ } /** Start a loopback-only HTTP tunnel helper. Each accepted stream gets fresh enrollment. */ -export async function startVmTunnelHelper({ tls, enrollment, runId, proxyHost, proxyPort, host = "127.0.0.1", port = 0, +export async function startVmTunnelHelper({ tls, enrollment, runId, runCredential, proxyHost, proxyPort, host = "127.0.0.1", port = 0, now = Date.now, random = Math.random, wait = delay } = {}) { const runBasedEnrollment = typeof runId === "string" && /^[0-9a-f-]{36}$/i.test(runId); - if (!tls || (!runBasedEnrollment && typeof enrollment !== "function") || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { + if (!tls || (!runBasedEnrollment && typeof enrollment !== "function") || (runBasedEnrollment && !/^fzt_run_[A-Za-z0-9_-]{43,200}$/.test(runCredential ?? "")) + || typeof proxyHost !== "string" || !Number.isInteger(proxyPort)) { throw new TypeError("VM tunnel helper requires TLS, enrollment and proxy address"); } const sockets = new Set(); @@ -145,7 +146,7 @@ export async function startVmTunnelHelper({ tls, enrollment, runId, proxyHost, p const connectTimer = setTimeout(() => remote.destroy(new Error("proxy TLS connection timed out")), HANDSHAKE_TIMEOUT_MS); try { await once(remote, "secureConnect"); } finally { clearTimeout(connectTimer); } - remote.write(runBasedEnrollment ? `FACTORIZE-TUNNEL/2 ${runId}\n` : `FACTORIZE-TUNNEL/1 ${grant.token}\n`); + remote.write(runBasedEnrollment ? `FACTORIZE-TUNNEL/2 ${runId} ${runCredential}\n` : `FACTORIZE-TUNNEL/1 ${grant.token}\n`); await waitForTunnelAcceptance(remote); local.pipe(remote); remote.pipe(local); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 16e0ab93..1501dc9f 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -4,6 +4,8 @@ import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyE const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; +const runId = "33333333-3333-4333-8333-333333333333"; +const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R".repeat(43)}`; test("boot registration uses a one-call control credential, HTTPS, exact path and fresh process epoch", async () => { let request; @@ -74,7 +76,8 @@ test("proxy enrollment authenticator binds issued and consumed state to the mTLS }, }); const connection = await authenticateRun({ - runId: "33333333-3333-4333-8333-333333333333", + runId, + runCredential, vmIdentity: "fly:vm-1", bootEpoch, }); @@ -83,8 +86,8 @@ test("proxy enrollment authenticator binds issued and consumed state to the mTLS "https://factorize.example/internal/credential-proxy/consume-enrollment", ]); assert.deepEqual(requests.map(request => request.body), [ - { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1" }, - { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-1", enrollmentToken: "e".repeat(43) }, + { runId, vmIdentity: "fly:vm-1", runCredential }, + { runId, vmIdentity: "fly:vm-1", enrollmentToken: "e".repeat(43) }, ]); assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${controlToken}`)); assert.equal(connection.authenticated, true); @@ -94,9 +97,12 @@ test("proxy enrollment authenticator binds issued and consumed state to the mTLS assert.equal(connection.connectionId, "enrollment-1"); assert.deepEqual(connection.grants, grants); await assert.rejects(authenticateRun({ - runId: "33333333-3333-4333-8333-333333333333", + runId, + runCredential, vmIdentity: "fly:vm-1", bootEpoch: "44444444-4444-4444-8444-444444444444", }), /invalid proxy enrollment identity/); assert.equal(requests.length, 2, "a stale boot cannot issue another enrollment"); + await assert.rejects(authenticateRun({ runId, vmIdentity: "fly:vm-1", bootEpoch }), /invalid proxy enrollment identity/); + assert.equal(requests.length, 2, "a missing job run credential cannot issue enrollment"); }); diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 59785b0c..5098aed7 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -223,11 +223,12 @@ test("VM helper uses mutual TLS and consumed enrollment to reach only its grante } }); -test("VM helper sends only run identity while the proxy performs authenticated enrollment", { timeout: 5_000 }, async () => { +test("VM helper authenticates the active run while the proxy performs authenticated enrollment", { timeout: 5_000 }, async () => { const runtime = createProxyRuntime(); const requests = []; const enrollmentId = "11111111-1111-4111-8111-111111111111"; const runId = "33333333-3333-4333-8333-333333333333"; + const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R".repeat(43)}`; const tunnel = await startAuthenticatedProxyTunnel({ tls: { key, cert, ca: cert }, runtime, @@ -259,6 +260,7 @@ test("VM helper sends only run identity while the proxy performs authenticated e const local = await startVmTunnelHelper({ tls: { key, cert, ca: cert, servername: "api.example.test" }, runId, + runCredential, proxyHost: "127.0.0.1", proxyPort: tunnel.address().port, }); @@ -268,7 +270,7 @@ test("VM helper sends only run identity while the proxy performs authenticated e assert.deepEqual(requests.map(({ url, authorization, body }) => ({ path: new URL(url).pathname, authorization, body, })), [ - { path: "/internal/credential-proxy/enroll", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test" } }, + { path: "/internal/credential-proxy/enroll", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test", runCredential } }, { path: "/internal/credential-proxy/consume-enrollment", authorization: `Bearer fzc_${"c".repeat(43)}`, body: { runId, vmIdentity: "api.example.test", enrollmentToken: "e".repeat(43) } }, ]); } finally { From 3fcb724c96c00f59e06d7bf1791b5f70b66c1616 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:30:26 +0000 Subject: [PATCH 064/102] Keep run credential out of consume request --- packages/credential-proxy/src/control-client.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index faf9ad3e..192acaa4 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -73,7 +73,7 @@ export function createProxyEnrollmentAuthenticator({ controlOrigin, controlCrede const options = { controlOrigin, token: credential, fetchImpl }; const requested = { runId, vmIdentity, runCredential }; const enrollment = await issueProxyEnrollment({ ...options, body: requested }); - const binding = await consumeProxyEnrollment({ ...options, body: { ...requested, enrollmentToken: enrollment.token } }); + const binding = await consumeProxyEnrollment({ ...options, body: { runId, vmIdentity, enrollmentToken: enrollment.token } }); if (binding.runId !== runId || binding.vmIdentity !== vmIdentity || binding.bootEpoch !== bootEpoch || typeof binding.tenantId !== "string" || typeof binding.proxyIdentity !== "string" || binding.enrollmentId !== enrollment.enrollmentId || binding.grantRevision !== enrollment.grantRevision) { From 62b072938e82ca6951fed0ac4eb8ca01eabd47ef Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:30:39 +0000 Subject: [PATCH 065/102] Cover invalid run enrollment bindings --- packages/api/test/proxy-control.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 2c8d91dc..f36674b8 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -81,13 +81,13 @@ describe("dedicated proxy boot registration endpoint", () => { }); it("refuses enrollment when the run credential is absent, invalid, cross-tenant, mismatched, or lacks integration scope", async () => { - for (const body of [ - { runId, vmIdentity: "fly:vm-123" }, - { runId, vmIdentity: "fly:vm-123", runCredential: "fzt_run_invalid" }, - { runId: "44444444-4444-4444-8444-444444444444", vmIdentity: "fly:vm-123", runCredential }, - ]) { + for (const [body, expectedStatus] of [ + [{ runId, vmIdentity: "fly:vm-123" }, 400], + [{ runId, vmIdentity: "fly:vm-123", runCredential: "fzt_run_invalid" }, 403], + [{ runId: "44444444-4444-4444-8444-444444444444", vmIdentity: "fly:vm-123", runCredential }, 403], + ] as const) { const env = envFor(); - expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body }), env)).status).toBe(403); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body }), env)).status).toBe(expectedStatus); expect(env.transactionQuery).not.toHaveBeenCalled(); } const crossTenant = envFor(); From 9e85768479f5715bca17f9db21e270c3ee61a5b9 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:32:47 +0000 Subject: [PATCH 066/102] Narrow active run identity before enrollment --- packages/api/src/proxy-control.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index fbc1d125..e38a6c68 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -51,7 +51,7 @@ export async function proxyControlFetch(request: Request, env: Parameters Date: Wed, 11 Nov 2026 21:54:30 +0000 Subject: [PATCH 067/102] Close proxy tunnels when a run is revoked --- packages/credential-proxy/src/runtime.mjs | 22 ++++++- .../test/runtime-revocation.test.mjs | 57 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 packages/credential-proxy/test/runtime-revocation.test.mjs diff --git a/packages/credential-proxy/src/runtime.mjs b/packages/credential-proxy/src/runtime.mjs index 06cf340c..d7f76cf9 100644 --- a/packages/credential-proxy/src/runtime.mjs +++ b/packages/credential-proxy/src/runtime.mjs @@ -16,10 +16,13 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL let shutdown = false; let locking = false; const listeners = new Set(); + const listenerRuns = new Map(); + const revokedRuns = new Set(); async function closeListeners() { const current = [...listeners]; listeners.clear(); + listenerRuns.clear(); await Promise.allSettled(current.map((listener) => listener.close())); } @@ -49,20 +52,33 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL if (shutdown) throw new Error("runtime is shutting down"); if (state !== "unlocked") throw new Error("proxy is locked"); if (connection?.bootEpoch !== bootEpoch) throw new Error("enrollment belongs to a different proxy boot"); + if (typeof connection.runId !== "string" || !connection.runId) throw new TypeError("authenticated connection requires a run identity"); + if (revokedRuns.has(connection.runId)) throw new Error("run grants have been revoked"); // The transport adapter calls this only after atomically consuming and // authenticating enrollment. Grants are the consumed, current run state. const listener = await listenerFactory({ connection }); - if (state !== "unlocked" || shutdown) { + if (state !== "unlocked" || shutdown || revokedRuns.has(connection.runId)) { await listener.close(); - throw new Error("proxy locked while accepting the connection"); + throw new Error(revokedRuns.has(connection.runId) ? "run grants have been revoked" : "proxy locked while accepting the connection"); } listeners.add(listener); + // Keep the run identity beside the listener so a control-plane revocation + // can tear down every established connection without retaining grants. + listenerRuns.set(listener, connection.runId); return listener; }, async disconnect(connectionId) { const matching = [...listeners].filter((listener) => listener.connectionId === connectionId); await Promise.allSettled(matching.map((listener) => listener.close())); - for (const listener of matching) listeners.delete(listener); + for (const listener of matching) { listeners.delete(listener); listenerRuns.delete(listener); } + }, + async revokeRun(runId) { + if (typeof runId !== "string" || !runId) throw new TypeError("run identity is required for revocation"); + revokedRuns.add(runId); + const matching = [...listeners].filter((listener) => listenerRuns.get(listener) === runId); + await Promise.allSettled(matching.map((listener) => listener.close())); + for (const listener of matching) { listeners.delete(listener); listenerRuns.delete(listener); } + return matching.length; }, async shutdown() { shutdown = true; diff --git a/packages/credential-proxy/test/runtime-revocation.test.mjs b/packages/credential-proxy/test/runtime-revocation.test.mjs new file mode 100644 index 00000000..76bd6f13 --- /dev/null +++ b/packages/credential-proxy/test/runtime-revocation.test.mjs @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createProxyRuntime } from "../src/runtime.mjs"; + +const bootEpoch = "11111111-1111-4111-8111-111111111111"; + +function connection(runId, connectionId) { + return { authenticated: true, enrollmentConsumed: true, runId, connectionId, bootEpoch }; +} + +test("run revocation closes every established connection and rejects later enrollment", async () => { + const closed = []; + const runtime = createProxyRuntime({ + bootEpoch, + listenerFactory: async ({ connection: value }) => ({ + connectionId: value.connectionId, + close: async () => { closed.push(value.connectionId); }, + }), + }); + runtime.unlock("synthetic-unlock-key"); + try { + await runtime.acceptAuthenticatedConnection(connection("run-a", "connection-a1")); + await runtime.acceptAuthenticatedConnection(connection("run-a", "connection-a2")); + await runtime.acceptAuthenticatedConnection(connection("run-b", "connection-b1")); + + assert.equal(await runtime.revokeRun("run-a"), 2); + assert.deepEqual(closed.sort(), ["connection-a1", "connection-a2"]); + await assert.rejects(runtime.acceptAuthenticatedConnection(connection("run-a", "connection-a3")), /revoked/); + + const stillActive = await runtime.acceptAuthenticatedConnection(connection("run-b", "connection-b2")); + assert.equal(stillActive.connectionId, "connection-b2"); + assert.equal(await runtime.revokeRun("run-a"), 0); + assert.deepEqual(closed.sort(), ["connection-a1", "connection-a2"]); + } finally { + await runtime.shutdown(); + } +}); + +test("revocation racing listener creation closes the new listener before accepting it", async () => { + let resolveListener; + let closed = false; + const runtime = createProxyRuntime({ + bootEpoch, + listenerFactory: async () => new Promise(resolve => { resolveListener = resolve; }), + }); + runtime.unlock("synthetic-unlock-key"); + try { + const accepting = runtime.acceptAuthenticatedConnection(connection("run-race", "connection-race")); + await new Promise(resolve => setImmediate(resolve)); + await runtime.revokeRun("run-race"); + resolveListener({ connectionId: "connection-race", close: async () => { closed = true; } }); + await assert.rejects(accepting, /revoked/); + assert.equal(closed, true); + } finally { + await runtime.shutdown(); + } +}); From 184cd43bc43ce05439495045a15ed02080bbb714 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:17:25 +0000 Subject: [PATCH 068/102] Add decrypted run integration grant resolver --- .../credential-proxy/src/decrypted-grants.mjs | 98 +++++++++++++++++++ .../test/decrypted-grants.test.mjs | 69 +++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 packages/credential-proxy/src/decrypted-grants.mjs create mode 100644 packages/credential-proxy/test/decrypted-grants.test.mjs diff --git a/packages/credential-proxy/src/decrypted-grants.mjs b/packages/credential-proxy/src/decrypted-grants.mjs new file mode 100644 index 00000000..683ff547 --- /dev/null +++ b/packages/credential-proxy/src/decrypted-grants.mjs @@ -0,0 +1,98 @@ +import { webcrypto } from "node:crypto"; + +const KEY_PREFIX = "fzk1."; +const KEY_BYTES = 32; +const NONCE_BYTES = 12; +const SALT = new TextEncoder().encode("factorize.unlock-key.v1"); +const INFO = new TextEncoder().encode("factorize.record.aes-256-gcm.v1"); +const encoder = new TextEncoder(); +const decoder = new TextDecoder("utf-8", { fatal: true }); + +function decodeBase64Url(value, label) { + if (typeof value !== "string" || !/^[A-Za-z0-9_-]+$/.test(value)) throw new TypeError(`malformed ${label}`); + const bytes = Buffer.from(value, "base64url"); + if (bytes.toString("base64url") !== value) throw new TypeError(`malformed ${label}`); + return new Uint8Array(bytes); +} + +function decodeStoredCiphertext(value) { + if (typeof value !== "string" || value.length > 65_536) throw new TypeError("malformed integration ciphertext"); + const parts = value.split("."); + if (parts.length !== 2 || !parts.every(part => /^[A-Za-z0-9+/]+={0,2}$/.test(part))) throw new TypeError("malformed integration ciphertext"); + const nonce = Buffer.from(parts[0], "base64"); + const ciphertext = Buffer.from(parts[1], "base64"); + if (nonce.toString("base64") !== parts[0] || ciphertext.toString("base64") !== parts[1] + || nonce.byteLength !== NONCE_BYTES || ciphertext.byteLength < 16) throw new TypeError("malformed integration ciphertext"); + return { nonce: new Uint8Array(nonce), ciphertext: new Uint8Array(ciphertext) }; +} + +function recordAad(ownerId, record) { + if (typeof ownerId !== "string" || !ownerId || typeof record?.id !== "string" || !record.id + || typeof record.origin !== "string" || !record.origin || !Number.isSafeInteger(record.revision) || record.revision < 1) { + throw new TypeError("malformed integration record binding"); + } + return encoder.encode(JSON.stringify(["factorize-record-v1", ownerId, record.id, "bearer", record.origin, record.revision])); +} + +async function deriveKey(key) { + if (typeof key !== "string" || !key.startsWith(KEY_PREFIX)) throw new TypeError("malformed unlock key"); + const raw = decodeBase64Url(key.slice(KEY_PREFIX.length), "unlock key"); + if (raw.byteLength !== KEY_BYTES) throw new TypeError("malformed unlock key"); + try { + const material = await webcrypto.subtle.importKey("raw", raw, "HKDF", false, ["deriveKey"]); + return await webcrypto.subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt: SALT, info: INFO }, material, + { name: "AES-GCM", length: 256 }, false, ["decrypt"]); + } finally { + raw.fill(0); + } +} + +function validateOrigin(origin) { + let parsed; + try { parsed = new URL(origin); } catch { throw new TypeError("invalid integration origin"); } + if (parsed.protocol !== "https:" || parsed.origin !== origin || parsed.username || parsed.password) throw new TypeError("invalid integration origin"); + return parsed.origin; +} + +/** + * Resolve only the selected bearer records from a current, server-decrypted + * snapshot. The API supplies ciphertext after removing its own encryption + * layer; this helper never sends the user's unlock key or plaintext upstream. + */ +export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resolveIntegrationIds } = {}) { + if (typeof readSnapshot !== "function" || typeof getUnlockKey !== "function" + || (resolveIntegrationIds !== undefined && typeof resolveIntegrationIds !== "function")) throw new TypeError("grant resolver requires snapshot and unlock-key readers"); + return async function resolveGrants(binding) { + if (!binding || typeof binding.tenantId !== "string" || !binding.tenantId + || (binding.integrationIds !== undefined && !Array.isArray(binding.integrationIds))) throw new TypeError("invalid run integration selection"); + const integrationIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding); + if (!Array.isArray(integrationIds) || integrationIds.some(id => typeof id !== "string" || !id) + || new Set(integrationIds).size !== integrationIds.length) throw new TypeError("invalid run integration selection"); + if (integrationIds.length === 0) return []; + const snapshot = await readSnapshot(binding); + if (!snapshot || !Array.isArray(snapshot.records) || typeof snapshot.ownerId !== "string" || !snapshot.ownerId) throw new Error("integration snapshot is unavailable"); + const selected = new Set(integrationIds); + const records = snapshot.records.filter(record => selected.has(record?.id) && record.kind === "bearer" && record.deletedAt == null); + if (records.length !== selected.size) throw new Error("selected integration is unavailable"); + const unlockKey = await getUnlockKey(); + if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); + const aesKey = await deriveKey(unlockKey); + const grants = []; + for (const record of records) { + const origin = validateOrigin(record.origin); + const { nonce, ciphertext } = decodeStoredCiphertext(record.encryptedPayload); + const plaintext = new Uint8Array(await webcrypto.subtle.decrypt({ name: "AES-GCM", iv: nonce, + additionalData: recordAad(snapshot.ownerId, record), tagLength: 128 }, aesKey, ciphertext)); + try { + const credential = decoder.decode(plaintext); + if (!credential || credential.length > 16_384 || /[\u0000-\u001f\u007f]/.test(credential)) throw new TypeError("invalid bearer credential"); + grants.push(Object.freeze({ id: record.id, origin, credential })); + } finally { + plaintext.fill(0); + nonce.fill(0); + ciphertext.fill(0); + } + } + return grants; + }; +} diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs new file mode 100644 index 00000000..d0f873aa --- /dev/null +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { webcrypto } from "node:crypto"; +import { test } from "node:test"; +import { createDecryptedGrantResolver } from "../src/decrypted-grants.mjs"; + +const encoder = new TextEncoder(); +const salt = encoder.encode("factorize.unlock-key.v1"); +const info = encoder.encode("factorize.record.aes-256-gcm.v1"); +const ownerId = "owner-fixture"; +const unlockKey = `fzk1.${Buffer.from(webcrypto.getRandomValues(new Uint8Array(32))).toString("base64url")}`; + +async function encryptCredential({ id, origin, revision, credential }) { + const raw = Buffer.from(unlockKey.slice("fzk1.".length), "base64url"); + const material = await webcrypto.subtle.importKey("raw", raw, "HKDF", false, ["deriveKey"]); + const aesKey = await webcrypto.subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt, info }, material, + { name: "AES-GCM", length: 256 }, false, ["encrypt"]); + const nonce = webcrypto.getRandomValues(new Uint8Array(12)); + const additionalData = encoder.encode(JSON.stringify(["factorize-record-v1", ownerId, id, "bearer", origin, revision])); + const ciphertext = new Uint8Array(await webcrypto.subtle.encrypt({ name: "AES-GCM", iv: nonce, additionalData, tagLength: 128 }, aesKey, encoder.encode(credential))); + return `${Buffer.from(nonce).toString("base64")}.${Buffer.from(ciphertext).toString("base64")}`; +} + +test("resolver decrypts only selected active bearer records using the shared record binding", async () => { + const first = { id: "integration-a", kind: "bearer", origin: "https://api.example.test", revision: 3, deletedAt: null }; + const second = { id: "integration-b", kind: "bearer", origin: "https://other.example.test", revision: 4, deletedAt: null }; + const records = [ + { ...first, encryptedPayload: await encryptCredential({ ...first, credential: "first-token" }) }, + { ...second, encryptedPayload: await encryptCredential({ ...second, credential: "second-token" }) }, + { id: "deleted", kind: "bearer", origin: "https://api.example.test", revision: 5, deletedAt: new Date(), encryptedPayload: "unused" }, + ]; + const resolver = createDecryptedGrantResolver({ + readSnapshot: async () => ({ ownerId, records }), + getUnlockKey: async () => unlockKey, + }); + assert.deepEqual(await resolver({ tenantId: "tenant-fixture", integrationIds: ["integration-b"] }), [ + { id: "integration-b", origin: "https://other.example.test", credential: "second-token" }, + ]); + assert.deepEqual(await resolver({ tenantId: "tenant-fixture", integrationIds: [] }), []); + const runResolver = createDecryptedGrantResolver({ + readSnapshot: async () => ({ ownerId, records }), + resolveIntegrationIds: async binding => binding.runId === "run-fixture" ? ["integration-a"] : [], + getUnlockKey: async () => unlockKey, + }); + assert.deepEqual(await runResolver({ tenantId: "tenant-fixture", runId: "run-fixture" }), [ + { id: "integration-a", origin: "https://api.example.test", credential: "first-token" }, + ]); +}); + +test("resolver rejects missing, deleted, ambiguous, malformed, and misbound grants", async () => { + const record = { id: "integration-a", kind: "bearer", origin: "https://api.example.test", revision: 3, deletedAt: null }; + const encryptedPayload = await encryptCredential({ ...record, credential: "synthetic-token" }); + const resolver = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [{ ...record, encryptedPayload }] }), getUnlockKey: async () => unlockKey }); + await assert.rejects(resolver({ tenantId: "tenant-fixture", integrationIds: ["missing"] }), /unavailable/); + await assert.rejects(resolver({ tenantId: "tenant-fixture", integrationIds: ["integration-a", "integration-a"] }), /selection/); + const tampered = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId: "other-owner", records: [{ ...record, encryptedPayload }] }), getUnlockKey: async () => unlockKey }); + await assert.rejects(tampered({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] })); + const locked = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [{ ...record, encryptedPayload }] }), getUnlockKey: async () => undefined }); + await assert.rejects(locked({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] }), /locked/); +}); + +test("resolver rejects non-HTTPS destinations and invalid credential bytes", async () => { + const badOrigin = { id: "integration-a", kind: "bearer", origin: "http://api.example.test", revision: 3, deletedAt: null, + encryptedPayload: await encryptCredential({ id: "integration-a", origin: "http://api.example.test", revision: 3, credential: "token" }) }; + const badOriginResolver = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [badOrigin] }), getUnlockKey: async () => unlockKey }); + await assert.rejects(badOriginResolver({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] }), /integration origin/); + const badCredential = { ...badOrigin, origin: "https://api.example.test", encryptedPayload: await encryptCredential({ id: "integration-a", origin: "https://api.example.test", revision: 3, credential: "bad\ncredential" }) }; + const badCredentialResolver = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [badCredential] }), getUnlockKey: async () => unlockKey }); + await assert.rejects(badCredentialResolver({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] }), /credential/); +}); From 7457e58a6e56154bdcf6b5334265d347897ff454 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:31:01 +0000 Subject: [PATCH 069/102] Expose selected integrations to authenticated proxy --- packages/api/src/proxy-control.ts | 24 ++++++++++++++++- packages/api/test/proxy-control.test.ts | 27 +++++++++++++++++++ .../credential-proxy/src/control-client.mjs | 21 +++++++++++++++ .../test/control-client.test.mjs | 16 ++++++++++- 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index e38a6c68..d3bb08b5 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -1,6 +1,7 @@ import { databaseFor } from "./postgres/database"; import { ProxyControlRepository } from "./postgres/proxy-control-repository"; import { RunEnrollmentRepository } from "./postgres/run-enrollment-repository"; +import { EncryptedIntegrationRepository } from "./postgres/encrypted-integration-repository"; import { authenticateRunCredential } from "./run-credentials"; const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "Cache-Control": "no-store" } }); @@ -8,7 +9,7 @@ const json = (body: unknown, status = 200) => Response.json(body, { status, head /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment"].includes(url.pathname)) return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; @@ -40,6 +41,27 @@ export async function proxyControlFetch(request: Request, env: Parameters !["runId", "grantRevision"].includes(key)) + || typeof (body as any).runId !== "string" || !Number.isSafeInteger((body as any).grantRevision) || (body as any).grantRevision < 0) return json({ error: "invalid_request" }, 400); + const database = databaseFor(env); + const active = await database.pool.query<{ execution_config: Record; grant_revision: string | number }>(`SELECT r.execution_config,g.grant_revision + FROM app.runs r JOIN app.credential_run_grants g ON g.tenant_id=r.tenant_id AND g.run_id=r.id + JOIN app.credential_proxy_states p ON p.tenant_id=g.tenant_id AND p.proxy_identity=g.proxy_identity AND p.boot_epoch=g.boot_epoch + WHERE r.tenant_id=$1 AND r.id=$2 AND r.state IN ('starting','running','blocked') + AND g.proxy_identity=$3 AND g.boot_epoch=$4 AND g.grant_revision=$5 AND g.revoked_at IS NULL + AND p.disabled_at IS NULL AND p.revoked_at IS NULL`, [proxy.tenantId, (body as any).runId, proxy.proxyIdentity, proxy.bootEpoch, (body as any).grantRevision]); + const run = active.rows[0]; + if (!run) return json({ error: "integration_snapshot_unavailable" }, 403); + const selected = run.execution_config?.portableIntegrationIds; + if (!Array.isArray(selected) || selected.some(id => typeof id !== "string") || new Set(selected).size !== selected.length) return json({ error: "integration_snapshot_unavailable" }, 403); + const records = selected.length + ? (await new EncryptedIntegrationRepository(database, proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("bearer")) + .filter(record => selected.includes(record.id)) + : []; + if (records.length !== selected.length) return json({ error: "integration_snapshot_unavailable" }, 403); + return json({ ownerId: proxy.tenantId, records }); + } const enrollments = new RunEnrollmentRepository(databaseFor(env)); if (url.pathname === "/internal/credential-proxy/enroll") { if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "vmIdentity", "runCredential"].includes(key)) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index f36674b8..e1c64539 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { proxyControlFetch } from "../src/proxy-control"; +import { encrypt } from "../src/crypto"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -120,4 +121,30 @@ describe("dedicated proxy boot registration endpoint", () => { const rejected = await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/enroll", body: { runId: "33333333-3333-4333-8333-333333333333", vmIdentity: "fly:vm-123", tenantId: "other" } }), env); expect(rejected.status).toBe(400); }); + + it("returns only the current run's selected user-encrypted bearer records to its registered proxy", async () => { + const integrationId = "55555555-5555-4555-8555-555555555555"; + const key = "test-server-encryption-key"; + const env = envFor(); + env.CREDENTIAL_ENCRYPTION_KEY = key; + env.query.mockImplementation(async (sql: string) => { + if (sql.includes("proxy_token_digest")) return { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 }; + if (sql.includes("FROM app.runs r")) return { rows: [{ execution_config: { portableIntegrationIds: [integrationId] }, grant_revision: 3 }], rowCount: 1 }; + if (sql.includes("FROM app.integration_records")) return { rows: [ + { id: integrationId, kind: "bearer", name: "selected", origin: "https://api.example", description: null, revision: 4, encrypted_payload: await encrypt("user-ciphertext", key) }, + { id: "66666666-6666-4666-8666-666666666666", kind: "bearer", name: "not-selected", origin: "https://other.example", description: null, revision: 5, encrypted_payload: await encrypt("other-ciphertext", key) }, + ], rowCount: 2 }; + return { rows: [], rowCount: 0 }; + }); + const response = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/integration-snapshot", + body: { runId, grantRevision: 3 }, + }), env); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ownerId: "11111111-1111-4111-8111-111111111111", records: [ + { id: integrationId, kind: "bearer", name: "selected", origin: "https://api.example", description: null, revision: 4, encryptedPayload: "user-ciphertext" }, + ] }); + expect(env.query.mock.calls.map((call: any[]) => call[0])).toEqual(expect.arrayContaining([expect.stringContaining("grant_revision=$5"), expect.stringContaining("FROM app.integration_records")])); + }); }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 192acaa4..0912b108 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -52,6 +52,27 @@ export async function consumeProxyEnrollment(options) { return result.binding; } +/** Read the current user-encrypted integration records selected by an active run. + * The control plane decrypts only Factorize's outer encryption layer. */ +export async function readProxyIntegrationSnapshot({ controlOrigin, token, binding, fetchImpl = fetch } = {}) { + if (!binding || !uuidPattern.test(binding.runId ?? "") || !Number.isSafeInteger(binding.grantRevision) || binding.grantRevision < 0) { + throw new TypeError("invalid proxy integration snapshot binding"); + } + const result = await proxyControlRequest({ + controlOrigin, + token, + path: "/internal/credential-proxy/integration-snapshot", + body: { runId: binding.runId, grantRevision: binding.grantRevision }, + fetchImpl, + }); + if (typeof result?.ownerId !== "string" || !Array.isArray(result.records) + || result.records.some(record => !record || typeof record.id !== "string" || record.kind !== "bearer" + || typeof record.origin !== "string" || typeof record.encryptedPayload !== "string" || !Number.isSafeInteger(record.revision))) { + throw new Error("invalid proxy integration snapshot response"); + } + return result; +} + const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; /** diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 1501dc9f..c8386843 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, registerProxyBoot } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, registerProxyBoot } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -20,6 +20,20 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an assert.deepEqual(JSON.parse(request.init.body), { bootEpoch }); }); +test("proxy snapshot reader requests only the bound run revision and validates encrypted records", async () => { + let request; + const snapshot = { ownerId: "owner-1", records: [{ id: "integration-1", kind: "bearer", origin: "https://api.example", revision: 4, encryptedPayload: "ciphertext" }] }; + const result = await readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async (url, init) => { + request = { url: String(url), init, body: JSON.parse(init.body) }; + return Response.json(snapshot); + } }); + assert.deepEqual(result, snapshot); + assert.equal(request.url, "https://factorize.example/internal/credential-proxy/integration-snapshot"); + assert.deepEqual(request.body, { runId, grantRevision: 7 }); + assert.equal(request.init.headers.Authorization, `Bearer ${token}`); + await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ownerId: "owner-1", records: [{ ...snapshot.records[0], kind: "model" }] }) }), /invalid proxy integration snapshot response/); +}); + test("boot registration rejects unsafe origins, invalid credentials and mismatched response bindings", async () => { await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch }), /HTTPS origin/); await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); From af71a90bafcbdc1d61effcfbc1de911768917d7e Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:34:21 +0000 Subject: [PATCH 070/102] Fix encrypted snapshot proxy test fixture --- packages/api/test/proxy-control.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index e1c64539..649e16e9 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -124,7 +124,7 @@ describe("dedicated proxy boot registration endpoint", () => { it("returns only the current run's selected user-encrypted bearer records to its registered proxy", async () => { const integrationId = "55555555-5555-4555-8555-555555555555"; - const key = "test-server-encryption-key"; + const key = Buffer.alloc(32, 7).toString("base64"); const env = envFor(); env.CREDENTIAL_ENCRYPTION_KEY = key; env.query.mockImplementation(async (sql: string) => { From 69fa35ef0fbcc7751063c4371f41a2a9d3472e96 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:45:26 +0000 Subject: [PATCH 071/102] Wire proxy snapshots into local grant resolution --- .../credential-proxy/src/decrypted-grants.mjs | 28 ++++++++++++++--- packages/credential-proxy/src/runtime.mjs | 5 +++ .../test/decrypted-grants.test.mjs | 31 ++++++++++++++++++- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/packages/credential-proxy/src/decrypted-grants.mjs b/packages/credential-proxy/src/decrypted-grants.mjs index 683ff547..5fde128f 100644 --- a/packages/credential-proxy/src/decrypted-grants.mjs +++ b/packages/credential-proxy/src/decrypted-grants.mjs @@ -1,4 +1,5 @@ import { webcrypto } from "node:crypto"; +import { readProxyIntegrationSnapshot } from "./control-client.mjs"; const KEY_PREFIX = "fzk1."; const KEY_BYTES = 32; @@ -65,15 +66,21 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol return async function resolveGrants(binding) { if (!binding || typeof binding.tenantId !== "string" || !binding.tenantId || (binding.integrationIds !== undefined && !Array.isArray(binding.integrationIds))) throw new TypeError("invalid run integration selection"); - const integrationIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding); - if (!Array.isArray(integrationIds) || integrationIds.some(id => typeof id !== "string" || !id) - || new Set(integrationIds).size !== integrationIds.length) throw new TypeError("invalid run integration selection"); - if (integrationIds.length === 0) return []; const snapshot = await readSnapshot(binding); if (!snapshot || !Array.isArray(snapshot.records) || typeof snapshot.ownerId !== "string" || !snapshot.ownerId) throw new Error("integration snapshot is unavailable"); + // Production snapshots are already filtered by the control plane using the + // active run's immutable selection and grant revision. Local callers may + // still provide IDs explicitly for additional narrowing. + const integrationIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding) ?? snapshot.records.map(record => record?.id); + if (!Array.isArray(integrationIds) || integrationIds.some(id => typeof id !== "string" || !id) + || new Set(integrationIds).size !== integrationIds.length) throw new TypeError("invalid run integration selection"); + if (integrationIds.length === 0) { + if (snapshot.records.length) throw new Error("integration snapshot does not match the run selection"); + return []; + } const selected = new Set(integrationIds); const records = snapshot.records.filter(record => selected.has(record?.id) && record.kind === "bearer" && record.deletedAt == null); - if (records.length !== selected.size) throw new Error("selected integration is unavailable"); + if (records.length !== selected.size || snapshot.records.length !== records.length) throw new Error("selected integration is unavailable"); const unlockKey = await getUnlockKey(); if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const aesKey = await deriveKey(unlockKey); @@ -96,3 +103,14 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol return grants; }; } + +/** Compose the boot-registered control client, current snapshot reader, and + * process-local unlock key into the resolver used by authenticated tunnels. */ +export function createProxyIntegrationGrantResolver({ runtime, controlOrigin, controlCredential, fetchImpl = fetch } = {}) { + if (!runtime || typeof runtime.getUnlockKey !== "function") throw new TypeError("proxy runtime does not expose its unlocked key to the grant resolver"); + if (typeof controlOrigin !== "string" || typeof controlCredential !== "string") throw new TypeError("proxy control registration is required"); + return createDecryptedGrantResolver({ + readSnapshot: binding => readProxyIntegrationSnapshot({ controlOrigin, token: controlCredential, binding, fetchImpl }), + getUnlockKey: () => runtime.getUnlockKey(), + }); +} diff --git a/packages/credential-proxy/src/runtime.mjs b/packages/credential-proxy/src/runtime.mjs index d7f76cf9..ac5db602 100644 --- a/packages/credential-proxy/src/runtime.mjs +++ b/packages/credential-proxy/src/runtime.mjs @@ -40,6 +40,11 @@ export function createProxyRuntime({ listenerFactory = createAuthenticatedProxyL unlockMaterial = Buffer.from(key, "utf8"); state = "unlocked"; }, + /** Return a transient copy only to the local grant resolver. */ + getUnlockKey() { + if (shutdown || locking || state !== "unlocked" || !unlockMaterial) throw new Error("proxy is locked"); + return unlockMaterial.toString("utf8"); + }, async lock() { locking = true; state = "locked"; diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs index d0f873aa..3dd01866 100644 --- a/packages/credential-proxy/test/decrypted-grants.test.mjs +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -1,7 +1,8 @@ import assert from "node:assert/strict"; import { webcrypto } from "node:crypto"; import { test } from "node:test"; -import { createDecryptedGrantResolver } from "../src/decrypted-grants.mjs"; +import { createDecryptedGrantResolver, createProxyIntegrationGrantResolver } from "../src/decrypted-grants.mjs"; +import { createProxyRuntime } from "../src/runtime.mjs"; const encoder = new TextEncoder(); const salt = encoder.encode("factorize.unlock-key.v1"); @@ -67,3 +68,31 @@ test("resolver rejects non-HTTPS destinations and invalid credential bytes", asy const badCredentialResolver = createDecryptedGrantResolver({ readSnapshot: async () => ({ ownerId, records: [badCredential] }), getUnlockKey: async () => unlockKey }); await assert.rejects(badCredentialResolver({ tenantId: "tenant-fixture", integrationIds: ["integration-a"] }), /credential/); }); + +test("production grant resolver reads the run-bound snapshot and only decrypts while the runtime is unlocked", async () => { + const record = { id: "integration-a", kind: "bearer", origin: "https://api.example.test", revision: 3, deletedAt: null }; + const encryptedPayload = await encryptCredential({ ...record, credential: "snapshot-token" }); + const runtime = createProxyRuntime({ bootEpoch: "11111111-1111-4111-8111-111111111111" }); + const requests = []; + const resolve = createProxyIntegrationGrantResolver({ + runtime, + controlOrigin: "https://factorize.example", + controlCredential: `fzc_${"c".repeat(43)}`, + fetchImpl: async (url, init) => { + requests.push({ url: String(url), authorization: new Headers(init.headers).get("authorization"), body: JSON.parse(init.body) }); + return Response.json({ ownerId, records: [{ ...record, encryptedPayload }] }); + }, + }); + const binding = { runId: "22222222-2222-4222-8222-222222222222", grantRevision: 7 }; + await assert.rejects(resolve(binding), /locked/); + runtime.unlock(unlockKey); + assert.deepEqual(await resolve(binding), [{ id: record.id, origin: record.origin, credential: "snapshot-token" }]); + assert.deepEqual(requests, [{ + url: "https://factorize.example/internal/credential-proxy/integration-snapshot", + authorization: `Bearer fzc_${"c".repeat(43)}`, + body: binding, + }]); + await runtime.lock(); + await assert.rejects(resolve(binding), /locked/); + await runtime.shutdown(); +}); From 3467c0385d923a7571b0ab28f4f350c27e536adb Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:48:34 +0000 Subject: [PATCH 072/102] Preserve grant resolver selection boundaries --- packages/credential-proxy/src/decrypted-grants.mjs | 3 ++- packages/credential-proxy/test/decrypted-grants.test.mjs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/credential-proxy/src/decrypted-grants.mjs b/packages/credential-proxy/src/decrypted-grants.mjs index 5fde128f..4be1d19b 100644 --- a/packages/credential-proxy/src/decrypted-grants.mjs +++ b/packages/credential-proxy/src/decrypted-grants.mjs @@ -80,7 +80,8 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol } const selected = new Set(integrationIds); const records = snapshot.records.filter(record => selected.has(record?.id) && record.kind === "bearer" && record.deletedAt == null); - if (records.length !== selected.size || snapshot.records.length !== records.length) throw new Error("selected integration is unavailable"); + const selectionWasNarrowedLocally = binding.integrationIds !== undefined || resolveIntegrationIds !== undefined; + if (records.length !== selected.size || (!selectionWasNarrowedLocally && snapshot.records.length !== records.length)) throw new Error("selected integration is unavailable"); const unlockKey = await getUnlockKey(); if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const aesKey = await deriveKey(unlockKey); diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs index 3dd01866..a3467672 100644 --- a/packages/credential-proxy/test/decrypted-grants.test.mjs +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -83,7 +83,7 @@ test("production grant resolver reads the run-bound snapshot and only decrypts w return Response.json({ ownerId, records: [{ ...record, encryptedPayload }] }); }, }); - const binding = { runId: "22222222-2222-4222-8222-222222222222", grantRevision: 7 }; + const binding = { tenantId: ownerId, runId: "22222222-2222-4222-8222-222222222222", grantRevision: 7 }; await assert.rejects(resolve(binding), /locked/); runtime.unlock(unlockKey); assert.deepEqual(await resolve(binding), [{ id: record.id, origin: record.origin, credential: "snapshot-token" }]); From a3a74a2fa857cc666747d4baa3b02302dcadb193 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:49:39 +0000 Subject: [PATCH 073/102] Resolve empty and locked grants safely --- packages/credential-proxy/src/decrypted-grants.mjs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/credential-proxy/src/decrypted-grants.mjs b/packages/credential-proxy/src/decrypted-grants.mjs index 4be1d19b..247a1bf3 100644 --- a/packages/credential-proxy/src/decrypted-grants.mjs +++ b/packages/credential-proxy/src/decrypted-grants.mjs @@ -66,12 +66,18 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol return async function resolveGrants(binding) { if (!binding || typeof binding.tenantId !== "string" || !binding.tenantId || (binding.integrationIds !== undefined && !Array.isArray(binding.integrationIds))) throw new TypeError("invalid run integration selection"); + const requestedIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding); + if (requestedIds !== undefined && (!Array.isArray(requestedIds) || requestedIds.some(id => typeof id !== "string" || !id) + || new Set(requestedIds).size !== requestedIds.length)) throw new TypeError("invalid run integration selection"); + if (requestedIds?.length === 0) return []; + const unlockKey = await getUnlockKey(); + if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const snapshot = await readSnapshot(binding); if (!snapshot || !Array.isArray(snapshot.records) || typeof snapshot.ownerId !== "string" || !snapshot.ownerId) throw new Error("integration snapshot is unavailable"); // Production snapshots are already filtered by the control plane using the // active run's immutable selection and grant revision. Local callers may // still provide IDs explicitly for additional narrowing. - const integrationIds = binding.integrationIds ?? await resolveIntegrationIds?.(binding) ?? snapshot.records.map(record => record?.id); + const integrationIds = requestedIds ?? snapshot.records.map(record => record?.id); if (!Array.isArray(integrationIds) || integrationIds.some(id => typeof id !== "string" || !id) || new Set(integrationIds).size !== integrationIds.length) throw new TypeError("invalid run integration selection"); if (integrationIds.length === 0) { @@ -82,8 +88,6 @@ export function createDecryptedGrantResolver({ readSnapshot, getUnlockKey, resol const records = snapshot.records.filter(record => selected.has(record?.id) && record.kind === "bearer" && record.deletedAt == null); const selectionWasNarrowedLocally = binding.integrationIds !== undefined || resolveIntegrationIds !== undefined; if (records.length !== selected.size || (!selectionWasNarrowedLocally && snapshot.records.length !== records.length)) throw new Error("selected integration is unavailable"); - const unlockKey = await getUnlockKey(); - if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const aesKey = await deriveKey(unlockKey); const grants = []; for (const record of records) { From 2872e4e319e828d4f06edbd736c16fff5dd67980 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:49:56 +0000 Subject: [PATCH 074/102] Match snapshot control request contract --- packages/credential-proxy/test/decrypted-grants.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs index a3467672..18ad4d62 100644 --- a/packages/credential-proxy/test/decrypted-grants.test.mjs +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -90,7 +90,7 @@ test("production grant resolver reads the run-bound snapshot and only decrypts w assert.deepEqual(requests, [{ url: "https://factorize.example/internal/credential-proxy/integration-snapshot", authorization: `Bearer fzc_${"c".repeat(43)}`, - body: binding, + body: { runId: binding.runId, grantRevision: binding.grantRevision }, }]); await runtime.lock(); await assert.rejects(resolve(binding), /locked/); From 41f35d10aeb64f6db43b4153476e497aaa592e6b Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 23:08:07 +0000 Subject: [PATCH 075/102] Bind tunnel VM identity to certificate SAN --- .../credential-proxy/src/tunnel-transport.mjs | 19 ++++++++++++--- .../test/fixtures/upstream-cert.pem | 24 +++++++++---------- .../test/proxy-listener.e2e.test.mjs | 17 ++++++++++++- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/packages/credential-proxy/src/tunnel-transport.mjs b/packages/credential-proxy/src/tunnel-transport.mjs index 28d9d9a9..d61f6b99 100644 --- a/packages/credential-proxy/src/tunnel-transport.mjs +++ b/packages/credential-proxy/src/tunnel-transport.mjs @@ -9,10 +9,23 @@ const CONTROL_RETRY_WINDOW_MS = 5 * 60_000; const CONTROL_RETRY_MAX_MS = 30_000; const CONTROL_RETRY_SLOW_MS = 60_000; -function peerIdentity(socket) { +export function peerIdentity(socket) { const certificate = socket.getPeerCertificate(); - if (!socket.authorized || !certificate?.subject?.CN) throw new Error("tunnel client certificate is not authorized"); - return certificate.subject.CN; + if (!socket.authorized || !certificate?.subject?.CN || typeof certificate.subjectaltname !== "string") { + throw new Error("tunnel client certificate is not authorized"); + } + const identities = certificate.subjectaltname.split(/,\s*/).filter(value => value.startsWith("URI:")); + if (identities.length !== 1) throw new Error("tunnel client certificate must contain one VM identity URI"); + const match = /^URI:urn:factorize:vm:([A-Za-z0-9_-]+)$/.exec(identities[0]); + if (!match) throw new Error("tunnel client certificate has an invalid VM identity URI"); + const identityBytes = Buffer.from(match[1], "base64url"); + const identity = identityBytes.toString("utf8"); + if (!identity || identityBytes.toString("base64url") !== match[1] + || identity.length > 255 || /[\u0000-\u001f\u007f]/.test(identity) + || certificate.subject.CN !== identity) { + throw new Error("tunnel client certificate identity is invalid or inconsistent"); + } + return identity; } /** diff --git a/packages/credential-proxy/test/fixtures/upstream-cert.pem b/packages/credential-proxy/test/fixtures/upstream-cert.pem index 685d5670..afec81f5 100644 --- a/packages/credential-proxy/test/fixtures/upstream-cert.pem +++ b/packages/credential-proxy/test/fixtures/upstream-cert.pem @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNDCCAhygAwIBAgIUWvCDWltyenSy441ZnfAvqkjgIwUwDQYJKoZIhvcNAQEL -BQAwGzEZMBcGA1UEAwwQYXBpLmV4YW1wbGUudGVzdDAeFw0yNjExMTExNTQwNTha -Fw0zNjExMDgxNTQwNThaMBsxGTAXBgNVBAMMEGFwaS5leGFtcGxlLnRlc3QwggEi +MIIDTjCCAjagAwIBAgIUPy+fWru0gF7QNal9Xg4vXdIuUCgwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQYXBpLmV4YW1wbGUudGVzdDAeFw0yNjExMTEyMzA3Mjla +Fw0zNjExMDgyMzA3MjlaMBsxGTAXBgNVBAMMEGFwaS5leGFtcGxlLnRlc3QwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDClgGTbXnob9GHbVxMt1lSmsfW 6lSfuOEBEuE5n96bKcfSsVKMRKvw3j8PyGasF+DTQTwqFlcavHzmmVkOBpQYLSz3 bvD0RnKJ8CXhZwcRSA77BENRJQJ3TRr9WIAVrz3dgkAEl5Tv4Y+0KILRqZMx+vg1 R0CuAzt6JbhGTvFugRAgeSJootV/O9Uu+z4SqtEbOkpj7isRUJa1uirAmKoFucIU sa/4wTBkWIL74Z2iEMdmNM0HQ0xTsoVmFVJb+yxCoCr5rzIdPuGDyQeqWaWtju7Z kVJjUskyYgSrycUivxAElY9aedgmyZxoBIHiwdFsSt/Nu5Gj9mOahqAfAPdjAgMB -AAGjcDBuMB0GA1UdDgQWBBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAfBgNVHSMEGDAW -gBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAPBgNVHRMBAf8EBTADAQH/MBsGA1UdEQQU -MBKCEGFwaS5leGFtcGxlLnRlc3QwDQYJKoZIhvcNAQELBQADggEBAFOW5O4OKDe8 -6MWztml7Yt9Rvn6zcDTM4xsMKhMDITAlorXuAfu51JGgFOsqiqNP9L9cpN57k5JS -5wYA9Fy+cFiiXtkAqImSvjSf1gvlzKpHCh1s0dGjIYnoStoWbE0PzVweSQ2zCQy/ -aQC11E3/CyS2+toZR+i1zffaBLjbA6T4GToOKCrXdGl8Wv7vCyTbEMA6ouEspkJO -U/glcrkekGgIZVC10di70LI+LkgqsnuDpFIALb7qDMtZQuqnIEU/wyapORpQn6Lm -WjHPEcWSHe5dlHEQWO0P3XXEA2mbr+fboVWWqQ/5gt7aBTLOVfV9rSkQqGpvriPQ -RmX+kW6AbK4= +AAGjgYkwgYYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAqQwRAYDVR0R +BD0wO4IQYXBpLmV4YW1wbGUudGVzdIYndXJuOmZhY3Rvcml6ZTp2bTpZWEJwTG1W +NFlXMXdiR1V1ZEdWemRBMB0GA1UdDgQWBBRnufSGyhOp/K9MV0Q1L6bh3O/uyzAN +BgkqhkiG9w0BAQsFAAOCAQEASBvpzgJtGPUxB8ao20lUU2J95DRPFH6BEPoJq38+ +62D7YTNIesVPykdnZQy/8uVWczIAKURAU/MRv10N8K6nfo/PAx3Hi2m8qbnoTcRa +2JLoE++bQltKI7GAg//wviMpb3CjgObjoIYVa8pdHvR2PLEIroJuGitOyZTWDBFh +bu5VcH5FDkCLD8DzatgGyEVDDR9vELzS5m1j9WgVXAHnMhnKegmvhhJhsdT+/LEE +gtNFdHYNRPGbxhJ6Kc8CbmSZN7T8zUjCTHfcAB5iz6lrQRbbbxzIekXeqzv+Ya0C +vnY+CxOlQab0mac1Y72EnoDMSkvAywIbdwe9qcpITlrKmQ== -----END CERTIFICATE----- diff --git a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs index 5098aed7..a470bf0a 100644 --- a/packages/credential-proxy/test/proxy-listener.e2e.test.mjs +++ b/packages/credential-proxy/test/proxy-listener.e2e.test.mjs @@ -7,7 +7,7 @@ import { test } from "node:test"; import { createBearerForwarder } from "../src/bearer-forwarder.mjs"; import { createAuthenticatedProxyListener } from "../src/proxy-listener.mjs"; import { createProxyRuntime } from "../src/runtime.mjs"; -import { createAuthenticatedTunnelServer, startAuthenticatedProxyTunnel, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; +import { createAuthenticatedTunnelServer, peerIdentity, startAuthenticatedProxyTunnel, startVmTunnelHelper } from "../src/tunnel-transport.mjs"; const fixtures = new URL("./fixtures/", import.meta.url); const key = await readFile(new URL("upstream-key.pem", fixtures)); @@ -18,6 +18,21 @@ async function waitForEvent(emitter, event) { return pending; } +test("mTLS VM identity comes from one canonical URI SAN bound to the certificate CN", () => { + const identity = "api.example.test"; + const encoded = Buffer.from(identity).toString("base64url"); + const socket = (subjectaltname, commonName = identity, authorized = true) => ({ + authorized, + getPeerCertificate: () => ({ subject: { CN: commonName }, subjectaltname }), + }); + assert.equal(peerIdentity(socket(`DNS:api.example.test, URI:urn:factorize:vm:${encoded}`)), identity); + assert.throws(() => peerIdentity(socket("DNS:api.example.test")), /one VM identity URI/); + assert.throws(() => peerIdentity(socket(`URI:urn:factorize:vm:${encoded}, URI:urn:factorize:vm:${encoded}`)), /one VM identity URI/); + assert.throws(() => peerIdentity(socket("URI:urn:factorize:vm:not_canonical=")), /invalid VM identity URI/); + assert.throws(() => peerIdentity(socket(`URI:urn:factorize:vm:${encoded}`, "other-vm")), /inconsistent/); + assert.throws(() => peerIdentity(socket(`URI:urn:factorize:vm:${encoded}`, identity, false)), /not authorized/); +}); + test("authenticated tunnel listener streams over TLS, confines grants and cancels upstream work", { timeout: 5_000 }, async () => { let observedAuthorization; let upstreamEnded = false; From 434c008518bf18570d4124794ac31e28239dbbaa Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 23:47:48 +0000 Subject: [PATCH 076/102] GEN-2824 provision mTLS identities and revoke tunnels --- .../src/fly-credential-proxy-provisioner.ts | 25 ++++++ packages/api/src/proxy-control.ts | 10 ++- .../fly-credential-proxy-provisioner.test.ts | 29 ++++++- packages/api/test/proxy-control.test.ts | 22 +++++ .../src/bootstrap-control.mjs | 28 +++++- .../credential-proxy/src/control-client.mjs | 52 +++++++++++ packages/credential-proxy/src/main.mjs | 52 +++++++++-- .../credential-proxy/src/tls-identity.mjs | 87 +++++++++++++++++++ .../credential-proxy/src/vm-certificate.mjs | 61 +++++++++++++ .../test/bootstrap-control.test.mjs | 32 +++++-- .../test/control-client.test.mjs | 29 ++++++- .../test/tls-identity.test.mjs | 50 +++++++++++ 12 files changed, 458 insertions(+), 19 deletions(-) create mode 100644 packages/credential-proxy/src/tls-identity.mjs create mode 100644 packages/credential-proxy/src/vm-certificate.mjs create mode 100644 packages/credential-proxy/test/tls-identity.test.mjs diff --git a/packages/api/src/fly-credential-proxy-provisioner.ts b/packages/api/src/fly-credential-proxy-provisioner.ts index ad8751d4..101b0ba5 100644 --- a/packages/api/src/fly-credential-proxy-provisioner.ts +++ b/packages/api/src/fly-credential-proxy-provisioner.ts @@ -28,6 +28,9 @@ export type FlyCredentialProxyMachineClient = { export type FlyCredentialProxyGuest = { /** The reviewed image startup command consumes this payload from stdin only. */ startProxy(input: ProxyBootstrapProvisioning & { credential: string }): Promise; + /** Signs a VM-generated CSR over the provider's private guest channel. Only + * the public certificate and CA trust bundle return to the caller. */ + issueVmCertificate(input: { bootEpoch: string; vmIdentity: string; csr: string }): Promise<{ certificate: string; trustBundle: string }>; }; /** Provider exec adapter: the bootstrap record is the command's stdin payload. */ @@ -42,6 +45,7 @@ export function createFlyCredentialProxyGuest(options: { return { async startProxy(input) { const result = await transport.run(["/usr/local/bin/factorize-proxy-startup"], `${JSON.stringify({ + type: "register-boot", controlOrigin: input.controlOrigin, bootEpoch: input.bootEpoch, credential: input.credential, @@ -49,6 +53,22 @@ export function createFlyCredentialProxyGuest(options: { if (!result.ok) throw new Error("Credential proxy startup command failed"); if (result.stdout.trim() !== "Credential proxy started") throw new Error("Credential proxy startup was not confirmed"); }, + async issueVmCertificate(input) { + if (!input || !UUID_V4.test(input.bootEpoch) || !/^[A-Za-z0-9:._-]{1,255}$/.test(input.vmIdentity) + || typeof input.csr !== "string" || input.csr.length < 100 || input.csr.length > 16_384) throw new TypeError("Invalid VM certificate provisioning request"); + const result = await transport.run(["/usr/local/bin/factorize-proxy-certify-vm"], `${JSON.stringify(input)}\n`); + if (!result.ok) throw new Error("Credential proxy VM certificate provisioning failed"); + let response: unknown; + try { response = JSON.parse(result.stdout); } catch { throw new Error("Credential proxy returned an invalid VM certificate response"); } + if (!response || typeof response !== "object" || Array.isArray(response) + || Object.keys(response).sort().join(",") !== "bootEpoch,certificate,trustBundle,vmIdentity" + || (response as any).bootEpoch !== input.bootEpoch || (response as any).vmIdentity !== input.vmIdentity + || typeof (response as any).certificate !== "string" || !(response as any).certificate.startsWith("-----BEGIN CERTIFICATE-----") + || typeof (response as any).trustBundle !== "string" || !(response as any).trustBundle.startsWith("-----BEGIN CERTIFICATE-----")) { + throw new Error("Credential proxy returned an invalid VM certificate response"); + } + return { certificate: (response as any).certificate, trustBundle: (response as any).trustBundle }; + }, }; } @@ -62,6 +82,7 @@ export type FlyCredentialProxyProvisionerOptions = { function validate(input: FlyCredentialProxyConfig): void { if (!input.tenantId || !input.app || !input.operationId || input.operationId.length > 200) throw new TypeError("Invalid credential proxy provisioning identity"); + if (!/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(input.app)) throw new TypeError("Invalid credential proxy Fly app name"); if (!IMAGE_DIGEST.test(input.image)) throw new TypeError("Credential proxy image must be pinned by sha256 digest"); if (!/^[a-z0-9]{3}$/.test(input.region)) throw new TypeError("Invalid credential proxy region"); if (!/^[a-z0-9-]{1,64}$/.test(input.size)) throw new TypeError("Invalid credential proxy size"); @@ -89,6 +110,10 @@ export function createFlyCredentialProxyProvisioner(options: FlyCredentialProxyP region: input.region, cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", candidateBootEpoch], guest: { cpu_kind: "shared", cpus: 1, memory_mb: input.memoryMb ?? 512 }, + env: { + FACTORIZE_PROXY_TUNNEL_SERVER_NAME: `${input.app}.internal`, + FACTORIZE_PROXY_TUNNEL_PORT: "9090", + }, metadata: { factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId, diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index d3bb08b5..fa91e53c 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -9,7 +9,7 @@ const json = (body: unknown, status = 200) => Response.json(body, { status, head /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot"].includes(url.pathname)) return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot", "/internal/credential-proxy/revocations"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; @@ -41,6 +41,14 @@ export async function proxyControlFetch(request: Request, env: Parameters key !== "bootEpoch") + || typeof (body as any).bootEpoch !== "string" || (body as any).bootEpoch !== proxy.bootEpoch) return json({ error: "invalid_request" }, 400); + const revoked = await databaseFor(env).pool.query<{ run_id: string }>(`SELECT run_id FROM app.credential_run_grants + WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch=$3 AND revoked_at IS NOT NULL ORDER BY revoked_at,run_id`, + [proxy.tenantId, proxy.proxyIdentity, proxy.bootEpoch]); + return json({ revokedRunIds: revoked.rows.map(row => row.run_id) }); + } if (url.pathname === "/internal/credential-proxy/integration-snapshot") { if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "grantRevision"].includes(key)) || typeof (body as any).runId !== "string" || !Number.isSafeInteger((body as any).grantRevision) || (body as any).grantRevision < 0) return json({ error: "invalid_request" }, 400); diff --git a/packages/api/test/fly-credential-proxy-provisioner.test.ts b/packages/api/test/fly-credential-proxy-provisioner.test.ts index aeecf1ca..d2b1a976 100644 --- a/packages/api/test/fly-credential-proxy-provisioner.test.ts +++ b/packages/api/test/fly-credential-proxy-provisioner.test.ts @@ -23,7 +23,7 @@ function fixture() { startMachine: vi.fn(async () => undefined), destroyMachine: vi.fn(async () => undefined), }; - const guest = vi.fn(() => ({ startProxy: vi.fn(async () => undefined) })); + const guest = vi.fn(() => ({ startProxy: vi.fn(async () => undefined), issueVmCertificate: vi.fn(async () => ({ certificate: "", trustBundle: "" })) })); const provisioner = createFlyCredentialProxyProvisioner({ control, machines, guest, newBootEpoch: () => epoch }); return { control, machines, guest, provisioner }; } @@ -37,6 +37,7 @@ describe("Fly credential proxy startup", () => { image: input.image, region: input.region, cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", epoch], + env: { FACTORIZE_PROXY_TUNNEL_SERVER_NAME: "factorize-proxies.internal", FACTORIZE_PROXY_TUNNEL_PORT: "9090" }, restart: { policy: "no" }, auto_start: false, metadata: expect.objectContaining({ factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId }), @@ -57,7 +58,7 @@ describe("Fly credential proxy startup", () => { it("revokes bootstrap authority and removes a Machine when startup fails", async () => { const { control, machines, guest } = fixture(); - guest.mockReturnValue({ startProxy: vi.fn().mockRejectedValue(new Error("private startup channel failed")) }); + guest.mockReturnValue({ startProxy: vi.fn().mockRejectedValue(new Error("private startup channel failed")), issueVmCertificate: vi.fn() }); const provisioner = createFlyCredentialProxyProvisioner({ control, machines, guest, newBootEpoch: () => epoch }); await expect(provisioner.start(input)).rejects.toThrow("private startup channel failed"); expect(control.revokeCredential).toHaveBeenCalledWith(input.tenantId, "fly:factorize-proxies:machine-1"); @@ -68,6 +69,7 @@ describe("Fly credential proxy startup", () => { const { machines, provisioner } = fixture(); await expect(provisioner.start({ ...input, image: "ghcr.io/example/proxy:latest" })).rejects.toThrow("pinned by sha256 digest"); await expect(provisioner.start({ ...input, controlOrigin: "http://factorize.example" })).rejects.toThrow("HTTPS origin"); + await expect(provisioner.start({ ...input, app: "invalid/app" })).rejects.toThrow("Fly app name"); expect(machines.createMachine).not.toHaveBeenCalled(); }); }); @@ -88,6 +90,29 @@ describe("Fly credential proxy guest adapter", () => { const guest = createFlyCredentialProxyGuest({ app: input.app, machine: machine.id, token: "server-token-fixture", spawnProcess }); await guest.startProxy({ tenantId: input.tenantId, proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, credential: `fzc_${"c".repeat(43)}` }); expect(calls[0]?.[1]).toContain("/usr/local/bin/factorize-proxy-startup"); + expect(String(calls[1]?.[1])).toContain(`"type":"register-boot"`); expect(String(calls[1]?.[1])).toContain(`"credential":"fzc_${"c".repeat(43)}"`); }); + + it("requests a VM CSR signature through the private guest channel and verifies both bindings", async () => { + const calls: unknown[][] = []; + const spawnProcess = ((command: string, args: string[], options: { stdio: string[] }) => { + calls.push([command, args, options]); + const stdoutHandlers: Array<(chunk: Buffer) => void> = []; + const child: any = { pid: undefined, stdin: { end: (value: string) => { calls.push(["stdin", value]); } }, + stdout: { on: (_event: string, callback: (chunk: Buffer) => void) => stdoutHandlers.push(callback) }, + stderr: { on: () => undefined }, on: (event: string, callback: (code: number) => void) => { + if (event === "close") queueMicrotask(() => { stdoutHandlers.forEach(handler => handler(Buffer.from(JSON.stringify({ + bootEpoch: epoch, vmIdentity: "fly:job-1", certificate: "-----BEGIN CERTIFICATE-----cert", trustBundle: "-----BEGIN CERTIFICATE-----ca", + }) + "\n"))); callback(0); }); + }, kill: () => undefined }; + return child; + }) as any; + const guest = createFlyCredentialProxyGuest({ app: input.app, machine: machine.id, token: "server-token-fixture", spawnProcess }); + await expect(guest.issueVmCertificate({ bootEpoch: epoch, vmIdentity: "fly:job-1", csr: `-----BEGIN CERTIFICATE REQUEST-----\n${"A".repeat(120)}\n-----END CERTIFICATE REQUEST-----` })) + .resolves.toEqual({ certificate: "-----BEGIN CERTIFICATE-----cert", trustBundle: "-----BEGIN CERTIFICATE-----ca" }); + expect(calls[0]?.[1]).toContain("/usr/local/bin/factorize-proxy-certify-vm"); + expect(String(calls[1]?.[1])).toContain("\"vmIdentity\":\"fly:job-1\""); + await expect(guest.issueVmCertificate({ bootEpoch: epoch, vmIdentity: "fly:job-1", csr: "bad" })).rejects.toThrow("Invalid VM certificate provisioning request"); + }); }); diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 649e16e9..7a8cd90d 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -147,4 +147,26 @@ describe("dedicated proxy boot registration endpoint", () => { ] }); expect(env.query.mock.calls.map((call: any[]) => call[0])).toEqual(expect.arrayContaining([expect.stringContaining("grant_revision=$5"), expect.stringContaining("FROM app.integration_records")])); }); + + it("returns durable revoked runs only for the authenticated current proxy boot", async () => { + const env = envFor(); + env.query.mockImplementation(async (sql: string) => sql.includes("proxy_token_digest") + ? { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 } + : { rows: [{ run_id: runId }], rowCount: 1 }); + const response = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/revocations", + body: { bootEpoch }, + }), env); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ revokedRunIds: [runId] }); + expect(env.query.mock.calls[1][0]).toContain("revoked_at IS NOT NULL"); + expect(env.query.mock.calls[1][1]).toEqual(["11111111-1111-4111-8111-111111111111", "proxy-fixture", bootEpoch]); + const stale = await proxyControlFetch(request({ + authorization: `Bearer ${token}`, + path: "/internal/credential-proxy/revocations", + body: { bootEpoch: "44444444-4444-4444-8444-444444444444" }, + }), env); + expect(stale.status).toBe(400); + }); }); diff --git a/packages/credential-proxy/src/bootstrap-control.mjs b/packages/credential-proxy/src/bootstrap-control.mjs index 332bdcc0..8d17234c 100644 --- a/packages/credential-proxy/src/bootstrap-control.mjs +++ b/packages/credential-proxy/src/bootstrap-control.mjs @@ -6,8 +6,9 @@ import { registerProxyBoot } from "./control-client.mjs"; const MAX_BOOTSTRAP_BYTES = 4096; /** Local-only receiver for a provider exec channel; credentials never enter env or files. */ -export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot, onRegistered = () => {} } = {}) { - if (!runtime || typeof register !== "function" || typeof onRegistered !== "function") throw new TypeError("proxy bootstrap control requires a runtime, registrar, and registration callback"); +export async function startProxyBootstrapControl({ runtime, socketPath = "/run/factorize-proxy/bootstrap.sock", register = registerProxyBoot, onRegistered = () => {}, issueVmCertificate } = {}) { + if (!runtime || typeof register !== "function" || typeof onRegistered !== "function" + || (issueVmCertificate !== undefined && typeof issueVmCertificate !== "function")) throw new TypeError("proxy bootstrap control requires a runtime, registrar, and registration callback"); await mkdir(dirname(socketPath), { recursive: true, mode: 0o700 }); await chmod(dirname(socketPath), 0o700); await rm(socketPath, { force: true }); @@ -31,8 +32,29 @@ export async function startProxyBootstrapControl({ runtime, socketPath = "/run/f let request; try { request = JSON.parse(data.subarray(0, end).toString("utf8")); } catch { return finish({ error: "invalid bootstrap request" }); } + if (request?.type === "issue-vm-certificate") { + if (!issueVmCertificate || request.bootEpoch !== runtime.bootEpoch + || typeof request.vmIdentity !== "string" || typeof request.csr !== "string" + || Object.keys(request).sort().join(",") !== "bootEpoch,csr,type,vmIdentity") { + return finish({ error: "invalid certificate request" }); + } + processing = true; + socket.pause(); + socket.removeAllListeners("data"); + try { + const issued = await issueVmCertificate({ vmIdentity: request.vmIdentity, csr: request.csr, bootEpoch: runtime.bootEpoch }); + if (typeof issued?.certificate !== "string" || typeof issued?.trustBundle !== "string") throw new Error("invalid certificate response"); + finish({ issued: true, bootEpoch: runtime.bootEpoch, vmIdentity: request.vmIdentity, certificate: issued.certificate, trustBundle: issued.trustBundle }); + } catch { + finish({ error: "certificate provisioning failed" }); + } finally { + request.csr = ""; + } + return; + } + if (request?.type !== "register-boot") return finish({ error: "invalid bootstrap request" }); if (!request || typeof request !== "object" || Array.isArray(request) - || Object.keys(request).sort().join(",") !== "bootEpoch,controlOrigin,credential" + || Object.keys(request).sort().join(",") !== "bootEpoch,controlOrigin,credential,type" || request.bootEpoch !== runtime.bootEpoch || typeof request.credential !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(request.credential)) { return finish({ error: "invalid bootstrap request" }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 0912b108..fd1b4c46 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -73,6 +73,58 @@ export async function readProxyIntegrationSnapshot({ controlOrigin, token, bindi return result; } +/** Read durable revocations for the registered proxy boot. Replaying the full + * set is intentional: it closes runs revoked while this proxy was offline and + * runtime.revokeRun is idempotent. */ +export async function readProxyRunRevocations({ controlOrigin, token, bootEpoch, fetchImpl = fetch } = {}) { + if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); + const result = await proxyControlRequest({ + controlOrigin, + token, + path: "/internal/credential-proxy/revocations", + body: { bootEpoch }, + fetchImpl, + }); + if (!Array.isArray(result?.revokedRunIds) || result.revokedRunIds.some(id => !uuidPattern.test(id))) { + throw new Error("invalid proxy revocation response"); + } + return result.revokedRunIds; +} + +/** Poll durable control-plane state. A failed request leaves established + * connections running and the next bounded poll retries when communication + * recovers; it does not add an authorization lease or expiry. */ +export function startProxyRunRevocationMonitor({ controlOrigin, controlCredential, bootEpoch, runtime, intervalMs = 5_000, fetchImpl = fetch } = {}) { + if (!runtime || typeof runtime.revokeRun !== "function" || !Number.isInteger(intervalMs) || intervalMs < 100 || intervalMs > 60_000) { + throw new TypeError("invalid proxy revocation monitor configuration"); + } + let stopped = false; + let timer; + let polling; + const poll = async () => { + if (stopped || polling) return; + polling = (async () => { + try { + const runIds = await readProxyRunRevocations({ controlOrigin, token: controlCredential, bootEpoch, fetchImpl }); + for (const runId of runIds) await runtime.revokeRun(runId); + } catch { + // Keep existing connections alive during an outage and retry on schedule. + } finally { + polling = undefined; + if (!stopped) timer = setTimeout(poll, intervalMs); + } + })(); + }; + void poll(); + return Object.freeze({ + async close() { + stopped = true; + clearTimeout(timer); + await polling; + }, + }); +} + const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; /** diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index d63c3454..60ab7333 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -1,20 +1,56 @@ import { createInterface } from "node:readline"; import { randomUUID } from "node:crypto"; import { createProxyRuntime } from "./runtime.mjs"; -import { registerProxyBoot } from "./control-client.mjs"; +import { registerProxyBoot, startProxyRunRevocationMonitor } from "./control-client.mjs"; import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; +import { createProxyIntegrationGrantResolver } from "./decrypted-grants.mjs"; +import { startAuthenticatedProxyTunnel } from "./tunnel-transport.mjs"; +import { createProxyTlsAuthority } from "./tls-identity.mjs"; const epochIndex = process.argv.indexOf("--boot-epoch"); const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; const runtime = createProxyRuntime({ bootEpoch }); +const tunnelPort = Number(process.env.FACTORIZE_PROXY_TUNNEL_PORT ?? "9090"); +if (!Number.isInteger(tunnelPort) || tunnelPort < 1 || tunnelPort > 65535) throw new Error("invalid proxy tunnel port"); +const tlsAuthority = await createProxyTlsAuthority({ + serverName: process.env.FACTORIZE_PROXY_TUNNEL_SERVER_NAME ?? "localhost", + bootEpoch: runtime.bootEpoch, +}); let controlCredential; let controlOrigin; +let tunnel; +let revocationMonitor; + +async function activateRegistration(registration, origin) { + if (tunnel) throw new Error("proxy boot is already registered"); + const resolveGrants = createProxyIntegrationGrantResolver({ runtime, controlOrigin: origin, controlCredential: registration.controlCredential }); + const started = await startAuthenticatedProxyTunnel({ + tls: tlsAuthority.tls, + runtime, + controlOrigin: origin, + controlCredential: registration.controlCredential, + resolveGrants, + host: process.env.FACTORIZE_PROXY_TUNNEL_HOST ?? "0.0.0.0", + port: tunnelPort, + }); + controlCredential = registration.controlCredential; + controlOrigin = origin; + tunnel = started; + revocationMonitor = startProxyRunRevocationMonitor({ + controlOrigin: origin, + controlCredential: registration.controlCredential, + bootEpoch: runtime.bootEpoch, + runtime, + }); +} + const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET, - onRegistered(registration, request) { - controlCredential = registration.controlCredential; - controlOrigin = request.controlOrigin; + onRegistered: (registration, request) => activateRegistration(registration, request.controlOrigin), + issueVmCertificate: ({ vmIdentity, csr }) => { + if (!tunnel) throw new Error("proxy tunnel is not registered"); + return tlsAuthority.signVmCsr({ vmIdentity, csr }); }, }); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); @@ -34,7 +70,12 @@ async function stop() { closing = true; controlCredential = undefined; controlOrigin = undefined; + await revocationMonitor?.close(); + revocationMonitor = undefined; await runtime.shutdown(); + await tunnel?.close().catch(() => {}); + tunnel = undefined; + await tlsAuthority.close(); await bootstrapControl.close(); lines.close(); clearTimeout(shutdownDeadline); @@ -64,8 +105,7 @@ lines.on("line", async (line) => { command.credential = undefined; try { const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); - controlCredential = registration.controlCredential; - controlOrigin = command.controlOrigin; + await activateRegistration(registration, command.controlOrigin); respond({ type: "boot-registered", proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { respond({ type: "error", error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/src/tls-identity.mjs b/packages/credential-proxy/src/tls-identity.mjs new file mode 100644 index 00000000..933fde50 --- /dev/null +++ b/packages/credential-proxy/src/tls-identity.mjs @@ -0,0 +1,87 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFile = promisify(execFileCallback); +const SAFE_VM_IDENTITY = /^[A-Za-z0-9:._-]{1,255}$/; +const SAFE_DNS_NAME = /^(?=.{1,253}$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(?:\.(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?))*$/; + +async function openssl(args) { + try { + await execFile("openssl", args, { timeout: 10_000, maxBuffer: 64 * 1024, windowsHide: true }); + } catch { + // Never include command output or key material in errors. + throw new Error("proxy TLS identity operation failed"); + } +} + +function pem(bytes) { + return Buffer.from(bytes).toString("utf8"); +} + +/** + * Creates a per-process private CA and proxy server identity on a memory-backed + * filesystem. The CA private key never leaves this process or tmpfs. Job VMs + * generate their own private keys and submit CSRs; only their signed public + * certificate and the public trust root are returned for secure distribution. + */ +export async function createProxyTlsAuthority({ serverName, bootEpoch } = {}) { + if (typeof serverName !== "string" || !SAFE_DNS_NAME.test(serverName)) throw new TypeError("proxy TLS server name is invalid"); + if (typeof bootEpoch !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(bootEpoch)) throw new TypeError("proxy TLS boot epoch is invalid"); + const directory = await mkdtemp(join("/dev/shm", "factorize-proxy-tls-")); + const caKeyPath = join(directory, "ca-key.pem"); + const caCertPath = join(directory, "ca-cert.pem"); + const serverKeyPath = join(directory, "server-key.pem"); + const serverCertPath = join(directory, "server-cert.pem"); + try { + await openssl(["genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", "-out", caKeyPath]); + await chmod(caKeyPath, 0o600); + await openssl(["req", "-new", "-x509", "-key", caKeyPath, "-sha256", "-days", "1", "-subj", `/CN=factorize-proxy-${bootEpoch}`, "-addext", "basicConstraints=critical,CA:TRUE,pathlen:0", "-addext", "keyUsage=critical,keyCertSign,cRLSign", "-out", caCertPath]); + await openssl(["genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", "-out", serverKeyPath]); + await chmod(serverKeyPath, 0o600); + await openssl(["req", "-new", "-key", serverKeyPath, "-subj", `/CN=${serverName}`, "-out", join(directory, "server.csr")]); + await writeFile(join(directory, "server.ext"), `basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyAgreement\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:${serverName}\n`, { mode: 0o600 }); + await openssl(["x509", "-req", "-in", join(directory, "server.csr"), "-CA", caCertPath, "-CAkey", caKeyPath, "-CAcreateserial", "-days", "1", "-sha256", "-extfile", join(directory, "server.ext"), "-out", serverCertPath]); + const ca = await readFile(caCertPath); + const key = await readFile(serverKeyPath); + const cert = await readFile(serverCertPath); + + let closed = false; + return Object.freeze({ + tls: Object.freeze({ key, cert, ca }), + trustBundle: pem(ca), + async signVmCsr({ vmIdentity, csr } = {}) { + if (closed) throw new Error("proxy TLS identity authority is closed"); + if (typeof vmIdentity !== "string" || !SAFE_VM_IDENTITY.test(vmIdentity) + || typeof csr !== "string" || csr.length < 100 || csr.length > 16_384 + || !csr.startsWith("-----BEGIN CERTIFICATE REQUEST-----\n") + || !csr.trimEnd().endsWith("-----END CERTIFICATE REQUEST-----")) { + throw new TypeError("invalid VM certificate request"); + } + const identityUri = `urn:factorize:vm:${Buffer.from(vmIdentity, "utf8").toString("base64url")}`; + const csrPath = join(directory, `vm-${randomUUID()}.csr`); + const certPath = join(directory, `vm-${randomUUID()}.pem`); + const extensionPath = join(directory, `vm-${randomUUID()}.ext`); + await writeFile(csrPath, csr, { mode: 0o600 }); + await writeFile(extensionPath, `basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyAgreement\nextendedKeyUsage=clientAuth\nsubjectAltName=URI:${identityUri}\n`, { mode: 0o600 }); + try { + await openssl(["req", "-in", csrPath, "-verify", "-noout"]); + await openssl(["x509", "-req", "-in", csrPath, "-CA", caCertPath, "-CAkey", caKeyPath, "-CAcreateserial", "-days", "1", "-sha256", "-subj", `/CN=${vmIdentity}`, "-extfile", extensionPath, "-out", certPath]); + return Object.freeze({ certificate: pem(await readFile(certPath)), trustBundle: pem(ca) }); + } finally { + await Promise.all([rm(csrPath, { force: true }), rm(certPath, { force: true }), rm(extensionPath, { force: true })]); + } + }, + async close() { + if (closed) return; + closed = true; + await rm(directory, { recursive: true, force: true }); + }, + }); + } catch (error) { + await rm(directory, { recursive: true, force: true }); + throw error; + } +} diff --git a/packages/credential-proxy/src/vm-certificate.mjs b/packages/credential-proxy/src/vm-certificate.mjs new file mode 100644 index 00000000..a06a743a --- /dev/null +++ b/packages/credential-proxy/src/vm-certificate.mjs @@ -0,0 +1,61 @@ +import { connect } from "node:net"; + +const SOCKET_PATH = "/run/factorize-proxy/bootstrap.sock"; +const MAX_REQUEST_BYTES = 20 * 1024; +const MAX_RESPONSE_BYTES = 40 * 1024; + +async function readRequest() { + let length = 0; + const chunks = []; + for await (const chunk of process.stdin) { + length += chunk.byteLength; + if (length > MAX_REQUEST_BYTES) throw new Error("invalid certificate request"); + chunks.push(chunk); + } + if (!length || chunks.at(-1).at(-1) !== 10) throw new Error("invalid certificate request"); + const value = JSON.parse(Buffer.concat(chunks).toString("utf8")); + if (!value || typeof value !== "object" || Array.isArray(value) + || Object.keys(value).sort().join(",") !== "bootEpoch,csr,vmIdentity" + || typeof value.bootEpoch !== "string" || typeof value.csr !== "string" || typeof value.vmIdentity !== "string") { + throw new Error("invalid certificate request"); + } + return { type: "issue-vm-certificate", ...value }; +} + +async function send(request) { + const socket = connect(SOCKET_PATH); + try { + return await new Promise((resolve, reject) => { + let data = Buffer.alloc(0); + socket.on("data", chunk => { + data = Buffer.concat([data, chunk]); + if (data.length > MAX_RESPONSE_BYTES) { + reject(new Error("invalid certificate response")); + socket.destroy(); + return; + } + const end = data.indexOf(10); + if (end < 0) return; + try { resolve(JSON.parse(data.subarray(0, end).toString("utf8"))); } + catch { reject(new Error("invalid certificate response")); } + socket.end(); + }); + socket.once("error", () => reject(new Error("proxy certificate service unavailable"))); + socket.setTimeout(7000, () => reject(new Error("proxy certificate service timed out"))); + socket.once("connect", () => socket.write(`${JSON.stringify(request)}\n`)); + }); + } finally { + socket.destroy(); + } +} + +try { + const request = await readRequest(); + const result = await send(request); + if (result?.issued !== true || result.bootEpoch !== request.bootEpoch || result.vmIdentity !== request.vmIdentity + || typeof result.certificate !== "string" || typeof result.trustBundle !== "string") throw new Error("VM certificate was not issued"); + process.stdout.write(`${JSON.stringify({ bootEpoch: result.bootEpoch, vmIdentity: result.vmIdentity, certificate: result.certificate, trustBundle: result.trustBundle })}\n`); +} catch { + process.stderr.write("VM certificate provisioning failed\n"); + process.exitCode = 1; +} diff --git a/packages/credential-proxy/test/bootstrap-control.test.mjs b/packages/credential-proxy/test/bootstrap-control.test.mjs index ece8a05e..5e7956ac 100644 --- a/packages/credential-proxy/test/bootstrap-control.test.mjs +++ b/packages/credential-proxy/test/bootstrap-control.test.mjs @@ -17,11 +17,11 @@ afterEach(async () => { directory = undefined; }); -async function fixture(register, onRegistered) { +async function fixture(register, onRegistered, issueVmCertificate) { directory = await mkdtemp(join(tmpdir(), "factorize-proxy-startup-")); const socketPath = join(directory, "control.sock"); const runtime = { bootEpoch: epoch }; - service = await startProxyBootstrapControl({ runtime, socketPath, register, onRegistered }); + service = await startProxyBootstrapControl({ runtime, socketPath, register, onRegistered, issueVmCertificate }); return socketPath; } @@ -42,7 +42,7 @@ test("local startup control redeems the one-use credential and leaves a private async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }; }, (value, request) => registered.push({ value, request }), ); - const result = await request(socketPath, { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + const result = await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); assert.deepEqual(result, { registered: true, proxyIdentity: "fly:app:machine", bootEpoch: epoch }); assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch }]); assert.deepEqual(registered, [{ @@ -57,9 +57,29 @@ test("local startup control rejects run credentials, epoch substitution, and ext const calls = []; const socketPath = await fixture(async value => { calls.push(value); return {}; }); for (const payload of [ - { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, - { controlOrigin: "https://factorize.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, - { controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, + { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, + { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, + { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, ]) assert.deepEqual(await request(socketPath, payload), { error: "invalid bootstrap request" }); assert.deepEqual(calls, []); }); + +test("local provider channel returns only a boot-bound VM certificate and public trust bundle", async () => { + const csr = `-----BEGIN CERTIFICATE REQUEST-----\n${"A".repeat(120)}\n-----END CERTIFICATE REQUEST-----`; + let registered = false; + const socketPath = await fixture( + async () => ({ proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }), + async () => { registered = true; }, + async request => { + assert.equal(registered, true); + return { certificate: `cert-for:${request.vmIdentity}`, trustBundle: "public-ca" }; + }, + ); + const payload = { type: "issue-vm-certificate", bootEpoch: epoch, vmIdentity: "fly:job-1", csr }; + assert.deepEqual(await request(socketPath, payload), { error: "certificate provisioning failed" }); + await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + assert.deepEqual(await request(socketPath, payload), { + issued: true, bootEpoch: epoch, vmIdentity: "fly:job-1", certificate: "cert-for:fly:job-1", trustBundle: "public-ca", + }); + assert.equal(JSON.stringify(await request(socketPath, payload)).includes(token), false); +}); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index c8386843..1c275e3b 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, registerProxyBoot } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, readProxyRunRevocations, registerProxyBoot, startProxyRunRevocationMonitor } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -34,6 +34,33 @@ test("proxy snapshot reader requests only the bound run revision and validates e await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ownerId: "owner-1", records: [{ ...snapshot.records[0], kind: "model" }] }) }), /invalid proxy integration snapshot response/); }); +test("proxy revocation monitor applies durable run revocations and tolerates an outage", async () => { + const requests = []; + const revoked = []; + let failFirst = true; + const fetchImpl = async (url, init) => { + requests.push({ url: String(url), init, body: JSON.parse(init.body) }); + if (failFirst) { failFirst = false; throw new Error("offline fixture"); } + return Response.json({ revokedRunIds: [runId] }); + }; + const runtime = { revokeRun: async id => { revoked.push(id); } }; + const monitor = startProxyRunRevocationMonitor({ + controlOrigin: "https://factorize.example", controlCredential: token, bootEpoch, runtime, intervalMs: 100, fetchImpl, + }); + try { + const deadline = Date.now() + 1_000; + while (!revoked.length && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 10)); + assert.deepEqual(revoked, [runId]); + assert.equal(requests.length, 2, "a failed poll leaves the proxy live and retries through the control plane"); + assert.ok(requests.every(request => request.url === "https://factorize.example/internal/credential-proxy/revocations")); + assert.ok(requests.every(request => request.init.headers.Authorization === `Bearer ${token}`)); + assert.ok(requests.every(request => request.body.bootEpoch === bootEpoch)); + } finally { + await monitor.close(); + } + await assert.rejects(readProxyRunRevocations({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async () => Response.json({ revokedRunIds: ["invalid"] }) }), /invalid proxy revocation response/); +}); + test("boot registration rejects unsafe origins, invalid credentials and mismatched response bindings", async () => { await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch }), /HTTPS origin/); await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); diff --git a/packages/credential-proxy/test/tls-identity.test.mjs b/packages/credential-proxy/test/tls-identity.test.mjs new file mode 100644 index 00000000..dd09554d --- /dev/null +++ b/packages/credential-proxy/test/tls-identity.test.mjs @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { execFile as execFileCallback } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { test } from "node:test"; +import { createProxyTlsAuthority } from "../src/tls-identity.mjs"; + +const execFile = promisify(execFileCallback); +const bootEpoch = "11111111-1111-4111-8111-111111111111"; + +test("proxy TLS authority provisions a boot-bound server cert and signs only canonical VM identities", async () => { + const directory = await mkdtemp(join(tmpdir(), "factorize-tls-test-")); + const authority = await createProxyTlsAuthority({ serverName: "proxy.internal.test", bootEpoch }); + try { + const keyPath = join(directory, "vm-key.pem"); + const csrPath = join(directory, "vm.csr"); + await execFile("openssl", ["genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", "-out", keyPath]); + await execFile("openssl", ["req", "-new", "-key", keyPath, "-subj", "/CN=ignored-by-issuer", "-out", csrPath]); + const csr = await readFile(csrPath, "utf8"); + const issued = await authority.signVmCsr({ vmIdentity: "fly:machine-7", csr }); + assert.equal(issued.trustBundle, authority.trustBundle); + assert.match(issued.certificate, /BEGIN CERTIFICATE/); + await writeFile(join(directory, "root.pem"), issued.trustBundle, { mode: 0o600 }); + await writeFile(join(directory, "client.pem"), issued.certificate, { mode: 0o600 }); + await execFile("openssl", ["verify", "-CAfile", join(directory, "root.pem"), join(directory, "client.pem")]); + const { stdout: clientText } = await execFile("openssl", ["x509", "-in", join(directory, "client.pem"), "-noout", "-subject", "-ext", "subjectAltName"]); + assert.match(clientText, /CN\s*=\s*fly:machine-7/); + assert.match(clientText, /URI:urn:factorize:vm:Zmx5Om1hY2hpbmUtNw/); + const { stdout: usageText } = await execFile("openssl", ["x509", "-in", join(directory, "client.pem"), "-noout", "-ext", "extendedKeyUsage"]); + assert.match(usageText, /TLS Web Client Authentication/); + + await writeFile(join(directory, "server.pem"), authority.tls.cert, { mode: 0o600 }); + await writeFile(join(directory, "server-root.pem"), authority.tls.ca, { mode: 0o600 }); + await execFile("openssl", ["verify", "-CAfile", join(directory, "server-root.pem"), "-verify_hostname", "proxy.internal.test", join(directory, "server.pem")]); + await assert.rejects(authority.signVmCsr({ vmIdentity: "../machine", csr }), /invalid VM certificate request/); + await assert.rejects(authority.signVmCsr({ vmIdentity: "fly:machine-7", csr: "not a CSR" })); + await authority.close(); + await assert.rejects(authority.signVmCsr({ vmIdentity: "fly:machine-7", csr }), /authority is closed/); + } finally { + await authority.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("proxy TLS authority rejects malformed server names and boot identities", async () => { + await assert.rejects(createProxyTlsAuthority({ serverName: "https://proxy.test", bootEpoch })); + await assert.rejects(createProxyTlsAuthority({ serverName: "proxy.test", bootEpoch: "old" })); +}); From 41b91d22f6316e438e12141f6953e0513616a70a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 23:58:36 +0000 Subject: [PATCH 077/102] GEN-2824 include selected model account snapshot --- packages/api/src/proxy-control.ts | 15 ++++++++++++++- packages/api/test/proxy-control.test.ts | 14 +++++++++----- packages/credential-proxy/src/control-client.mjs | 5 ++++- .../credential-proxy/test/control-client.test.mjs | 3 ++- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index fa91e53c..18ef39f3 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -68,7 +68,20 @@ export async function proxyControlFetch(request: Request, env: Parameters selected.includes(record.id)) : []; if (records.length !== selected.length) return json({ error: "integration_snapshot_unavailable" }, 403); - return json({ ownerId: proxy.tenantId, records }); + const modelAccess = run.execution_config?.modelAccess ?? "native"; + const modelAccountId = run.execution_config?.modelAccountId; + let modelAccount = null; + if (modelAccess === "portable") { + if (typeof modelAccountId !== "string") return json({ error: "integration_snapshot_unavailable" }, 403); + modelAccount = (await new EncryptedIntegrationRepository(database, proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("model")) + .find(record => record.id === modelAccountId) ?? null; + if (!modelAccount) return json({ error: "integration_snapshot_unavailable" }, 403); + } else if (modelAccess !== "native" || modelAccountId !== undefined) { + return json({ error: "integration_snapshot_unavailable" }, 403); + } + // Only the selected model account's user-encrypted payload crosses the + // control plane. It remains encrypted until the proxy is unlocked. + return json({ ownerId: proxy.tenantId, records, modelAccount }); } const enrollments = new RunEnrollmentRepository(databaseFor(env)); if (url.pathname === "/internal/credential-proxy/enroll") { diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 7a8cd90d..42a02a58 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -122,17 +122,21 @@ describe("dedicated proxy boot registration endpoint", () => { expect(rejected.status).toBe(400); }); - it("returns only the current run's selected user-encrypted bearer records to its registered proxy", async () => { + it("returns only the current run's selected user-encrypted bearer and model records to its registered proxy", async () => { const integrationId = "55555555-5555-4555-8555-555555555555"; + const modelAccountId = "77777777-7777-4777-8777-777777777777"; const key = Buffer.alloc(32, 7).toString("base64"); const env = envFor(); env.CREDENTIAL_ENCRYPTION_KEY = key; - env.query.mockImplementation(async (sql: string) => { + env.query.mockImplementation(async (sql: string, values: unknown[] = []) => { if (sql.includes("proxy_token_digest")) return { rows: [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }], rowCount: 1 }; - if (sql.includes("FROM app.runs r")) return { rows: [{ execution_config: { portableIntegrationIds: [integrationId] }, grant_revision: 3 }], rowCount: 1 }; - if (sql.includes("FROM app.integration_records")) return { rows: [ + if (sql.includes("FROM app.runs r")) return { rows: [{ execution_config: { portableIntegrationIds: [integrationId], modelAccess: "portable", modelAccountId }, grant_revision: 3 }], rowCount: 1 }; + if (sql.includes("FROM app.integration_records")) return values[1] === "bearer" ? { rows: [ { id: integrationId, kind: "bearer", name: "selected", origin: "https://api.example", description: null, revision: 4, encrypted_payload: await encrypt("user-ciphertext", key) }, { id: "66666666-6666-4666-8666-666666666666", kind: "bearer", name: "not-selected", origin: "https://other.example", description: null, revision: 5, encrypted_payload: await encrypt("other-ciphertext", key) }, + ], rowCount: 2 } : { rows: [ + { id: modelAccountId, kind: "model", name: "selected-model", origin: null, description: "fixture account", revision: 6, encrypted_payload: await encrypt("model-user-ciphertext", key) }, + { id: "88888888-8888-4888-8888-888888888888", kind: "model", name: "other-model", origin: null, description: null, revision: 7, encrypted_payload: await encrypt("other-model-ciphertext", key) }, ], rowCount: 2 }; return { rows: [], rowCount: 0 }; }); @@ -144,7 +148,7 @@ describe("dedicated proxy boot registration endpoint", () => { expect(response.status).toBe(200); expect(await response.json()).toEqual({ ownerId: "11111111-1111-4111-8111-111111111111", records: [ { id: integrationId, kind: "bearer", name: "selected", origin: "https://api.example", description: null, revision: 4, encryptedPayload: "user-ciphertext" }, - ] }); + ], modelAccount: { id: modelAccountId, kind: "model", name: "selected-model", origin: null, description: "fixture account", revision: 6, encryptedPayload: "model-user-ciphertext" } }); expect(env.query.mock.calls.map((call: any[]) => call[0])).toEqual(expect.arrayContaining([expect.stringContaining("grant_revision=$5"), expect.stringContaining("FROM app.integration_records")])); }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index fd1b4c46..f36c3fa4 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -65,9 +65,12 @@ export async function readProxyIntegrationSnapshot({ controlOrigin, token, bindi body: { runId: binding.runId, grantRevision: binding.grantRevision }, fetchImpl, }); + const modelAccount = result?.modelAccount; if (typeof result?.ownerId !== "string" || !Array.isArray(result.records) || result.records.some(record => !record || typeof record.id !== "string" || record.kind !== "bearer" - || typeof record.origin !== "string" || typeof record.encryptedPayload !== "string" || !Number.isSafeInteger(record.revision))) { + || typeof record.origin !== "string" || typeof record.encryptedPayload !== "string" || !Number.isSafeInteger(record.revision)) + || !(modelAccount === null || (modelAccount && typeof modelAccount.id === "string" && modelAccount.kind === "model" + && modelAccount.origin === null && typeof modelAccount.encryptedPayload === "string" && Number.isSafeInteger(modelAccount.revision)))) { throw new Error("invalid proxy integration snapshot response"); } return result; diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 1c275e3b..b3ab4cbd 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -22,7 +22,7 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an test("proxy snapshot reader requests only the bound run revision and validates encrypted records", async () => { let request; - const snapshot = { ownerId: "owner-1", records: [{ id: "integration-1", kind: "bearer", origin: "https://api.example", revision: 4, encryptedPayload: "ciphertext" }] }; + const snapshot = { ownerId: "owner-1", records: [{ id: "integration-1", kind: "bearer", origin: "https://api.example", revision: 4, encryptedPayload: "ciphertext" }], modelAccount: { id: "model-1", kind: "model", origin: null, revision: 5, encryptedPayload: "model-ciphertext" } }; const result = await readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async (url, init) => { request = { url: String(url), init, body: JSON.parse(init.body) }; return Response.json(snapshot); @@ -32,6 +32,7 @@ test("proxy snapshot reader requests only the bound run revision and validates e assert.deepEqual(request.body, { runId, grantRevision: 7 }); assert.equal(request.init.headers.Authorization, `Bearer ${token}`); await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ownerId: "owner-1", records: [{ ...snapshot.records[0], kind: "model" }] }) }), /invalid proxy integration snapshot response/); + await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ...snapshot, modelAccount: { ...snapshot.modelAccount, kind: "bearer" } }) }), /invalid proxy integration snapshot response/); }); test("proxy revocation monitor applies durable run revocations and tolerates an outage", async () => { From 141190986c459be79695afb70bb902e1494b1c1d Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:02:00 +0000 Subject: [PATCH 078/102] GEN-2824 update snapshot fixture for model selection --- packages/credential-proxy/test/decrypted-grants.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/credential-proxy/test/decrypted-grants.test.mjs b/packages/credential-proxy/test/decrypted-grants.test.mjs index 18ad4d62..5b41501e 100644 --- a/packages/credential-proxy/test/decrypted-grants.test.mjs +++ b/packages/credential-proxy/test/decrypted-grants.test.mjs @@ -80,7 +80,7 @@ test("production grant resolver reads the run-bound snapshot and only decrypts w controlCredential: `fzc_${"c".repeat(43)}`, fetchImpl: async (url, init) => { requests.push({ url: String(url), authorization: new Headers(init.headers).get("authorization"), body: JSON.parse(init.body) }); - return Response.json({ ownerId, records: [{ ...record, encryptedPayload }] }); + return Response.json({ ownerId, records: [{ ...record, encryptedPayload }], modelAccount: null }); }, }); const binding = { tenantId: ownerId, runId: "22222222-2222-4222-8222-222222222222", grantRevision: 7 }; From 850e36a18d6e7e68db975e73af3e508239e07eef Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:25:34 +0000 Subject: [PATCH 079/102] GEN-2824 add isolated Codex auth helper --- .../src/codex-auth-helper.mjs | 158 ++++++++++++++++++ .../test/codex-auth-helper.test.mjs | 84 ++++++++++ 2 files changed, 242 insertions(+) create mode 100644 packages/credential-proxy/src/codex-auth-helper.mjs create mode 100644 packages/credential-proxy/test/codex-auth-helper.test.mjs diff --git a/packages/credential-proxy/src/codex-auth-helper.mjs b/packages/credential-proxy/src/codex-auth-helper.mjs new file mode 100644 index 00000000..199d906f --- /dev/null +++ b/packages/credential-proxy/src/codex-auth-helper.mjs @@ -0,0 +1,158 @@ +import { mkdir, lstat } from "node:fs/promises"; +import { resolve, sep } from "node:path"; +import { spawn as spawnDefault } from "node:child_process"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const PINNED_VERSION = "codex-cli 0.167.1"; +const METHODS = new Set(["device-login", "account-status", "refresh", "cancel", "logout"]); + +function accountHome(root, accountId) { + if (typeof accountId !== "string" || !UUID.test(accountId)) throw new TypeError("invalid model account id"); + const base = resolve(root); + const home = resolve(base, accountId); + if (!home.startsWith(`${base}${sep}`)) throw new TypeError("invalid model account id"); + return { base, home }; +} + +/** + * Narrow in-process adapter for Codex authentication commands. Callers must + * keep this object on the proxy's private control channel; it is not a job API. + * Each account gets its own normal Codex credential home. + */ +export function createCodexAuthHelper({ + codexPath = "/usr/local/bin/codex", + accountHomeRoot = "/var/lib/factorize/codex-accounts", + spawnProcess = spawnDefault, + expectedVersion = PINNED_VERSION, +} = {}) { + const logins = new Map(); + + async function prepareHome(accountId) { + const { base, home } = accountHome(accountHomeRoot, accountId); + await mkdir(base, { recursive: true, mode: 0o700 }); + await mkdir(home, { recursive: true, mode: 0o700 }); + const stat = await lstat(home); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Codex account home is not a private directory"); + return home; + } + + function run(accountId, args, { timeoutMs = 10_000, input, onStdout } = {}) { + return prepareHome(accountId).then(home => new Promise((resolveRun, rejectRun) => { + let stdout = "", stderr = "", settled = false; + const child = spawnProcess(codexPath, args, { + env: { PATH: process.env.PATH ?? "", CODEX_HOME: home, HOME: home }, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + const finish = (error, result) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) rejectRun(error); + else resolveRun(result); + }; + const collect = (current, chunk, limit) => { + const next = current + chunk.toString("utf8"); + if (Buffer.byteLength(next) > limit) { + child.kill("SIGTERM"); + finish(new Error("Codex authentication helper output exceeded its limit")); + return current; + } + return next; + }; + child.stdout.on("data", chunk => { + stdout = collect(stdout, chunk, 16_384); + if (!settled) onStdout?.(chunk.toString("utf8")); + }); + child.stderr.on("data", chunk => { stderr = collect(stderr, chunk, 16_384); }); + child.once("error", () => finish(new Error("Codex authentication helper could not start"))); + child.once("close", (code, signal) => { + if (code === 0) finish(null, { code, stdout, stderr }); + else finish(new Error(signal ? `Codex authentication helper stopped (${signal})` : `Codex authentication helper failed (${code ?? "unknown"})`)); + }); + const timer = setTimeout(() => { + child.kill("SIGTERM"); + finish(new Error("Codex authentication helper timed out")); + }, timeoutMs); + timer.unref?.(); + if (input !== undefined) child.stdin.end(input); + else child.stdin.end(); + return child; + })); + } + + async function checkVersion(accountId) { + // `--version` is the only non-authentication command accepted by this helper. + const result = await run(accountId, ["--version"]); + if (result.stdout.trim() !== expectedVersion) throw new Error("Unsupported Codex version for model account authentication"); + } + + async function invoke(request) { + if (!request || typeof request !== "object" || Array.isArray(request) + || !METHODS.has(request.method) || Object.keys(request).some(key => !["method", "accountId"].includes(key))) { + throw new TypeError("unsupported Codex authentication operation"); + } + const { accountId } = request; + accountHome(accountHomeRoot, accountId); + await checkVersion(accountId); + if (request.method === "device-login") { + if (logins.has(accountId)) throw new Error("device authorization is already running for this account"); + const home = await prepareHome(accountId); + let child; + const completion = new Promise((resolveRun, rejectRun) => { + child = spawnProcess(codexPath, ["login", "--device-auth"], { + env: { PATH: process.env.PATH ?? "", CODEX_HOME: home, HOME: home }, + stdio: ["ignore", "pipe", "pipe"], windowsHide: true, + }); + let stdout = ""; + const timer = setTimeout(() => { child.kill("SIGTERM"); rejectRun(new Error("Codex device authorization timed out")); }, 15 * 60_000); + timer.unref?.(); + child.stdout.on("data", chunk => { + stdout = (stdout + chunk.toString("utf8")).slice(-16_384); + }); + child.stderr.on("data", chunk => { + // Retain no provider error text, which could include device details. + if (chunk.byteLength > 16_384) child.kill("SIGTERM"); + }); + child.once("error", () => { clearTimeout(timer); rejectRun(new Error("Codex authentication helper could not start")); }); + child.once("close", (code, signal) => { + clearTimeout(timer); + if (code === 0) resolveRun({ state: "authorized", output: stdout }); + else rejectRun(new Error(signal ? `Codex device authorization stopped (${signal})` : `Codex device authorization failed (${code ?? "unknown"})`)); + }); + }); + const entry = { child, completion }; + logins.set(accountId, entry); + completion.finally(() => { if (logins.get(accountId) === entry) logins.delete(accountId); }).catch(() => {}); + return { state: "pending", completion }; + } + if (request.method === "cancel") { + const login = logins.get(accountId); + if (!login) return { state: "idle" }; + login.child.kill("SIGTERM"); + logins.delete(accountId); + return { state: "canceled" }; + } + if (request.method === "logout") { + const login = logins.get(accountId); + if (login) { + login.child.kill("SIGTERM"); + logins.delete(accountId); + } + await run(accountId, ["logout"]); + return { state: "logged-out" }; + } + if (request.method === "account-status") { + const result = await run(accountId, ["login", "status"]); + return { state: result.stdout.trim() ? "authenticated" : "unauthenticated" }; + } + // Codex owns refresh semantics in its authentication manager. A status call + // exercises that manager without opening the agent command surface. + const result = await run(accountId, ["login", "status"]); + return { state: result.stdout.trim() ? "refreshed" : "reauthentication-required" }; + } + + return Object.freeze({ invoke }); +} + +export { PINNED_VERSION as CODEX_AUTH_HELPER_VERSION }; diff --git a/packages/credential-proxy/test/codex-auth-helper.test.mjs b/packages/credential-proxy/test/codex-auth-helper.test.mjs new file mode 100644 index 00000000..1b386c1d --- /dev/null +++ b/packages/credential-proxy/test/codex-auth-helper.test.mjs @@ -0,0 +1,84 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createCodexAuthHelper, CODEX_AUTH_HELPER_VERSION } from "../src/codex-auth-helper.mjs"; + +const accountA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const accountB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + +function fakeCodex() { + const calls = [], children = []; + const spawnProcess = (command, args, options) => { + calls.push({ command, args, options }); + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.stdin = new PassThrough(); + child.kill = signal => { + child.killed = signal; + queueMicrotask(() => child.emit("close", null, signal)); + return true; + }; + children.push(child); + if (args[0] !== "login" || args[1] !== "--device-auth") { + queueMicrotask(() => { + child.stdout.end(args[0] === "--version" ? `${CODEX_AUTH_HELPER_VERSION}\n` : "authenticated\n"); + child.stderr.end(); + child.emit("close", 0, null); + }); + } + return child; + }; + return { calls, children, spawnProcess }; +} + +test("Codex auth helper constrains commands and isolates account credential homes", async () => { + const directory = await mkdtemp(join(tmpdir(), "factorize-codex-auth-")); + try { + const fake = fakeCodex(); + const helper = createCodexAuthHelper({ codexPath: "/opt/codex", accountHomeRoot: directory, spawnProcess: fake.spawnProcess }); + assert.deepEqual(await helper.invoke({ method: "account-status", accountId: accountA }), { state: "authenticated" }); + assert.deepEqual(await helper.invoke({ method: "refresh", accountId: accountB }), { state: "refreshed" }); + assert.deepEqual(fake.calls.map(call => call.args), [ + ["--version"], ["login", "status"], ["--version"], ["login", "status"], + ]); + assert.ok(fake.calls.every(call => call.command === "/opt/codex")); + assert.equal(fake.calls[0].options.env.CODEX_HOME, join(directory, accountA)); + assert.equal(fake.calls[3].options.env.CODEX_HOME, join(directory, accountB)); + assert.equal(fake.calls[0].options.env.HOME, join(directory, accountA)); + assert.equal("OPENAI_API_KEY" in fake.calls[0].options.env, false); + await assert.rejects(helper.invoke({ method: "exec", accountId: accountA, args: ["--dangerously-bypass-approvals-and-sandbox"] }), /unsupported Codex authentication operation/); + await assert.rejects(helper.invoke({ method: "logout", accountId: "../../outside" }), /invalid model account id/); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +test("device authorization runs as a separate cancellable login process", async () => { + const directory = await mkdtemp(join(tmpdir(), "factorize-codex-auth-")); + try { + const fake = fakeCodex(); + const helper = createCodexAuthHelper({ accountHomeRoot: directory, spawnProcess: fake.spawnProcess }); + const result = await helper.invoke({ method: "device-login", accountId: accountA }); + assert.equal(result.state, "pending"); + assert.deepEqual(fake.calls.map(call => call.args), [["--version"], ["login", "--device-auth"]]); + assert.equal(fake.calls[1].options.env.CODEX_HOME, join(directory, accountA)); + assert.deepEqual(await helper.invoke({ method: "cancel", accountId: accountA }), { state: "canceled" }); + await assert.rejects(result.completion, /stopped/); + assert.deepEqual(await helper.invoke({ method: "cancel", accountId: accountA }), { state: "idle" }); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +test("logout terminates pending device login and invokes only Codex logout", async () => { + const directory = await mkdtemp(join(tmpdir(), "factorize-codex-auth-")); + try { + const fake = fakeCodex(); + const helper = createCodexAuthHelper({ accountHomeRoot: directory, spawnProcess: fake.spawnProcess }); + const result = await helper.invoke({ method: "device-login", accountId: accountA }); + assert.deepEqual(await helper.invoke({ method: "logout", accountId: accountA }), { state: "logged-out" }); + await assert.rejects(result.completion, /stopped/); + assert.deepEqual(fake.calls.map(call => call.args), [["--version"], ["login", "--device-auth"], ["--version"], ["logout"]]); + } finally { await rm(directory, { recursive: true, force: true }); } +}); From 75f9df0a69112c1ab8a2608a02fdfef52c3a13fe Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:41:37 +0000 Subject: [PATCH 080/102] GEN-2824 persist encrypted Codex account refreshes --- .../src/model-account-lifecycle.mjs | 154 ++++++++++++++++++ .../test/model-account-lifecycle.test.mjs | 113 +++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 packages/credential-proxy/src/model-account-lifecycle.mjs create mode 100644 packages/credential-proxy/test/model-account-lifecycle.test.mjs diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs new file mode 100644 index 00000000..236d4008 --- /dev/null +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -0,0 +1,154 @@ +import { readFile, rm } from "node:fs/promises"; +import { join, resolve, sep } from "node:path"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function cachePath(root, accountId) { + if (!UUID.test(accountId ?? "")) throw new TypeError("invalid model account id"); + const base = resolve(root), path = resolve(base, accountId, "auth.json"); + if (!path.startsWith(`${base}${sep}`)) throw new TypeError("invalid model account id"); + return path; +} + +/** + * Keep Codex token refresh and encrypted persistence inside the proxy. The + * persistence callback receives only user-encrypted auth-cache bytes; token + * JSON and the unlock key never leave this process in plaintext. + */ +export function createModelAccountLifecycle({ + authHelper, + accountHomeRoot = "/var/lib/factorize/codex-accounts", + getUnlockKey, + getAccount, + encryptCache, + persistEncrypted, + onUnsaved = () => {}, + retryDelaysMs = [100, 500, 2_000], + sleep = ms => new Promise(resolveSleep => setTimeout(resolveSleep, ms)), + removeCache = rm, + readCache = readFile, +} = {}) { + if (!authHelper || typeof authHelper.invoke !== "function" || typeof getUnlockKey !== "function" + || typeof getAccount !== "function" || typeof encryptCache !== "function" || typeof persistEncrypted !== "function" + || !Array.isArray(retryDelaysMs) || retryDelaysMs.length > 5 + || retryDelaysMs.some(ms => !Number.isSafeInteger(ms) || ms < 0 || ms > 30_000)) { + throw new TypeError("model account lifecycle configuration is invalid"); + } + + const generations = new Map(); + const queues = new Map(); + const unsaved = new Set(); + const currentGeneration = id => generations.get(id) ?? 0; + + function serialize(id, operation) { + const previous = queues.get(id) ?? Promise.resolve(); + const next = previous.catch(() => {}).then(operation); + queues.set(id, next); + return next.finally(() => { if (queues.get(id) === next) queues.delete(id); }); + } + + async function persist(id, generation) { + const account = await getAccount(id); + if (!account || account.id !== id || account.kind !== "model" || !Number.isSafeInteger(account.revision) + || typeof account.ownerId !== "string" || !account.ownerId || typeof account.keyGeneration !== "string" || !account.keyGeneration) { + throw new Error("model account persistence binding is unavailable"); + } + const unlockKey = await getUnlockKey(); + if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); + const plain = await readCache(cachePath(accountHomeRoot, id)); + if (!Buffer.isBuffer(plain) || plain.byteLength === 0 || plain.byteLength > 65_536) throw new Error("Codex token cache is missing or exceeds its limit"); + let encrypted; + try { + encrypted = await encryptCache(new Uint8Array(plain), unlockKey, { + ownerId: account.ownerId, + recordId: id, + recordType: "model", + destination: "factorize", + revision: account.revision + 1, + keyGeneration: account.keyGeneration, + }); + } finally { + plain.fill(0); + } + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted }; + for (let attempt = 0; ; attempt++) { + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + try { + await persistEncrypted(request); + unsaved.delete(id); + onUnsaved({ accountId: id, unsaved: false }); + return { state: "saved" }; + } catch (error) { + if (attempt >= retryDelaysMs.length) { + unsaved.add(id); + onUnsaved({ accountId: id, unsaved: true }); + return { state: "unsaved", attempts: attempt + 1 }; + } + await sleep(retryDelaysMs[attempt]); + } + } + } + + async function refresh(id) { + return serialize(id, async () => { + const generation = currentGeneration(id); + const result = await authHelper.invoke({ method: "refresh", accountId: id }); + if (result.state === "reauthentication-required") return result; + return persist(id, generation); + }); + } + + async function deviceLogin(id) { + return serialize(id, async () => { + const generation = currentGeneration(id); + const result = await authHelper.invoke({ method: "device-login", accountId: id }); + if (result.state !== "pending" || !result.completion || typeof result.completion.then !== "function") throw new Error("Codex device authorization did not start"); + const completion = Promise.resolve(result.completion).then(async outcome => { + if (currentGeneration(id) !== generation) return { state: "deleted" }; + if (outcome?.state !== "authorized") return { state: outcome?.state ?? "reauthentication-required" }; + return persist(id, generation); + }); + return { state: "pending", completion }; + }); + } + + async function cancel(id) { + return serialize(id, async () => authHelper.invoke({ method: "cancel", accountId: id })); + } + + async function deleteAccount(id) { + if (!UUID.test(id ?? "")) throw new TypeError("invalid model account id"); + // Invalidate in-flight save retries immediately; cleanup then waits behind + // any already running helper process before deleting its cache. + generations.set(id, currentGeneration(id) + 1); + return serialize(id, async () => { + await authHelper.invoke({ method: "cancel", accountId: id }).catch(() => {}); + await authHelper.invoke({ method: "logout", accountId: id }).catch(() => {}); + await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); + unsaved.delete(id); + onUnsaved({ accountId: id, unsaved: false }); + return { state: "deleted" }; + }); + } + + return Object.freeze({ + deviceLogin, + cancel, + refresh, + deleteAccount, + isUnsaved: id => unsaved.has(id), + /** Invoke after encrypted snapshots are reconciled at unlock. */ + reconcile: async id => serialize(id, async () => { + if (!await getAccount(id)) { + await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); + unsaved.delete(id); + return { state: "deleted" }; + } + const result = await authHelper.invoke({ method: "account-status", accountId: id }); + return result.state === "authenticated" ? persist(id, currentGeneration(id)) : { state: "reauthentication-required" }; + }), + }); +} + +export { cachePath as modelAccountCachePath }; diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs new file mode 100644 index 00000000..449fbdb4 --- /dev/null +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -0,0 +1,113 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createModelAccountLifecycle } from "../src/model-account-lifecycle.mjs"; + +const accountId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const account = { id: accountId, kind: "model", ownerId: "owner", keyGeneration: "generation", revision: 4 }; + +async function fixture(t, overrides = {}) { + const root = await mkdtemp(join(tmpdir(), "factorize-model-account-")); + t.after(() => rm(root, { recursive: true, force: true })); + const home = join(root, accountId); + await mkdir(home, { recursive: true }); + await writeFile(join(home, "auth.json"), JSON.stringify({ tokens: { access_token: "synthetic-access", refresh_token: "synthetic-refresh" } })); + const calls = [], events = [], methods = []; + const helper = { invoke: async request => { + methods.push(request.method); + if (request.method === "device-login") return { state: "pending", completion: overrides.completion ?? Promise.resolve({ state: "authorized" }) }; + if (request.method === "refresh") { calls.push("refresh"); return { state: "refreshed" }; } + if (request.method === "account-status") return { state: "authenticated" }; + return { state: request.method === "cancel" ? "canceled" : "logged-out" }; + } }; + let saves = 0; + const lifecycle = createModelAccountLifecycle({ + authHelper: helper, + accountHomeRoot: root, + getUnlockKey: async () => "synthetic-unlock-key", + getAccount: async () => overrides.getAccount ? overrides.getAccount() : account, + encryptCache: async (plain, key, binding) => { + assert.equal(key, "synthetic-unlock-key"); + events.push({ type: "encrypt", binding, plaintext: new TextDecoder().decode(plain) }); + return `encrypted.${Buffer.from(plain).toString("base64")}`; + }, + persistEncrypted: async request => { + saves++; + events.push({ type: "persist", request }); + if (overrides.persistEncrypted) return overrides.persistEncrypted(request, saves); + }, + onUnsaved: value => events.push({ type: "unsaved", ...value }), + retryDelaysMs: overrides.retryDelaysMs ?? [], + sleep: async () => {}, + ...overrides.options, + }); + return { root, home, lifecycle, events, methods, calls, get saves() { return saves; } }; +} + +test("device authorization persists only encrypted, account-bound cache after success", async t => { + const f = await fixture(t); + const login = await f.lifecycle.deviceLogin(accountId); + assert.equal(login.state, "pending"); + assert.deepEqual(await login.completion, { state: "saved" }); + const [encrypted, saved] = f.events.filter(event => event.type === "encrypt" || event.type === "persist"); + assert.equal(encrypted.binding.ownerId, "owner"); + assert.equal(encrypted.binding.recordType, "model"); + assert.equal(encrypted.binding.revision, 5); + assert.equal(saved.request.expectedRevision, 4); + assert.match(saved.request.encryptedPayload, /^encrypted\./); + assert.equal("unlockKey" in saved.request, false); +}); + +test("denied or expired device authorization does not persist token state", async t => { + const f = await fixture(t, { completion: Promise.resolve({ state: "reauthentication-required" }) }); + const login = await f.lifecycle.deviceLogin(accountId); + assert.deepEqual(await login.completion, { state: "reauthentication-required" }); + assert.equal(f.saves, 0); +}); + +test("refresh keeps the working local cache and reports bounded encrypted-save failures", async t => { + const f = await fixture(t, { retryDelaysMs: [0, 0], persistEncrypted: async () => { throw new Error("fixture outage"); } }); + assert.deepEqual(await f.lifecycle.refresh(accountId), { state: "unsaved", attempts: 3 }); + assert.equal(f.lifecycle.isUnsaved(accountId), true); + assert.equal(f.calls.length, 1); + assert.equal(f.saves, 3); + assert.deepEqual(f.events.filter(event => event.type === "unsaved").map(event => event.unsaved), [true]); +}); + +test("refresh requests for one account serialize and use the latest revision", async t => { + let revision = 0, inFlight = 0, maximumInFlight = 0; + const f = await fixture(t, { + getAccount: () => ({ ...account, revision }), + persistEncrypted: async () => { + inFlight++; + maximumInFlight = Math.max(maximumInFlight, inFlight); + await new Promise(resolve => setImmediate(resolve)); + revision++; + inFlight--; + }, + }); + await Promise.all([f.lifecycle.refresh(accountId), f.lifecycle.refresh(accountId)]); + assert.equal(maximumInFlight, 1); + assert.equal(f.saves, 2); +}); + +test("deletion invalidates a retry and removes the account cache", async t => { + let release; + const waiting = new Promise(resolve => { release = resolve; }); + const f = await fixture(t, { + retryDelaysMs: [0, 0, 0], + persistEncrypted: async (_request, attempt) => { + if (attempt === 1) { await waiting; throw new Error("first save failed"); } + }, + }); + const refreshing = f.lifecycle.refresh(accountId); + while (!f.saves) await new Promise(resolve => setImmediate(resolve)); + const deleting = f.lifecycle.deleteAccount(accountId); + release(); + await assert.rejects(refreshing, /changed before token persistence/); + assert.deepEqual(await deleting, { state: "deleted" }); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); +}); From 32cc5b750048e6e0422ff5d4b8b136ad4bbcb643 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:58:22 +0000 Subject: [PATCH 081/102] GEN-2824 encrypt Codex account caches in proxy --- .../src/model-account-crypto.mjs | 61 +++++++++++++++++++ .../test/model-account-crypto.test.mjs | 37 +++++++++++ 2 files changed, 98 insertions(+) create mode 100644 packages/credential-proxy/src/model-account-crypto.mjs create mode 100644 packages/credential-proxy/test/model-account-crypto.test.mjs diff --git a/packages/credential-proxy/src/model-account-crypto.mjs b/packages/credential-proxy/src/model-account-crypto.mjs new file mode 100644 index 00000000..4602c480 --- /dev/null +++ b/packages/credential-proxy/src/model-account-crypto.mjs @@ -0,0 +1,61 @@ +import { webcrypto } from "node:crypto"; + +const KEY_PREFIX = "fzk1."; +const KEY_BYTES = 32; +const NONCE_BYTES = 12; +const SALT = new TextEncoder().encode("factorize.unlock-key.v1"); +const INFO = new TextEncoder().encode("factorize.record.aes-256-gcm.v1"); +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function bindingData(binding) { + if (!binding || typeof binding.ownerId !== "string" || !binding.ownerId + || !UUID.test(binding.accountId ?? "") || !UUID.test(binding.keyGeneration ?? "") + || !Number.isSafeInteger(binding.revision) || binding.revision < 1) { + throw new TypeError("invalid model account encryption binding"); + } + return new TextEncoder().encode(JSON.stringify([ + "factorize-record-v1", binding.ownerId, binding.accountId, "model", "factorize", binding.keyGeneration, binding.revision, + ])); +} + +async function encryptionKey(unlockKey, usage) { + if (typeof unlockKey !== "string" || !unlockKey.startsWith(KEY_PREFIX)) throw new TypeError("malformed unlock key"); + const encoded = unlockKey.slice(KEY_PREFIX.length); + if (!/^[A-Za-z0-9_-]{43}$/.test(encoded)) throw new TypeError("malformed unlock key"); + const raw = Buffer.from(encoded, "base64url"); + if (raw.byteLength !== KEY_BYTES || raw.toString("base64url") !== encoded) throw new TypeError("malformed unlock key"); + try { + const material = await webcrypto.subtle.importKey("raw", raw, "HKDF", false, ["deriveKey"]); + return await webcrypto.subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt: SALT, info: INFO }, material, + { name: "AES-GCM", length: 256 }, false, [usage]); + } finally { raw.fill(0); } +} + +function decodeStored(value, label) { + if (typeof value !== "string" || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) throw new TypeError(`malformed ${label}`); + const bytes = Buffer.from(value, "base64"); + if (bytes.toString("base64") !== value) throw new TypeError(`malformed ${label}`); + return new Uint8Array(bytes); +} + +/** Encrypt a Codex auth cache for persistence. The cache and unlock key remain in the proxy process. */ +export async function encryptModelAccountCache(plaintext, unlockKey, binding) { + if (!(plaintext instanceof Uint8Array) || plaintext.byteLength === 0 || plaintext.byteLength > 65_536) throw new TypeError("invalid model account cache"); + const aad = bindingData(binding), nonce = webcrypto.getRandomValues(new Uint8Array(NONCE_BYTES)); + const key = await encryptionKey(unlockKey, "encrypt"); + const ciphertext = new Uint8Array(await webcrypto.subtle.encrypt({ name: "AES-GCM", iv: nonce, additionalData: aad, tagLength: 128 }, key, plaintext)); + return `${Buffer.from(nonce).toString("base64")}.${Buffer.from(ciphertext).toString("base64")}`; +} + +/** Decrypt only after the selected account, owner, key generation, and revision are verified. */ +export async function decryptModelAccountCache(encryptedPayload, unlockKey, binding) { + if (typeof encryptedPayload !== "string" || encryptedPayload.length > 65_536) throw new TypeError("malformed model account ciphertext"); + const parts = encryptedPayload.split("."); + if (parts.length !== 2) throw new TypeError("malformed model account ciphertext"); + const nonce = decodeStored(parts[0], "model account nonce"), ciphertext = decodeStored(parts[1], "model account ciphertext"); + if (nonce.byteLength !== NONCE_BYTES || ciphertext.byteLength < 16) throw new TypeError("malformed model account ciphertext"); + const aad = bindingData(binding), key = await encryptionKey(unlockKey, "decrypt"); + try { + return new Uint8Array(await webcrypto.subtle.decrypt({ name: "AES-GCM", iv: nonce, additionalData: aad, tagLength: 128 }, key, ciphertext)); + } finally { nonce.fill(0); ciphertext.fill(0); } +} diff --git a/packages/credential-proxy/test/model-account-crypto.test.mjs b/packages/credential-proxy/test/model-account-crypto.test.mjs new file mode 100644 index 00000000..2308a6df --- /dev/null +++ b/packages/credential-proxy/test/model-account-crypto.test.mjs @@ -0,0 +1,37 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; +import { decryptModelAccountCache, encryptModelAccountCache } from "../src/model-account-crypto.mjs"; + +const ownerId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const accountId = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; +const otherAccountId = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; +const keyGeneration = "dddddddd-dddd-4ddd-8ddd-dddddddddddd"; +const unlockKey = `fzk1.${randomBytes(32).toString("base64url")}`; +const binding = { ownerId, accountId, keyGeneration, revision: 7 }; + +test("model account cache round-trips only for its complete owner/account/generation/revision binding", async () => { + const plaintext = new TextEncoder().encode('{"access_token":"synthetic-model-token"}'); + const encrypted = await encryptModelAccountCache(plaintext, unlockKey, binding); + assert.equal(encrypted.includes("synthetic-model-token"), false); + const restored = await decryptModelAccountCache(encrypted, unlockKey, binding); + assert.deepEqual(restored, plaintext); + restored.fill(0); + plaintext.fill(0); + + for (const changed of [ + { ...binding, ownerId: "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee" }, + { ...binding, accountId: otherAccountId }, + { ...binding, keyGeneration: "ffffffff-ffff-4fff-8fff-ffffffffffff" }, + { ...binding, revision: 8 }, + ]) await assert.rejects(decryptModelAccountCache(encrypted, unlockKey, changed)); + await assert.rejects(decryptModelAccountCache(encrypted, `fzk1.${randomBytes(32).toString("base64url")}`, binding)); +}); + +test("model account cache rejects malformed keys, ciphertext, and metadata", async () => { + const plaintext = new TextEncoder().encode("synthetic-cache"); + await assert.rejects(encryptModelAccountCache(plaintext, "not-an-unlock-key", binding), /malformed unlock key/); + await assert.rejects(encryptModelAccountCache(plaintext, unlockKey, { ...binding, revision: 0 }), /invalid model account encryption binding/); + await assert.rejects(decryptModelAccountCache("bad.payload.extra", unlockKey, binding), /malformed model account ciphertext/); + await assert.rejects(decryptModelAccountCache("AA==.AA==", unlockKey, binding), /malformed model account ciphertext/); +}); From d5c433bf34f2b0d3e9d1a98fc7a94cb80c203ab7 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 01:18:17 +0000 Subject: [PATCH 082/102] GEN-2824 restore encrypted Codex caches on unlock --- .../src/model-account-lifecycle.mjs | 63 ++++++++++++++++++- .../test/model-account-lifecycle.test.mjs | 39 ++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index 236d4008..0ee1bcc3 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -1,4 +1,5 @@ -import { readFile, rm } from "node:fs/promises"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; import { join, resolve, sep } from "node:path"; const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; @@ -27,6 +28,11 @@ export function createModelAccountLifecycle({ sleep = ms => new Promise(resolveSleep => setTimeout(resolveSleep, ms)), removeCache = rm, readCache = readFile, + writeCache = writeFile, + makeDirectory = mkdir, + replaceFile = rename, + getEncryptedCache, + decryptCache, } = {}) { if (!authHelper || typeof authHelper.invoke !== "function" || typeof getUnlockKey !== "function" || typeof getAccount !== "function" || typeof encryptCache !== "function" || typeof persistEncrypted !== "function" @@ -34,6 +40,11 @@ export function createModelAccountLifecycle({ || retryDelaysMs.some(ms => !Number.isSafeInteger(ms) || ms < 0 || ms > 30_000)) { throw new TypeError("model account lifecycle configuration is invalid"); } + if ((getEncryptedCache === undefined) !== (decryptCache === undefined) + || (getEncryptedCache !== undefined && typeof getEncryptedCache !== "function") + || (decryptCache !== undefined && typeof decryptCache !== "function")) { + throw new TypeError("model account cache restoration configuration is invalid"); + } const generations = new Map(); const queues = new Map(); @@ -132,11 +143,57 @@ export function createModelAccountLifecycle({ }); } + async function restore(id) { + if (!getEncryptedCache || !decryptCache) throw new Error("model account cache restoration is unavailable"); + const account = await getAccount(id); + if (!account || account.id !== id || account.kind !== "model" || !Number.isSafeInteger(account.revision) + || typeof account.ownerId !== "string" || !account.ownerId || typeof account.keyGeneration !== "string" || !account.keyGeneration) { + await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); + return { state: "reauthentication-required" }; + } + const encrypted = await getEncryptedCache(id); + if (!encrypted || encrypted.accountId !== id || encrypted.ownerId !== account.ownerId + || encrypted.keyGeneration !== account.keyGeneration || !Number.isSafeInteger(encrypted.revision) + || typeof encrypted.encryptedPayload !== "string") { + await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); + return { state: "reauthentication-required" }; + } + const unlockKey = await getUnlockKey(); + if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); + let plain; + try { + plain = await decryptCache(encrypted.encryptedPayload, unlockKey, { + ownerId: account.ownerId, + recordId: id, + recordType: "model", + destination: "factorize", + revision: encrypted.revision, + keyGeneration: account.keyGeneration, + }); + if (!(plain instanceof Uint8Array) || plain.byteLength === 0 || plain.byteLength > 65_536) { + throw new Error("restored Codex token cache is invalid"); + } + const destination = cachePath(accountHomeRoot, id); + await makeDirectory(resolve(accountHomeRoot, id), { recursive: true, mode: 0o700 }); + const temporary = `${destination}.${randomUUID()}.tmp`; + try { + await writeCache(temporary, plain, { mode: 0o600, flag: "wx" }); + await replaceFile(temporary, destination); + } finally { + await rm(temporary, { force: true }); + } + } finally { + plain?.fill(0); + } + return { state: "restored", revision: encrypted.revision }; + } + return Object.freeze({ deviceLogin, cancel, refresh, deleteAccount, + restore, isUnsaved: id => unsaved.has(id), /** Invoke after encrypted snapshots are reconciled at unlock. */ reconcile: async id => serialize(id, async () => { @@ -145,6 +202,10 @@ export function createModelAccountLifecycle({ unsaved.delete(id); return { state: "deleted" }; } + if (getEncryptedCache) { + const restored = await restore(id); + if (restored.state !== "restored") return restored; + } const result = await authHelper.invoke({ method: "account-status", accountId: id }); return result.state === "authenticated" ? persist(id, currentGeneration(id)) : { state: "reauthentication-required" }; }), diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 449fbdb4..37ab14e4 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -111,3 +111,42 @@ test("deletion invalidates a retry and removes the account cache", async t => { await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); assert.ok(f.methods.includes("logout")); }); + +test("unlock reconciliation restores the encrypted Codex cache before checking account status", async t => { + const persisted = new TextEncoder().encode(JSON.stringify({ tokens: { access_token: "restored-access", refresh_token: "restored-refresh" } })); + const encryptedPayload = `cache.${Buffer.from(persisted).toString("base64")}`; + const f = await fixture(t, { options: { + getEncryptedCache: async id => ({ + accountId: id, ownerId: "owner", keyGeneration: "generation", revision: 8, encryptedPayload, + }), + decryptCache: async (payload, key, binding) => { + assert.equal(payload, encryptedPayload); + assert.equal(key, "synthetic-unlock-key"); + assert.deepEqual(binding, { + ownerId: "owner", recordId: accountId, recordType: "model", destination: "factorize", + revision: 8, keyGeneration: "generation", + }); + return new Uint8Array(persisted); + }, + } }); + + assert.deepEqual(await f.lifecycle.reconcile(accountId), { state: "saved" }); + assert.deepEqual(JSON.parse(await readFile(join(f.home, "auth.json"), "utf8")), { + tokens: { access_token: "restored-access", refresh_token: "restored-refresh" }, + }); + assert.ok(f.methods.indexOf("account-status") > -1); + assert.equal(f.events.find(event => event.type === "persist").request.expectedRevision, 4); +}); + +test("unlock reconciliation removes stale cache when the account key generation changed", async t => { + const f = await fixture(t, { options: { + getEncryptedCache: async id => ({ + accountId: id, ownerId: "owner", keyGeneration: "old-generation", revision: 8, encryptedPayload: "cache.payload", + }), + decryptCache: async () => { throw new Error("must not decrypt stale cache"); }, + } }); + + assert.deepEqual(await f.lifecycle.reconcile(accountId), { state: "reauthentication-required" }); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.equal(f.methods.includes("account-status"), false); +}); From 4653583bc4d4887e380874e3c35353b2cb88671a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 01:35:23 +0000 Subject: [PATCH 083/102] GEN-2824 persist encrypted model caches by revision --- .../encrypted-integration-repository.ts | 37 ++++++++++++++++++ packages/api/src/proxy-control.ts | 38 +++++++++++++++++-- .../credential-proxy/src/control-client.mjs | 35 +++++++++++++++++ .../src/model-account-lifecycle.mjs | 2 +- 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/packages/api/src/postgres/encrypted-integration-repository.ts b/packages/api/src/postgres/encrypted-integration-repository.ts index 9e02d3fa..3451086a 100644 --- a/packages/api/src/postgres/encrypted-integration-repository.ts +++ b/packages/api/src/postgres/encrypted-integration-repository.ts @@ -126,6 +126,43 @@ export class EncryptedIntegrationRepository { return Promise.all(rows.map(async row => ({ id: row.id, kind: row.kind, name: row.name, origin: row.origin, description: row.description, revision: safeRevision(row.revision), encryptedPayload: await decrypt(row.encrypted_payload, this.serverEncryptionKey) }))); } + /** + * Replace only a model account's user-encrypted Codex cache. The account + * revision is independent of the collection revision, so unrelated bearer + * edits do not invalidate the cache's authenticated-encryption binding. + */ + async saveModelCache(input: { id: string; expectedRevision: number; encryptedPayload: string; idempotencyKey: string }): Promise<{ revision: number; collectionRevision: number; duplicate: boolean }> { + const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (!uuid.test(input.id) || !uuid.test(input.idempotencyKey) || !Number.isSafeInteger(input.expectedRevision) || input.expectedRevision < 0 + || !isUserCiphertext(input.encryptedPayload)) throw new IntegrationValidationError("model cache binding is invalid"); + const encryptedPayload = await encrypt(input.encryptedPayload, this.serverEncryptionKey); + return this.database.transaction(async client => { + await client.query("INSERT INTO app.integration_collections(tenant_id) VALUES ($1) ON CONFLICT DO NOTHING", [this.tenantId]); + const collection = (await client.query<{ revision: string | number }>("SELECT revision FROM app.integration_collections WHERE tenant_id=$1 FOR UPDATE", [this.tenantId])).rows[0]!; + const currentCollectionRevision = safeRevision(collection.revision); + const current = (await client.query<{ kind: EncryptedIntegrationKind; name: string; origin: string | null; description: string | null; revision: string | number; encrypted_payload: string | null; deleted_at: Date | null }>( + "SELECT kind,name,origin,description,revision,encrypted_payload,deleted_at FROM app.integration_records WHERE tenant_id=$1 AND id=$2 FOR UPDATE", [this.tenantId, input.id])).rows[0]; + if (!current || current.kind !== "model" || current.deleted_at || current.revision === null) throw new IntegrationValidationError("model account is unavailable"); + const currentRevision = safeRevision(current.revision); + const previousChange = (await client.query<{ revision: string | number; record_id: string; operation: string; encrypted_payload: string | null }>( + "SELECT revision,record_id,operation,encrypted_payload FROM app.integration_changes WHERE tenant_id=$1 AND idempotency_key=$2", [this.tenantId,input.idempotencyKey])).rows[0]; + if (previousChange) { + const same = previousChange.record_id === input.id && previousChange.operation === "upsert" && previousChange.encrypted_payload !== null + && await decrypt(previousChange.encrypted_payload, this.serverEncryptionKey) === input.encryptedPayload; + if (!same) throw new IntegrationIdempotencyConflictError(); + return { revision: Math.max(0, input.expectedRevision + 1), collectionRevision: safeRevision(previousChange.revision), duplicate: true }; + } + if (currentRevision !== input.expectedRevision) throw new StaleIntegrationRevisionError(currentRevision); + const collectionRevision = currentCollectionRevision + 1; + const revision = currentRevision + 1; + await client.query("UPDATE app.integration_collections SET revision=$2,updated_at=now() WHERE tenant_id=$1", [this.tenantId, collectionRevision]); + await client.query("UPDATE app.integration_records SET revision=$3,encrypted_payload=$4,updated_at=now() WHERE tenant_id=$1 AND id=$2", [this.tenantId,input.id,revision,encryptedPayload]); + await client.query(`INSERT INTO app.integration_changes(tenant_id,revision,id,idempotency_key,operation,record_id,kind,name,origin,description,encrypted_payload) + VALUES ($1,$2,$3,$4,'upsert',$5,'model',$6,NULL,$7,$8)`, [this.tenantId,collectionRevision,crypto.randomUUID(),input.idempotencyKey,input.id,current.name,current.description,encryptedPayload]); + return { revision, collectionRevision, duplicate: false }; + }); + } + async changesSince(revision: number, limit = 100): Promise { if (!Number.isSafeInteger(revision) || revision < 0) throw new IntegrationValidationError("revision must be a non-negative safe integer"); if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500) throw new IntegrationValidationError("limit must be between 1 and 500"); diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index 18ef39f3..a1653350 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -9,14 +9,15 @@ const json = (body: unknown, status = 200) => Response.json(body, { status, head /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot", "/internal/credential-proxy/revocations"].includes(url.pathname)) return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot", "/internal/credential-proxy/model-account-cache", "/internal/credential-proxy/revocations"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; const token = authorization.startsWith("Bearer ") ? authorization.slice(7) : ""; if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return json({ error: "unauthorized" }, 401); const declaredLength = Number(request.headers.get("content-length") ?? "0"); - if (declaredLength > 1024) return json({ error: "invalid_request" }, 413); + const maxBodyBytes = url.pathname === "/internal/credential-proxy/model-account-cache" ? 70_000 : 1024; + if (declaredLength > maxBodyBytes) return json({ error: "invalid_request" }, 413); let body: unknown; try { const reader = request.body?.getReader(); @@ -24,7 +25,7 @@ export async function proxyControlFetch(request: Request, env: Parameters 1024) { await reader.cancel(); return json({ error: "invalid_request" }, 413); } + if (length > maxBodyBytes) { await reader.cancel(); return json({ error: "invalid_request" }, 413); } chunks.push(value); } const bytes = new Uint8Array(length); let offset = 0; @@ -49,6 +50,37 @@ export async function proxyControlFetch(request: Request, env: Parameters row.run_id) }); } + if (url.pathname === "/internal/credential-proxy/model-account-cache") { + if (!body || typeof body !== "object" || Array.isArray(body)) return json({ error: "invalid_request" }, 400); + const input = body as Record; + const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (input.operation === "read" && Object.keys(input).sort().join(",") === "accountId,operation" + && typeof input.accountId === "string" && uuid.test(input.accountId)) { + const account = (await new EncryptedIntegrationRepository(databaseFor(env), proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("model")) + .find(record => record.id === input.accountId); + if (!account) return json({ error: "model_account_unavailable" }, 404); + const keyState = await databaseFor(env).pool.query<{ generation_id: string }>("SELECT generation_id FROM app.unlock_key_verifications WHERE tenant_id=$1", [proxy.tenantId]); + if (!keyState.rows[0]) return json({ error: "model_account_unavailable" }, 404); + return json({ ownerId: proxy.tenantId, account: { ...account, keyGeneration: keyState.rows[0].generation_id } }); + } + if (input.operation !== "save" || Object.keys(input).sort().join(",") !== "accountId,encryptedPayload,expectedRevision,idempotencyKey,operation" + || typeof input.accountId !== "string" || !uuid.test(input.accountId) + || typeof input.idempotencyKey !== "string" || !uuid.test(input.idempotencyKey) + || !Number.isSafeInteger(input.expectedRevision) || (input.expectedRevision as number) < 0 + || typeof input.encryptedPayload !== "string") return json({ error: "invalid_request" }, 400); + try { + const saved = await new EncryptedIntegrationRepository(databaseFor(env), proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).saveModelCache({ + id: input.accountId, expectedRevision: input.expectedRevision as number, + idempotencyKey: input.idempotencyKey, encryptedPayload: input.encryptedPayload, + }); + return json({ ...saved, accountId: input.accountId }); + } catch (error) { + if (error instanceof Error && error.name === "StaleIntegrationRevisionError") return json({ error: "stale_revision" }, 409); + if (error instanceof Error && error.name === "IntegrationIdempotencyConflictError") return json({ error: "idempotency_conflict" }, 409); + if (error instanceof Error && error.name === "IntegrationValidationError") return json({ error: "model_account_unavailable" }, 404); + throw error; + } + } if (url.pathname === "/internal/credential-proxy/integration-snapshot") { if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).some(key => !["runId", "grantRevision"].includes(key)) || typeof (body as any).runId !== "string" || !Number.isSafeInteger((body as any).grantRevision) || (body as any).grantRevision < 0) return json({ error: "invalid_request" }, 400); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index f36c3fa4..e8d89798 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -76,6 +76,41 @@ export async function readProxyIntegrationSnapshot({ controlOrigin, token, bindi return result; } +/** Concrete proxy-side adapters for revision-bound encrypted model caches. */ +export function createModelAccountPersistenceAdapter({ controlOrigin, controlCredential, ownerId, fetchImpl = fetch } = {}) { + if (typeof ownerId !== "string" || !ownerId || !/^fzc_[A-Za-z0-9_-]{43}$/.test(controlCredential ?? "")) { + throw new TypeError("model account persistence requires the registered proxy identity"); + } + const read = async accountId => { + if (!uuidPattern.test(accountId ?? "")) throw new TypeError("invalid model account id"); + const result = await proxyControlRequest({ controlOrigin, token: controlCredential, path: "/internal/credential-proxy/model-account-cache", body: { operation: "read", accountId }, fetchImpl }); + const account = result?.account; + if (result?.ownerId !== ownerId || !account || account.id !== accountId || account.kind !== "model" + || account.origin !== null || typeof account.encryptedPayload !== "string" + || !Number.isSafeInteger(account.revision) || !uuidPattern.test(account.keyGeneration ?? "")) { + throw new Error("invalid model account persistence response"); + } + return { ...account, ownerId }; + }; + return Object.freeze({ + getAccount: read, + async getEncryptedCache(accountId) { + const account = await read(accountId); + return { accountId, ownerId, keyGeneration: account.keyGeneration, revision: account.revision, encryptedPayload: account.encryptedPayload }; + }, + async persistEncrypted({ id, expectedRevision, encryptedPayload, idempotencyKey }) { + if (!uuidPattern.test(id ?? "") || !uuidPattern.test(idempotencyKey ?? "") || !Number.isSafeInteger(expectedRevision) + || typeof encryptedPayload !== "string" || encryptedPayload.length > 65_536) throw new TypeError("invalid encrypted model account update"); + const result = await proxyControlRequest({ controlOrigin, token: controlCredential, path: "/internal/credential-proxy/model-account-cache", + body: { operation: "save", accountId: id, expectedRevision, encryptedPayload, idempotencyKey }, fetchImpl }); + if (result?.accountId !== id || result.revision !== expectedRevision + 1 || typeof result.duplicate !== "boolean") { + throw new Error("invalid model account persistence response"); + } + return result; + }, + }); +} + /** Read durable revocations for the registered proxy boot. Replaying the full * set is intentional: it closes runs revoked while this proxy was offline and * runtime.revokeRun is idempotent. */ diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index 0ee1bcc3..8e13d05f 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -82,7 +82,7 @@ export function createModelAccountLifecycle({ plain.fill(0); } if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); - const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted }; + const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted, idempotencyKey: randomUUID() }; for (let attempt = 0; ; attempt++) { if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); try { From 182b8bf6741185f2ee37225e2cf4f3a3b2de9f37 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:00:55 +0000 Subject: [PATCH 084/102] GEN-2824 compose model cache lifecycle into proxy startup --- packages/api/src/proxy-control.ts | 9 ++++++- packages/api/test/proxy-control.test.ts | 2 +- .../credential-proxy/src/control-client.mjs | 15 +++++++++-- packages/credential-proxy/src/main.mjs | 25 +++++++++++++++++ .../test/control-client.test.mjs | 27 ++++++++++++++++--- 5 files changed, 71 insertions(+), 7 deletions(-) diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index a1653350..e947d35b 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -37,7 +37,7 @@ export async function proxyControlFetch(request: Request, env: Parameters key !== "bootEpoch") || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); const registration = await repository.registerBoot(token, (body as any).bootEpoch); if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); - return json({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch, controlCredential: registration.controlCredential }); + return json({ registered: true, tenantId: registration.tenantId, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch, controlCredential: registration.controlCredential }); } const proxy = await repository.authenticateProxy(token); @@ -54,6 +54,13 @@ export async function proxyControlFetch(request: Request, env: Parameters; const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (input.operation === "list" && Object.keys(input).length === 1) { + const accounts = await new EncryptedIntegrationRepository(databaseFor(env), proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("model"); + const keyState = await databaseFor(env).pool.query<{ generation_id: string }>("SELECT generation_id FROM app.unlock_key_verifications WHERE tenant_id=$1", [proxy.tenantId]); + if (!keyState.rows[0]) return json({ error: "model_account_unavailable" }, 404); + return json({ ownerId: proxy.tenantId, keyGeneration: keyState.rows[0].generation_id, + accounts: accounts.map(({ id, revision }) => ({ id, revision, kind: "model", origin: null })) }); + } if (input.operation === "read" && Object.keys(input).sort().join(",") === "accountId,operation" && typeof input.accountId === "string" && uuid.test(input.accountId)) { const account = (await new EncryptedIntegrationRepository(databaseFor(env), proxy.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).records("model")) diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index 42a02a58..ae33a978 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -30,7 +30,7 @@ describe("dedicated proxy boot registration endpoint", () => { const env = envFor(); const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: expect.stringMatching(/^fzc_[A-Za-z0-9_-]{43}$/) }); + expect(await response.json()).toMatchObject({ registered: true, tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: expect.stringMatching(/^fzc_[A-Za-z0-9_-]{43}$/) }); expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(4); expect(env.transactionQuery.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(env.transactionQuery.mock.calls[2][0]).toContain("consumed_at=now()"); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index e8d89798..6ea36bab 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -22,8 +22,9 @@ export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetch }); if (!response.ok) throw new Error("proxy boot registration rejected"); const result = await response.json(); - if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(result.controlCredential ?? "")) throw new Error("invalid proxy boot registration response"); - return { proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch, controlCredential: result.controlCredential }; + if (result?.registered !== true || result.bootEpoch !== bootEpoch || typeof result.proxyIdentity !== "string" + || !uuidPattern.test(result.tenantId ?? "") || !/^fzc_[A-Za-z0-9_-]{43}$/.test(result.controlCredential ?? "")) throw new Error("invalid proxy boot registration response"); + return { tenantId: result.tenantId, proxyIdentity: result.proxyIdentity, bootEpoch: result.bootEpoch, controlCredential: result.controlCredential }; } finally { credential = ""; } @@ -93,6 +94,16 @@ export function createModelAccountPersistenceAdapter({ controlOrigin, controlCre return { ...account, ownerId }; }; return Object.freeze({ + async listAccounts() { + const result = await proxyControlRequest({ controlOrigin, token: controlCredential, path: "/internal/credential-proxy/model-account-cache", body: { operation: "list" }, fetchImpl }); + if (result?.ownerId !== ownerId || !Array.isArray(result.accounts) + || !uuidPattern.test(result.keyGeneration ?? "") + || result.accounts.some(account => !account || !uuidPattern.test(account.id ?? "") || account.kind !== "model" + || account.origin !== null || !Number.isSafeInteger(account.revision))) { + throw new Error("invalid model account list response"); + } + return result.accounts.map(account => ({ ...account, ownerId, keyGeneration: result.keyGeneration })); + }, getAccount: read, async getEncryptedCache(accountId) { const account = await read(accountId); diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 60ab7333..4b2765bb 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -6,6 +6,10 @@ import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; import { createProxyIntegrationGrantResolver } from "./decrypted-grants.mjs"; import { startAuthenticatedProxyTunnel } from "./tunnel-transport.mjs"; import { createProxyTlsAuthority } from "./tls-identity.mjs"; +import { createCodexAuthHelper } from "./codex-auth-helper.mjs"; +import { createModelAccountLifecycle } from "./model-account-lifecycle.mjs"; +import { createModelAccountPersistenceAdapter } from "./control-client.mjs"; +import { decryptModelAccountCache, encryptModelAccountCache } from "./model-account-crypto.mjs"; const epochIndex = process.argv.indexOf("--boot-epoch"); const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; @@ -20,9 +24,26 @@ let controlCredential; let controlOrigin; let tunnel; let revocationMonitor; +let modelAccounts; +let modelAccountLifecycle; async function activateRegistration(registration, origin) { if (tunnel) throw new Error("proxy boot is already registered"); + const persistence = createModelAccountPersistenceAdapter({ + controlOrigin: origin, + controlCredential: registration.controlCredential, + ownerId: registration.tenantId, + }); + modelAccounts = persistence; + modelAccountLifecycle = createModelAccountLifecycle({ + authHelper: createCodexAuthHelper(), + getUnlockKey: () => runtime.getUnlockKey(), + getAccount: persistence.getAccount, + getEncryptedCache: persistence.getEncryptedCache, + persistEncrypted: persistence.persistEncrypted, + encryptCache: encryptModelAccountCache, + decryptCache: decryptModelAccountCache, + }); const resolveGrants = createProxyIntegrationGrantResolver({ runtime, controlOrigin: origin, controlCredential: registration.controlCredential }); const started = await startAuthenticatedProxyTunnel({ tls: tlsAuthority.tls, @@ -96,6 +117,10 @@ lines.on("line", async (line) => { if (command?.type === "status") respondStatus(); else if (command?.type === "unlock") { runtime.unlock(command.key); + if (modelAccountLifecycle && modelAccounts) { + const accounts = await modelAccounts.listAccounts(); + for (const account of accounts) await modelAccountLifecycle.reconcile(account.id); + } respondStatus(); } else if (command?.type === "lock") { await runtime.lock(); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index b3ab4cbd..e25342cf 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, readProxyRunRevocations, registerProxyBoot, startProxyRunRevocationMonitor } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createModelAccountPersistenceAdapter, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, readProxyRunRevocations, registerProxyBoot, startProxyRunRevocationMonitor } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; @@ -11,9 +11,9 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an let request; const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async (url, init) => { request = { url: String(url), init }; - return Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); + return Response.json({ registered: true, tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); } }); - assert.deepEqual(result, { proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); + assert.deepEqual(result, { tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); assert.equal(request.url, "https://factorize.example/internal/credential-proxy/register-boot"); assert.equal(request.init.headers.Authorization, `Bearer ${token}`); assert.equal(request.init.redirect, "error"); @@ -35,6 +35,27 @@ test("proxy snapshot reader requests only the bound run revision and validates e await assert.rejects(readProxyIntegrationSnapshot({ controlOrigin: "https://factorize.example", token, binding: { runId, grantRevision: 7 }, fetchImpl: async () => Response.json({ ...snapshot, modelAccount: { ...snapshot.modelAccount, kind: "bearer" } }) }), /invalid proxy integration snapshot response/); }); +test("model account persistence adapter lists owner-bound account revisions over proxy authentication", async () => { + const ownerId = "11111111-1111-4111-8111-111111111111"; + let request; + const adapter = createModelAccountPersistenceAdapter({ + controlOrigin: "https://factorize.example", controlCredential: token, ownerId, + fetchImpl: async (url, init) => { + request = { url: String(url), init, body: JSON.parse(init.body) }; + return Response.json({ ownerId, keyGeneration: "22222222-2222-4222-8222-222222222222", + accounts: [{ id: "33333333-3333-4333-8333-333333333333", kind: "model", origin: null, revision: 4 }] }); + }, + }); + assert.deepEqual(await adapter.listAccounts(), [{ id: "33333333-3333-4333-8333-333333333333", kind: "model", origin: null, revision: 4, ownerId, keyGeneration: "22222222-2222-4222-8222-222222222222" }]); + assert.equal(request.url, "https://factorize.example/internal/credential-proxy/model-account-cache"); + assert.deepEqual(request.body, { operation: "list" }); + assert.equal(request.init.headers.Authorization, `Bearer ${token}`); + await assert.rejects(adapter.listAccounts && createModelAccountPersistenceAdapter({ + controlOrigin: "https://factorize.example", controlCredential: token, ownerId, + fetchImpl: async () => Response.json({ ownerId: "22222222-2222-4222-8222-222222222222", keyGeneration: "22222222-2222-4222-8222-222222222222", accounts: [] }), + }).listAccounts(), /invalid model account list response/); +}); + test("proxy revocation monitor applies durable run revocations and tolerates an outage", async () => { const requests = []; const revoked = []; From 555656a9d461b39e2b714e82df61039576d216f1 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:18:46 +0000 Subject: [PATCH 085/102] GEN-2824 add private browser model authorization control --- .../credential-proxy/src/browser-control.mjs | 150 ++++++++++++++++++ .../test/browser-control.test.mjs | 76 +++++++++ 2 files changed, 226 insertions(+) create mode 100644 packages/credential-proxy/src/browser-control.mjs create mode 100644 packages/credential-proxy/test/browser-control.test.mjs diff --git a/packages/credential-proxy/src/browser-control.mjs b/packages/credential-proxy/src/browser-control.mjs new file mode 100644 index 00000000..5911ba14 --- /dev/null +++ b/packages/credential-proxy/src/browser-control.mjs @@ -0,0 +1,150 @@ +import { createServer } from "node:http"; +import { createPublicKey, verify } from "node:crypto"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const MAX_BODY_BYTES = 4096; + +function decodePart(value) { + if (!/^[A-Za-z0-9_-]+$/.test(value)) throw new Error("invalid token"); + return Buffer.from(value, "base64url"); +} + +function parseCapability(token, publicKey) { + if (typeof token !== "string" || token.length > 4096) throw new Error("invalid capability"); + const parts = token.split("."); + if (parts.length !== 3) throw new Error("invalid capability"); + const [headerPart, claimsPart, signaturePart] = parts; + const header = JSON.parse(decodePart(headerPart).toString("utf8")); + const claims = JSON.parse(decodePart(claimsPart).toString("utf8")); + if (header?.alg !== "EdDSA" || header?.typ !== "JWT" + || !verify(null, Buffer.from(`${headerPart}.${claimsPart}`), publicKey, decodePart(signaturePart))) { + throw new Error("invalid capability"); + } + return claims; +} + +function json(response, status, body, origin) { + const headers = { "Cache-Control": "no-store", "Content-Type": "application/json; charset=utf-8" }; + if (origin) { + headers["Access-Control-Allow-Origin"] = origin; + headers.Vary = "Origin"; + } + response.writeHead(status, headers).end(JSON.stringify(body)); +} + +async function readJson(request) { + const chunks = []; + let size = 0; + for await (const chunk of request) { + size += chunk.byteLength; + if (size > MAX_BODY_BYTES) throw Object.assign(new Error("request too large"), { status: 413 }); + chunks.push(chunk); + } + try { return JSON.parse(Buffer.concat(chunks).toString("utf8")); } + catch { throw Object.assign(new Error("invalid request"), { status: 400 }); } +} + +/** Private Fly service endpoint for owner-browser model-account operations. + * The Fly edge terminates public HTTPS; this listener must only be reachable + * through that service and never through the agent SSH integration channel. */ +export async function startBrowserControl({ + runtime, + lifecycle, + publicKey, + proxyId, + ownerId, + generation, + appOrigin, + host = "0.0.0.0", + port = 8080, + now = () => Math.floor(Date.now() / 1000), +} = {}) { + if (!runtime || !lifecycle || !publicKey || typeof proxyId !== "string" || !proxyId + || !UUID.test(ownerId ?? "") || !Number.isSafeInteger(generation) || generation < 1 + || !Number.isInteger(port) || port < 0 || port > 65535) { + throw new TypeError("browser control configuration is invalid"); + } + const expectedOrigin = new URL(appOrigin); + if (expectedOrigin.protocol !== "https:" || expectedOrigin.origin !== appOrigin) throw new TypeError("browser control origin must be canonical HTTPS"); + const verifyKey = publicKey?.type === "public" ? publicKey : createPublicKey(publicKey); + const consumed = new Map(); + const authorizations = new Map(); + + const server = createServer(async (request, response) => { + const origin = request.headers.origin; + if (origin !== appOrigin) return json(response, 403, { error: "origin_rejected" }); + if (request.method === "OPTIONS") { + if (request.headers["access-control-request-method"] !== "POST" + || request.headers["access-control-request-headers"]?.split(",").map(value => value.trim().toLowerCase()).sort().join(",") !== "authorization,content-type") { + return json(response, 403, { error: "preflight_rejected" }); + } + response.writeHead(204, { + "Access-Control-Allow-Origin": appOrigin, + "Access-Control-Allow-Methods": "POST", + "Access-Control-Allow-Headers": "Authorization, Content-Type", + "Access-Control-Max-Age": "60", + "Cache-Control": "no-store", + Vary: "Origin", + }).end(); + return; + } + if (request.method !== "POST") return json(response, 405, { error: "method_not_allowed" }, appOrigin); + const match = /^\/model-accounts\/([0-9a-f-]{36})\/(authorize|status|cancel)$/.exec(new URL(request.url ?? "/", "http://proxy.internal").pathname); + if (!match || !UUID.test(match[1])) return json(response, 404, { error: "not_found" }, appOrigin); + const [, accountId, operation] = match; + if (request.headers["content-type"]?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") { + return json(response, 415, { error: "unsupported_media_type" }, appOrigin); + } + const authorization = request.headers.authorization ?? ""; + if (!authorization.startsWith("Bearer ")) return json(response, 401, { error: "invalid_capability" }, appOrigin); + let claims; + try { claims = parseCapability(authorization.slice(7), verifyKey); } + catch { return json(response, 401, { error: "invalid_capability" }, appOrigin); } + const expectedAction = `model-account:${operation}`; + if (claims.iss !== "factorize" || claims.aud !== proxyId || claims.sub !== ownerId + || claims.action !== expectedAction || claims.accountId !== accountId || claims.origin !== appOrigin + || claims.generation !== generation || !Number.isSafeInteger(claims.exp) || claims.exp <= now() + || claims.exp > now() + 60 || typeof claims.jti !== "string" || !/^[A-Za-z0-9_-]{16,128}$/.test(claims.jti)) { + return json(response, 401, { error: "invalid_capability" }, appOrigin); + } + for (const [id, expiry] of consumed) if (expiry <= now()) consumed.delete(id); + if (consumed.has(claims.jti)) return json(response, 401, { error: "capability_replayed" }, appOrigin); + consumed.set(claims.jti, claims.exp); + + try { + const body = await readJson(request); + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).length) { + return json(response, 400, { error: "invalid_request" }, appOrigin); + } + if (operation === "authorize") { + if (runtime.state !== "unlocked") return json(response, 423, { error: "proxy_locked" }, appOrigin); + const login = await lifecycle.deviceLogin(accountId); + const state = { state: "pending" }; + authorizations.set(accountId, state); + Promise.resolve(login.completion).then(result => { + if (authorizations.get(accountId) === state) authorizations.set(accountId, { ...result }); + }, () => { + if (authorizations.get(accountId) === state) authorizations.set(accountId, { state: "failed" }); + }); + return json(response, 202, state, appOrigin); + } + if (operation === "status") return json(response, 200, authorizations.get(accountId) ?? { state: "idle" }, appOrigin); + const result = await lifecycle.cancel(accountId); + authorizations.set(accountId, { state: "canceled" }); + return json(response, 200, { state: result?.state === "idle" ? "idle" : "canceled" }, appOrigin); + } catch (error) { + return json(response, Number.isInteger(error?.status) ? error.status : 400, + { error: "model_account_operation_failed" }, appOrigin); + } + }); + server.on("connect", (_request, socket) => socket.end("HTTP/1.1 405 Method Not Allowed\r\nConnection: close\r\n\r\n")); + server.on("upgrade", (_request, socket) => socket.destroy()); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { server.removeListener("error", reject); resolve(); }); + }); + return Object.freeze({ + address: server.address(), + close: () => new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())), + }); +} diff --git a/packages/credential-proxy/test/browser-control.test.mjs b/packages/credential-proxy/test/browser-control.test.mjs new file mode 100644 index 00000000..0c7af413 --- /dev/null +++ b/packages/credential-proxy/test/browser-control.test.mjs @@ -0,0 +1,76 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { generateKeyPairSync, sign } from "node:crypto"; +import { startBrowserControl } from "../src/browser-control.mjs"; + +const ownerId = "11111111-1111-4111-8111-111111111111"; +const accountId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const proxyId = "fly:factorize-proxy:machine-1"; +const generation = 4; +const appOrigin = "https://factorize.example"; +const { publicKey, privateKey } = generateKeyPairSync("ed25519"); + +function capability({ action, exp = 1_900_000_030, aud = proxyId, sub = ownerId, origin = appOrigin, gen = generation, account = accountId, jti = `nonce-${crypto.randomUUID()}` }) { + const head = Buffer.from(JSON.stringify({ alg: "EdDSA", typ: "JWT" })).toString("base64url"); + const claims = Buffer.from(JSON.stringify({ iss: "factorize", aud, sub, action, origin, generation: gen, exp, accountId: account, jti })).toString("base64url"); + const input = `${head}.${claims}`; + return `${input}.${sign(null, Buffer.from(input), privateKey).toString("base64url")}`; +} + +test("private browser control requires exact origin and one-use owner/proxy/action capabilities", async t => { + let now = 1_900_000_000; + const calls = []; + const runtime = { state: "unlocked" }; + const lifecycle = { + async deviceLogin(id) { calls.push(["authorize", id]); return { state: "pending", completion: Promise.resolve({ state: "saved" }) }; }, + async cancel(id) { calls.push(["cancel", id]); return { state: "canceled" }; }, + }; + const server = await startBrowserControl({ runtime, lifecycle, publicKey, proxyId, ownerId, generation, appOrigin, port: 0, host: "127.0.0.1", now: () => now }); + t.after(() => server.close()); + const origin = `http://127.0.0.1:${server.address.port}`; + const post = (action, token, path = `/model-accounts/${accountId}/${action.split(":")[1]}`, requestOrigin = appOrigin) => fetch(origin + path, { + method: "POST", headers: { Origin: requestOrigin, "Content-Type": "application/json", Authorization: `Bearer ${token}` }, body: "{}", + }); + + const preflight = await fetch(origin + `/model-accounts/${accountId}/authorize`, { + method: "OPTIONS", headers: { Origin: appOrigin, "Access-Control-Request-Method": "POST", "Access-Control-Request-Headers": "content-type,authorization" }, + }); + assert.equal(preflight.status, 204); + assert.equal(preflight.headers.get("access-control-allow-origin"), appOrigin); + const foreign = await fetch(origin + `/model-accounts/${accountId}/authorize`, { method: "OPTIONS", headers: { Origin: "https://attacker.example", "Access-Control-Request-Method": "POST", "Access-Control-Request-Headers": "authorization,content-type" } }); + assert.equal(foreign.status, 403); + assert.equal(foreign.headers.get("access-control-allow-origin"), null); + + const authToken = capability({ action: "model-account:authorize" }); + const started = await post("model-account:authorize", authToken); + assert.equal(started.status, 202); + assert.deepEqual(await started.json(), { state: "pending" }); + assert.deepEqual(calls[0], ["authorize", accountId]); + const replay = await post("model-account:authorize", authToken); + assert.equal(replay.status, 401); + + const status = await post("model-account:status", capability({ action: "model-account:status" })); + assert.deepEqual(await status.json(), { state: "saved" }); + const wrongAudience = await post("model-account:cancel", capability({ action: "model-account:cancel", aud: "fly:other:machine" })); + assert.equal(wrongAudience.status, 401); + const expired = await post("model-account:cancel", capability({ action: "model-account:cancel", exp: now - 1 })); + assert.equal(expired.status, 401); + const wrongOrigin = await post("model-account:cancel", capability({ action: "model-account:cancel" }), undefined, "https://attacker.example"); + assert.equal(wrongOrigin.status, 403); + + const canceled = await post("model-account:cancel", capability({ action: "model-account:cancel" })); + assert.deepEqual(await canceled.json(), { state: "canceled" }); + assert.deepEqual(calls.at(-1), ["cancel", accountId]); + now += 61; +}); + +test("authorization is refused while locked and expiry remains bounded to one minute", async t => { + const runtime = { state: "locked" }; + const lifecycle = { async deviceLogin() { assert.fail("locked proxy must not start authorization"); }, async cancel() { return { state: "idle" }; } }; + const server = await startBrowserControl({ runtime, lifecycle, publicKey, proxyId, ownerId, generation, appOrigin, port: 0, host: "127.0.0.1", now: () => 1_900_000_000 }); + t.after(() => server.close()); + const response = await fetch(`http://127.0.0.1:${server.address.port}/model-accounts/${accountId}/authorize`, { + method: "POST", headers: { Origin: appOrigin, "Content-Type": "application/json", Authorization: `Bearer ${capability({ action: "model-account:authorize", exp: 1_900_000_061 })}` }, body: "{}", + }); + assert.equal(response.status, 401); +}); From 96d8272754ad57e8dc9359dd2e8e35368f4f708e Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:47:15 +0000 Subject: [PATCH 086/102] GEN-2824 issue paired model account browser capabilities --- packages/api-client/src/schema.d.ts | 66 ++++++++++++++++ .../0038_proxy_browser_control_pairings.sql | 8 ++ packages/api/src/api-contract.ts | 6 ++ packages/api/src/flow-service.ts | 18 +++++ .../src/fly-credential-proxy-provisioner.ts | 6 ++ .../src/postgres/proxy-control-repository.ts | 17 ++++- packages/api/src/postgres/schema.ts | 5 ++ packages/api/src/proxy-browser-capability.ts | 38 ++++++++++ packages/api/src/proxy-control.ts | 4 +- packages/api/src/proxy-provisioner.ts | 5 +- packages/api/src/types.ts | 3 + packages/api/test/api-contract.test.ts | 9 +++ .../fly-credential-proxy-provisioner.test.ts | 7 +- .../api/test/proxy-browser-capability.test.ts | 25 ++++++ .../api/test/proxy-control-repository.test.ts | 9 ++- packages/api/test/proxy-control.test.ts | 5 +- packages/api/test/proxy-provisioner.test.ts | 2 + .../src/bootstrap-control.mjs | 7 +- .../credential-proxy/src/browser-control.mjs | 2 +- .../credential-proxy/src/control-client.mjs | 6 +- packages/credential-proxy/src/main.mjs | 22 +++++- .../test/bootstrap-control.test.mjs | 14 ++-- .../test/control-client.test.mjs | 10 +-- packages/docs/openapi.yaml | 76 +++++++++++++++++++ 24 files changed, 336 insertions(+), 34 deletions(-) create mode 100644 packages/api/migrations/0038_proxy_browser_control_pairings.sql create mode 100644 packages/api/src/proxy-browser-capability.ts create mode 100644 packages/api/test/proxy-browser-capability.test.ts diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index a910f642..e7c92965 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -1,5 +1,29 @@ // Generated from packages/docs/openapi.yaml. Do not edit. export interface paths { + "/api/v1/model-accounts/{accountId}/browser-capability": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Issue a short-lived browser capability for a paired model account + * @description Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens. + * + * Requires the integrations:write scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + post: operations["post_api_v1_model_accounts_accountId_browser_capability"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/unlock-key/verification": { parameters: { query?: never; @@ -2683,6 +2707,48 @@ export interface components { } export type $defs = Record; export interface operations { + post_api_v1_model_accounts_accountId_browser_capability: { + parameters: { + query?: never; + header?: never; + path: { + accountId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + /** @enum {string} */ + action: "authorize" | "status" | "cancel"; + }; + }; + }; + responses: { + /** @description Paired proxy endpoint and one-use capability */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** Format: uri */ + endpoint: string; + capability: string; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 409: components["responses"]["Error409"]; + 415: components["responses"]["Error415"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; get_api_v1_unlock_key_verification: { parameters: { query?: never; diff --git a/packages/api/migrations/0038_proxy_browser_control_pairings.sql b/packages/api/migrations/0038_proxy_browser_control_pairings.sql new file mode 100644 index 00000000..45a8eb2e --- /dev/null +++ b/packages/api/migrations/0038_proxy_browser_control_pairings.sql @@ -0,0 +1,8 @@ +CREATE TABLE app.credential_proxy_browser_pairings ( + tenant_id uuid PRIMARY KEY REFERENCES app.tenants(id) ON DELETE CASCADE, + proxy_identity text NOT NULL, + boot_epoch uuid NOT NULL, + browser_origin text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (browser_origin ~ '^https://[^/]+$') +); diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index aa6e4b45..aae9e10a 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -135,6 +135,7 @@ const bearerIntegrationWriteResult = z.object({ revision: z.number().int().nonne const bearerIntegrationDeleteResult = z.object({ id: z.uuid(), deleted: z.literal(true), revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const unlockKeyVerification = z.object({ ownerId: z.string(), verification: z.object({ generationId: z.uuid(), encryptedVerification: z.string() }).strict().nullable() }).strict(); const unlockKeyVerificationInput = z.object({ generationId: z.uuid(), encryptedVerification: z.string().min(1).max(4096) }).strict(); +const modelAccountBrowserCapabilityInput = z.object({ action: z.enum(["authorize", "status", "cancel"]) }).strict(); function route( method: string, @@ -213,6 +214,11 @@ function authRoute(method: string, path: string, action: AuthAction, summary: st } export const API_ROUTES: RouteContract[] = [ + route("POST", "/api/v1/model-accounts/{accountId}/browser-capability", "integrations:write", { + summary: "Issue a short-lived browser capability for a paired model account", + description: "Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens.", + responses: { "200": { description: "Paired proxy endpoint and one-use capability", content: { "application/json": { schema: jsonSchema(z.object({ endpoint: z.url(), capability: z.string() }), "output") } } } }, + }, (service, { params, body }) => service.issueModelAccountBrowserCapability(params.accountId, body.action), { body: modelAccountBrowserCapabilityInput, maxBodyBytes: 1024, ownerSession: true, parameters: z.object({ accountId: z.uuid() }) }), route("GET", "/api/v1/unlock-key/verification", "integrations:read", { summary: "Get the current encrypted unlock-key verification record", description: "Requires an interactive owner browser session. Returns user-encrypted verification ciphertext and its key generation; the unlock key is never accepted or returned.", responses: { "200": { description: "Encrypted verification record or empty setup state", content: { "application/json": { schema: jsonSchema(unlockKeyVerification, "output") } } } } }, service => service.getUnlockKeyVerification(), { ownerSession: true }), route("POST", "/api/v1/unlock-key/verification", "integrations:write", { summary: "Create the encrypted unlock-key verification record", description: "Requires an interactive owner browser session. Stores only browser-encrypted verification ciphertext and permits one setup per account.", responses: { "200": { description: "Created verification record", content: { "application/json": { schema: jsonSchema(z.object({ generationId: z.uuid() }), "output") } } } } }, (service, { body }) => service.createUnlockKeyVerification(body), { body: unlockKeyVerificationInput, maxBodyBytes: 8192, ownerSession: true }), route("GET", "/api/v1/integrations/bearer", "flows:read", { diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index 93a390fe..865bda9f 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -17,6 +17,8 @@ import type { JobPageQuery } from "./job-contracts"; import { PostgresJobRepository, StaleJobEdit } from "./postgres/job-repository"; import { ConnectionRepository } from "./postgres/connection-repository"; import { EncryptedIntegrationRepository, IntegrationIdempotencyConflictError, IntegrationValidationError, StaleIntegrationRevisionError } from "./postgres/encrypted-integration-repository"; +import { ProxyControlRepository } from "./postgres/proxy-control-repository"; +import { issueModelAccountCapability } from "./proxy-browser-capability"; import { decrypt, encrypt } from "./crypto"; import { InvocationService, type Job, type Trigger } from "./job-domain"; import { RunQueryRepository } from "./postgres/run-query-repository"; @@ -571,6 +573,22 @@ export class ApiService { if (!rows.length) throw new ServiceError(409, "conflict", "An unlock key is already set up for this account."); return { generationId: input.generationId }; } + async issueModelAccountBrowserCapability(accountId: string, action: "authorize" | "status" | "cancel") { + await this.authorizeOwnerSession("integrations:write"); + if (!this.env.MODEL_ACCOUNT_CAPABILITY_PRIVATE_KEY) throw new ServiceError(503, "operation_failed", "Model-account browser authorization is unavailable."); + const [pairing, accounts] = await Promise.all([ + new ProxyControlRepository(databaseFor(this.env)).browserPairing(this.auth.tenantId), + this.encryptedIntegrations().records("model"), + ]); + if (!pairing) throw new ServiceError(503, "operation_failed", "The credential proxy is not paired for browser authorization."); + if (!accounts.some(account => account.id === accountId)) throw new ServiceError(404, "not_found", "Model account not found."); + const capability = await issueModelAccountCapability({ + privateKeyPem: this.env.MODEL_ACCOUNT_CAPABILITY_PRIVATE_KEY, + proxyIdentity: pairing.proxyIdentity, ownerId: this.auth.tenantId, accountId, action, + appOrigin: this.env.APP_ORIGIN, generation: pairing.bootEpoch, + }); + return { endpoint: `${pairing.browserOrigin}/model-accounts/${accountId}/${action}`, capability }; + } async removeTailIntegration(id: string) { await this.authorize("integrations:write"); if (!await this.connections().delete(`cloudflare-tail:${id}`)) throw new ServiceError(404, "not_found", "Integration not found"); return { id, deleted: true }; } async linearProjects() { await this.authorize("integrations:read"); return publicValue(z.array(namedOption), await this.providers().linearProjects()); } async linearOptions() { await this.authorize("integrations:read"); return publicValue(providerOptions, await this.providers().linearOptions()); } diff --git a/packages/api/src/fly-credential-proxy-provisioner.ts b/packages/api/src/fly-credential-proxy-provisioner.ts index 101b0ba5..c959537e 100644 --- a/packages/api/src/fly-credential-proxy-provisioner.ts +++ b/packages/api/src/fly-credential-proxy-provisioner.ts @@ -13,6 +13,7 @@ export type FlyCredentialProxyConfig = { image: string; region: string; size: string; + browserCapabilityPublicKey: string; memoryMb?: number; controlOrigin: string; /** Stable across retries of one provisioning operation. */ @@ -47,6 +48,7 @@ export function createFlyCredentialProxyGuest(options: { const result = await transport.run(["/usr/local/bin/factorize-proxy-startup"], `${JSON.stringify({ type: "register-boot", controlOrigin: input.controlOrigin, + browserOrigin: input.browserOrigin, bootEpoch: input.bootEpoch, credential: input.credential, })}\n`); @@ -86,6 +88,7 @@ function validate(input: FlyCredentialProxyConfig): void { if (!IMAGE_DIGEST.test(input.image)) throw new TypeError("Credential proxy image must be pinned by sha256 digest"); if (!/^[a-z0-9]{3}$/.test(input.region)) throw new TypeError("Invalid credential proxy region"); if (!/^[a-z0-9-]{1,64}$/.test(input.size)) throw new TypeError("Invalid credential proxy size"); + if (!input.browserCapabilityPublicKey.startsWith("-----BEGIN PUBLIC KEY-----")) throw new TypeError("Proxy browser capability public key is required"); if (input.memoryMb !== undefined && (!Number.isInteger(input.memoryMb) || input.memoryMb < 256 || input.memoryMb > 8192)) throw new TypeError("Credential proxy memory must be between 256 and 8192 MiB"); } @@ -113,7 +116,9 @@ export function createFlyCredentialProxyProvisioner(options: FlyCredentialProxyP env: { FACTORIZE_PROXY_TUNNEL_SERVER_NAME: `${input.app}.internal`, FACTORIZE_PROXY_TUNNEL_PORT: "9090", + FACTORIZE_PROXY_BROWSER_CAPABILITY_PUBLIC_KEY: input.browserCapabilityPublicKey, }, + services: [{ protocol: "tcp", internal_port: 8080, ports: [{ port: 443, handlers: ["tls", "http"] }], force_instance_key: true }], metadata: { factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId, @@ -136,6 +141,7 @@ export function createFlyCredentialProxyProvisioner(options: FlyCredentialProxyP proxyIdentity, bootEpoch, controlOrigin: origin.origin, + browserOrigin: `https://${input.app}.fly.dev`, }; try { await provisionCredentialProxy(bootstrap, { diff --git a/packages/api/src/postgres/proxy-control-repository.ts b/packages/api/src/postgres/proxy-control-repository.ts index 09fd28c9..047d0837 100644 --- a/packages/api/src/postgres/proxy-control-repository.ts +++ b/packages/api/src/postgres/proxy-control-repository.ts @@ -26,8 +26,10 @@ export class ProxyControlRepository { return row ? { tenantId: row.tenant_id, proxyIdentity: row.proxy_identity } : null; } - async registerBoot(token: string, bootEpoch: string): Promise { + async registerBoot(token: string, bootEpoch: string, browserOrigin: string): Promise { if (!uuid.test(bootEpoch)) throw new TypeError("bootEpoch must be a UUID"); + const parsedOrigin = new URL(browserOrigin); + if (parsedOrigin.protocol !== "https:" || parsedOrigin.origin !== browserOrigin || parsedOrigin.username || parsedOrigin.password) throw new TypeError("browser origin must be canonical HTTPS"); const tokenDigest = Buffer.from(await digest(token)); return this.database.transaction(async client => { const credential = await client.query<{ tenant_id: string; proxy_identity: string }>(`SELECT tenant_id,proxy_identity @@ -45,10 +47,23 @@ export class ProxyControlRepository { WHERE token_digest=$1 AND boot_epoch=$2 AND consumed_at IS NULL`, [tokenDigest, bootEpoch, Buffer.from(await digest(proxyToken))]); await client.query(`UPDATE app.credential_run_grants SET revoked_at=now() WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch<>$3 AND revoked_at IS NULL`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); + await client.query(`INSERT INTO app.credential_proxy_browser_pairings(tenant_id,proxy_identity,boot_epoch,browser_origin) + VALUES ($1,$2,$3,$4) ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity, + boot_epoch=excluded.boot_epoch,browser_origin=excluded.browser_origin,updated_at=now()`, + [identity.tenant_id, identity.proxy_identity, bootEpoch, browserOrigin]); return { tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch, controlCredential: proxyToken }; }); } + async browserPairing(tenantId: string): Promise<{ proxyIdentity: string; bootEpoch: string; browserOrigin: string } | null> { + const result = await this.database.pool.query<{ proxy_identity: string; boot_epoch: string; browser_origin: string }>( + `SELECT b.proxy_identity,b.boot_epoch,b.browser_origin FROM app.credential_proxy_browser_pairings b + JOIN app.credential_proxy_states p ON p.tenant_id=b.tenant_id AND p.proxy_identity=b.proxy_identity AND p.boot_epoch=b.boot_epoch + WHERE b.tenant_id=$1 AND p.disabled_at IS NULL AND p.revoked_at IS NULL`, [tenantId]); + const row = result.rows[0]; + return row ? { proxyIdentity: row.proxy_identity, bootEpoch: row.boot_epoch, browserOrigin: row.browser_origin } : null; + } + /** Authenticates the proxy's memory-held control credential against its current registered boot. */ async authenticateProxy(token: string): Promise | null> { if (typeof token !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(token)) return null; diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index f64f2d72..89dc11ce 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -53,6 +53,11 @@ export const integrationChanges = app.table("integration_changes", { export const credentialProxyStates = app.table("credential_proxy_states", { tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), disabledAt: timestamp("disabled_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), updatedAt: updatedAt(), }); +export const credentialProxyBrowserPairings = app.table("credential_proxy_browser_pairings", { + tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), + proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), + browserOrigin: text("browser_origin").notNull(), updatedAt: updatedAt(), +}); export const credentialProxyControlCredentials = app.table("credential_proxy_control_credentials", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), tokenDigest: bytea("token_digest").notNull().unique(), proxyTokenDigest: bytea("proxy_token_digest").unique(), revokedAt: timestamp("revoked_at", { withTimezone: true }), consumedAt: timestamp("consumed_at", { withTimezone: true }), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.proxyIdentity] })]); export const credentialRunGrants = app.table("credential_run_grants", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), vmIdentity: text("vm_identity").notNull(), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), grantRevision: bigint("grant_revision", { mode: "number" }).notNull(), revokedAt: timestamp("revoked_at", { withTimezone: true }), createdAt: createdAt(), diff --git a/packages/api/src/proxy-browser-capability.ts b/packages/api/src/proxy-browser-capability.ts new file mode 100644 index 00000000..fcb3ed9e --- /dev/null +++ b/packages/api/src/proxy-browser-capability.ts @@ -0,0 +1,38 @@ +const encoder = new TextEncoder(); +const base64url = (bytes: Uint8Array) => btoa(String.fromCharCode(...bytes)).replace(/=/g, "").replace(/\+/g, "-").replace(/\//g, "_"); + +function decodePem(pem: string): Uint8Array { + const body = pem.replace(/-----BEGIN PRIVATE KEY-----|-----END PRIVATE KEY-----|\s/g, ""); + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(body)) throw new TypeError("invalid browser capability signing key"); + const binary = atob(body); + return Uint8Array.from(binary, character => character.charCodeAt(0)); +} + +/** Issue a one-use capability accepted only by the paired proxy listener. */ +export async function issueModelAccountCapability(input: { + privateKeyPem: string; + proxyIdentity: string; + ownerId: string; + accountId: string; + action: "authorize" | "status" | "cancel"; + appOrigin: string; + generation: number | string; + now?: number; + jti?: string; +}): Promise { + const now = input.now ?? Math.floor(Date.now() / 1000); + const header = base64url(encoder.encode(JSON.stringify({ alg: "EdDSA", typ: "JWT" }))); + const claims = base64url(encoder.encode(JSON.stringify({ + iss: "factorize", aud: input.proxyIdentity, sub: input.ownerId, + action: `model-account:${input.action}`, accountId: input.accountId, + origin: input.appOrigin, generation: input.generation, exp: now + 60, + jti: input.jti ?? base64url(crypto.getRandomValues(new Uint8Array(24))), + }))); + const payload = `${header}.${claims}`; + const rawKey = decodePem(input.privateKeyPem); + const keyData = new ArrayBuffer(rawKey.byteLength); + new Uint8Array(keyData).set(rawKey); + const key = await crypto.subtle.importKey("pkcs8", keyData, { name: "Ed25519" }, false, ["sign"]); + const signature = await crypto.subtle.sign("Ed25519", key, encoder.encode(payload)); + return `${payload}.${base64url(new Uint8Array(signature))}`; +} diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index e947d35b..64db6ad6 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -34,8 +34,8 @@ export async function proxyControlFetch(request: Request, env: Parameters key !== "bootEpoch") || typeof (body as any).bootEpoch !== "string") return json({ error: "invalid_request" }, 400); - const registration = await repository.registerBoot(token, (body as any).bootEpoch); + if (!body || typeof body !== "object" || Array.isArray(body) || Object.keys(body).sort().join(",") !== "bootEpoch,browserOrigin" || typeof (body as any).bootEpoch !== "string" || typeof (body as any).browserOrigin !== "string") return json({ error: "invalid_request" }, 400); + const registration = await repository.registerBoot(token, (body as any).bootEpoch, (body as any).browserOrigin); if (!registration) return json({ error: "proxy_registration_unavailable" }, 403); return json({ registered: true, tenantId: registration.tenantId, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch, controlCredential: registration.controlCredential }); } diff --git a/packages/api/src/proxy-provisioner.ts b/packages/api/src/proxy-provisioner.ts index f6d4957a..c32e7e5d 100644 --- a/packages/api/src/proxy-provisioner.ts +++ b/packages/api/src/proxy-provisioner.ts @@ -6,6 +6,7 @@ export type ProxyBootstrapProvisioning = { /** A new UUID for every process boot. */ bootEpoch: string; controlOrigin: string; + browserOrigin: string; }; /** @@ -25,9 +26,11 @@ export async function provisionCredentialProxy( if (origin.protocol !== "https:" || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) { throw new TypeError("controlOrigin must be an HTTPS origin"); } + const browserOrigin = new URL(input.browserOrigin); + if (browserOrigin.protocol !== "https:" || browserOrigin.origin !== input.browserOrigin || browserOrigin.username || browserOrigin.password) throw new TypeError("browserOrigin must be an HTTPS origin"); const credential = await dependencies.control.provisionCredential(input.tenantId, input.proxyIdentity, input.bootEpoch); try { - await dependencies.startProxy({ ...input, controlOrigin: origin.origin, credential }); + await dependencies.startProxy({ ...input, controlOrigin: origin.origin, browserOrigin: browserOrigin.origin, credential }); } catch (error) { // A failed/ambiguous start must not leave a valid bootstrap credential. await dependencies.control.revokeCredential(input.tenantId, input.proxyIdentity).catch(() => false); diff --git a/packages/api/src/types.ts b/packages/api/src/types.ts index 1fc1e396..f70bbbfa 100644 --- a/packages/api/src/types.ts +++ b/packages/api/src/types.ts @@ -15,6 +15,9 @@ export interface Env { CLICKUP_CLIENT_SECRET?: string; CREDENTIAL_ENCRYPTION_KEY: string; SESSION_SIGNING_SECRET: string; + /** PKCS#8 Ed25519 key used only for short-lived proxy browser capabilities. */ + MODEL_ACCOUNT_CAPABILITY_PRIVATE_KEY?: string; + MODEL_ACCOUNT_CAPABILITY_PUBLIC_KEY?: string; GITHUB_APP_ID?: string; GITHUB_APP_SLUG?: string; GITHUB_APP_PRIVATE_KEY?: string; diff --git a/packages/api/test/api-contract.test.ts b/packages/api/test/api-contract.test.ts index 688d7528..7a803c25 100644 --- a/packages/api/test/api-contract.test.ts +++ b/packages/api/test/api-contract.test.ts @@ -64,6 +64,15 @@ describe("versioned API contract", () => { expect(post?.body?.safeParse({ generationId: "00000000-0000-4000-8000-000000000001", encryptedVerification: "ciphertext", key: "plain" }).success).toBe(false); expect(openapi.paths["/api/v1/unlock-key/verification"].post.requestBody).toBeDefined(); }); + it("publishes model-account proxy capabilities as owner-only API operations", () => { + const route = API_ROUTES.find(item => item.method === "POST" && item.path === "/api/v1/model-accounts/{accountId}/browser-capability"); + expect(route?.scope).toBe("integrations:write"); + expect(route?.ownerSession).toBe(true); + expect(route?.parameters.shape.accountId).toBeDefined(); + expect(route?.body?.safeParse({ action: "authorize" }).success).toBe(true); + expect(route?.body?.safeParse({ action: "authorize", accountId: "attacker" }).success).toBe(false); + expect(openapi.paths["/api/v1/model-accounts/{accountId}/browser-capability"].post.requestBody).toBeDefined(); + }); it("accepts code jobs without agent prompts or models and rejects missing scripts", () => { const code = { name: "Export", slug: "export", executionType: "code", script: "#!/bin/sh\nprintf '{\"status\":\"succeeded\"}' > \"$FACTORIZE_RESULT_PATH\"", executionConfigId: "exe", triggers: [] }; expect(jobInputSchema.safeParse(code).success).toBe(true); diff --git a/packages/api/test/fly-credential-proxy-provisioner.test.ts b/packages/api/test/fly-credential-proxy-provisioner.test.ts index d2b1a976..098dcc06 100644 --- a/packages/api/test/fly-credential-proxy-provisioner.test.ts +++ b/packages/api/test/fly-credential-proxy-provisioner.test.ts @@ -9,6 +9,7 @@ const input = { image: `ghcr.io/generationsoftware/credential-proxy@sha256:${"a".repeat(64)}`, region: "iad", size: "shared-cpu-1x", + browserCapabilityPublicKey: "-----BEGIN PUBLIC KEY-----\nfixture\n-----END PUBLIC KEY-----", controlOrigin: "https://factorize.example", operationId: "tenant-proxy-start-1", }; @@ -37,7 +38,8 @@ describe("Fly credential proxy startup", () => { image: input.image, region: input.region, cmd: ["/usr/local/bin/factorize-credential-proxy", "--boot-epoch", epoch], - env: { FACTORIZE_PROXY_TUNNEL_SERVER_NAME: "factorize-proxies.internal", FACTORIZE_PROXY_TUNNEL_PORT: "9090" }, + env: { FACTORIZE_PROXY_TUNNEL_SERVER_NAME: "factorize-proxies.internal", FACTORIZE_PROXY_TUNNEL_PORT: "9090", FACTORIZE_PROXY_BROWSER_CAPABILITY_PUBLIC_KEY: input.browserCapabilityPublicKey }, + services: [{ protocol: "tcp", internal_port: 8080, ports: [{ port: 443, handlers: ["tls", "http"] }], force_instance_key: true }], restart: { policy: "no" }, auto_start: false, metadata: expect.objectContaining({ factorize_role: "credential-proxy", factorize_tenant_id: input.tenantId }), @@ -52,6 +54,7 @@ describe("Fly credential proxy startup", () => { proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, + browserOrigin: `https://${input.app}.fly.dev`, credential: `fzc_${"b".repeat(43)}`, })); }); @@ -88,7 +91,7 @@ describe("Fly credential proxy guest adapter", () => { return child; }) as any; const guest = createFlyCredentialProxyGuest({ app: input.app, machine: machine.id, token: "server-token-fixture", spawnProcess }); - await guest.startProxy({ tenantId: input.tenantId, proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, credential: `fzc_${"c".repeat(43)}` }); + await guest.startProxy({ tenantId: input.tenantId, proxyIdentity: "fly:factorize-proxies:machine-1", bootEpoch: epoch, controlOrigin: input.controlOrigin, browserOrigin: `https://${input.app}.fly.dev`, credential: `fzc_${"c".repeat(43)}` }); expect(calls[0]?.[1]).toContain("/usr/local/bin/factorize-proxy-startup"); expect(String(calls[1]?.[1])).toContain(`"type":"register-boot"`); expect(String(calls[1]?.[1])).toContain(`"credential":"fzc_${"c".repeat(43)}"`); diff --git a/packages/api/test/proxy-browser-capability.test.ts b/packages/api/test/proxy-browser-capability.test.ts new file mode 100644 index 00000000..3729c98a --- /dev/null +++ b/packages/api/test/proxy-browser-capability.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { generateKeyPairSync, verify } from "node:crypto"; +import { issueModelAccountCapability } from "../src/proxy-browser-capability"; + +describe("proxy browser capabilities", () => { + it("signs an account/action/origin/boot-bound one-minute EdDSA token", async () => { + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + const token = await issueModelAccountCapability({ + privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(), + proxyIdentity: "fly:proxy:machine-1", ownerId: "11111111-1111-4111-8111-111111111111", + accountId: "22222222-2222-4222-8222-222222222222", action: "authorize", + appOrigin: "https://factorize.example", generation: "33333333-3333-4333-8333-333333333333", + now: 1_900_000_000, jti: "one-use-fixture-nonce-123456", + }); + const [header, claims, signature] = token.split("."); + expect(JSON.parse(Buffer.from(header!, "base64url").toString())).toEqual({ alg: "EdDSA", typ: "JWT" }); + expect(JSON.parse(Buffer.from(claims!, "base64url").toString())).toMatchObject({ + iss: "factorize", aud: "fly:proxy:machine-1", sub: "11111111-1111-4111-8111-111111111111", + action: "model-account:authorize", accountId: "22222222-2222-4222-8222-222222222222", + origin: "https://factorize.example", generation: "33333333-3333-4333-8333-333333333333", + exp: 1_900_000_060, jti: "one-use-fixture-nonce-123456", + }); + expect(verify(null, Buffer.from(`${header}.${claims}`), publicKey, Buffer.from(signature!, "base64url"))).toBe(true); + }); +}); diff --git a/packages/api/test/proxy-control-repository.test.ts b/packages/api/test/proxy-control-repository.test.ts index 9508486a..b622e0df 100644 --- a/packages/api/test/proxy-control-repository.test.ts +++ b/packages/api/test/proxy-control-repository.test.ts @@ -33,21 +33,22 @@ describe("dedicated credential proxy control identity", () => { const tx = { query: vi.fn().mockImplementation((sql: string) => sql.includes("credential_proxy_control_credentials") ? credentialQuery() : sql.includes("INSERT INTO app.credential_proxy_states") ? stateQuery() : Promise.resolve({ rows: [], rowCount: 1 })) }; (repository as any).database = { transaction: (work: (client: any) => Promise) => work(tx) }; const epoch = "22222222-2222-4222-8222-222222222222"; - const registration = await repository.registerBoot("fzc_" + "a".repeat(43), epoch); + const registration = await repository.registerBoot("fzc_" + "a".repeat(43), epoch, "https://proxy.example"); expect(registration).toMatchObject({ tenantId: identity.tenant_id, proxyIdentity: identity.proxy_identity, bootEpoch: epoch }); expect(registration?.controlCredential).toMatch(/^fzc_[A-Za-z0-9_-]{43}$/); - expect(tx.query.mock.calls.map(call => call[0])).toHaveLength(4); + expect(tx.query.mock.calls.map(call => call[0])).toHaveLength(5); expect(tx.query.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(tx.query.mock.calls[2][0]).toContain("consumed_at=now()"); expect(tx.query.mock.calls[3][0]).toContain("boot_epoch<>$3"); + expect(tx.query.mock.calls[4][0]).toContain("credential_proxy_browser_pairings"); credentialQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 }); - expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch)).toBeNull(); + expect(await repository.registerBoot("fzc_" + "a".repeat(43), epoch, "https://proxy.example")).toBeNull(); }); it("rejects malformed boot epochs before opening a transaction", async () => { const transaction = vi.fn(); const repository = new ProxyControlRepository({ transaction, pool: { query: vi.fn() } } as any); - await expect(repository.registerBoot("fzc_" + "a".repeat(43), "old-boot")).rejects.toThrow("bootEpoch must be a UUID"); + await expect(repository.registerBoot("fzc_" + "a".repeat(43), "old-boot", "https://proxy.example")).rejects.toThrow("bootEpoch must be a UUID"); expect(transaction).not.toHaveBeenCalled(); }); }); diff --git a/packages/api/test/proxy-control.test.ts b/packages/api/test/proxy-control.test.ts index ae33a978..3bf692dd 100644 --- a/packages/api/test/proxy-control.test.ts +++ b/packages/api/test/proxy-control.test.ts @@ -21,7 +21,7 @@ function envFor({ credentialFound = true, stateSaved = true } = {}) { return { DATABASE: { pool: { query }, transaction: (work: (client: any) => Promise) => work({ query: transactionQuery }) }, transactionQuery, query } as any; } -function request({ authorization, body = { bootEpoch }, method = "POST", path = "/internal/credential-proxy/register-boot" }: { authorization?: string; body?: unknown; method?: string; path?: string } = {}) { +function request({ authorization, body = { bootEpoch, browserOrigin: "https://proxy.example" }, method = "POST", path = "/internal/credential-proxy/register-boot" }: { authorization?: string; body?: unknown; method?: string; path?: string } = {}) { return new Request(`https://factorize.example${path}`, { method, headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}) }, body: method === "POST" ? JSON.stringify(body) : undefined }); } @@ -31,9 +31,10 @@ describe("dedicated proxy boot registration endpoint", () => { const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ registered: true, tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: expect.stringMatching(/^fzc_[A-Za-z0-9_-]{43}$/) }); - expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(4); + expect(env.transactionQuery.mock.calls.map((call: any[]) => call[0])).toHaveLength(5); expect(env.transactionQuery.mock.calls[0][0]).toContain("boot_epoch=$2 AND revoked_at IS NULL AND consumed_at IS NULL"); expect(env.transactionQuery.mock.calls[2][0]).toContain("consumed_at=now()"); + expect(env.transactionQuery.mock.calls[4][0]).toContain("credential_proxy_browser_pairings"); }); it("rejects guest job tokens before database access", async () => { diff --git a/packages/api/test/proxy-provisioner.test.ts b/packages/api/test/proxy-provisioner.test.ts index 1a107b53..bc6e9eca 100644 --- a/packages/api/test/proxy-provisioner.test.ts +++ b/packages/api/test/proxy-provisioner.test.ts @@ -6,6 +6,7 @@ const input = { proxyIdentity: "fly-app/proxy-machine", bootEpoch: "22222222-2222-4222-8222-222222222222", controlOrigin: "https://factorize.example/", + browserOrigin: "https://proxy.example", }; describe("credential proxy bootstrap provisioning", () => { @@ -40,6 +41,7 @@ describe("credential proxy bootstrap provisioning", () => { const control = { provisionCredential: vi.fn(), revokeCredential: vi.fn() }; await expect(provisionCredentialProxy({ ...input, controlOrigin: "http://factorize.example" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); await expect(provisionCredentialProxy({ ...input, controlOrigin: "https://factorize.example/path" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); + await expect(provisionCredentialProxy({ ...input, browserOrigin: "http://proxy.example" }, { control, startProxy: vi.fn() })).rejects.toThrow("HTTPS origin"); expect(control.provisionCredential).not.toHaveBeenCalled(); }); }); diff --git a/packages/credential-proxy/src/bootstrap-control.mjs b/packages/credential-proxy/src/bootstrap-control.mjs index 8d17234c..68ba7eb1 100644 --- a/packages/credential-proxy/src/bootstrap-control.mjs +++ b/packages/credential-proxy/src/bootstrap-control.mjs @@ -54,8 +54,9 @@ export async function startProxyBootstrapControl({ runtime, socketPath = "/run/f } if (request?.type !== "register-boot") return finish({ error: "invalid bootstrap request" }); if (!request || typeof request !== "object" || Array.isArray(request) - || Object.keys(request).sort().join(",") !== "bootEpoch,controlOrigin,credential,type" + || Object.keys(request).sort().join(",") !== "bootEpoch,browserOrigin,controlOrigin,credential,type" || request.bootEpoch !== runtime.bootEpoch || typeof request.credential !== "string" + || typeof request.browserOrigin !== "string" || !/^fzc_[A-Za-z0-9_-]{43}$/.test(request.credential)) { return finish({ error: "invalid bootstrap request" }); } @@ -64,10 +65,10 @@ export async function startProxyBootstrapControl({ runtime, socketPath = "/run/f socket.removeAllListeners("data"); let credential = request.credential; try { - const registration = await register({ controlOrigin: request.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + const registration = await register({ controlOrigin: request.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch, browserOrigin: request.browserOrigin }); // Keep the exchanged proxy credential inside the proxy process. The // one-shot startup command receives only the public registration result. - await onRegistered(registration, { controlOrigin: request.controlOrigin, bootEpoch: runtime.bootEpoch }); + await onRegistered(registration, { controlOrigin: request.controlOrigin, browserOrigin: request.browserOrigin, bootEpoch: runtime.bootEpoch }); finish({ registered: true, proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { finish({ error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/src/browser-control.mjs b/packages/credential-proxy/src/browser-control.mjs index 5911ba14..e87e1218 100644 --- a/packages/credential-proxy/src/browser-control.mjs +++ b/packages/credential-proxy/src/browser-control.mjs @@ -60,7 +60,7 @@ export async function startBrowserControl({ now = () => Math.floor(Date.now() / 1000), } = {}) { if (!runtime || !lifecycle || !publicKey || typeof proxyId !== "string" || !proxyId - || !UUID.test(ownerId ?? "") || !Number.isSafeInteger(generation) || generation < 1 + || !UUID.test(ownerId ?? "") || !(Number.isSafeInteger(generation) && generation > 0 || UUID.test(generation ?? "")) || !Number.isInteger(port) || port < 0 || port > 65535) { throw new TypeError("browser control configuration is invalid"); } diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 6ea36bab..9b4656c9 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -7,16 +7,18 @@ function validateOrigin(controlOrigin) { return origin; } -export async function registerProxyBoot({ controlOrigin, token, bootEpoch, fetchImpl = fetch }) { +export async function registerProxyBoot({ controlOrigin, token, bootEpoch, browserOrigin, fetchImpl = fetch }) { const origin = validateOrigin(controlOrigin); if (!/^fzc_[A-Za-z0-9_-]{43}$/.test(token ?? "")) throw new TypeError("invalid proxy control credential"); if (!bootEpochPattern.test(bootEpoch ?? "")) throw new TypeError("invalid proxy boot epoch"); + const browser = new URL(browserOrigin); + if (browser.protocol !== "https:" || browser.origin !== browserOrigin || browser.username || browser.password) throw new TypeError("invalid browser control origin"); let credential = token; try { const response = await fetchImpl(new URL("/internal/credential-proxy/register-boot", origin), { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${credential}` }, - body: JSON.stringify({ bootEpoch }), + body: JSON.stringify({ bootEpoch, browserOrigin }), signal: AbortSignal.timeout(10_000), redirect: "error", }); diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 4b2765bb..016c80a3 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -10,6 +10,7 @@ import { createCodexAuthHelper } from "./codex-auth-helper.mjs"; import { createModelAccountLifecycle } from "./model-account-lifecycle.mjs"; import { createModelAccountPersistenceAdapter } from "./control-client.mjs"; import { decryptModelAccountCache, encryptModelAccountCache } from "./model-account-crypto.mjs"; +import { startBrowserControl } from "./browser-control.mjs"; const epochIndex = process.argv.indexOf("--boot-epoch"); const bootEpoch = epochIndex < 0 ? randomUUID() : process.argv[epochIndex + 1]; @@ -26,8 +27,9 @@ let tunnel; let revocationMonitor; let modelAccounts; let modelAccountLifecycle; +let browserControl; -async function activateRegistration(registration, origin) { +async function activateRegistration(registration, origin, browserOrigin) { if (tunnel) throw new Error("proxy boot is already registered"); const persistence = createModelAccountPersistenceAdapter({ controlOrigin: origin, @@ -44,6 +46,16 @@ async function activateRegistration(registration, origin) { encryptCache: encryptModelAccountCache, decryptCache: decryptModelAccountCache, }); + if (!process.env.FACTORIZE_PROXY_BROWSER_CAPABILITY_PUBLIC_KEY) throw new Error("proxy browser capability verification key is unavailable"); + browserControl = await startBrowserControl({ + runtime, + lifecycle: modelAccountLifecycle, + publicKey: process.env.FACTORIZE_PROXY_BROWSER_CAPABILITY_PUBLIC_KEY, + proxyId: registration.proxyIdentity, + ownerId: registration.tenantId, + generation: registration.bootEpoch, + appOrigin: process.env.FACTORIZE_APP_ORIGIN ?? "https://app.factorize.sh", + }); const resolveGrants = createProxyIntegrationGrantResolver({ runtime, controlOrigin: origin, controlCredential: registration.controlCredential }); const started = await startAuthenticatedProxyTunnel({ tls: tlsAuthority.tls, @@ -68,7 +80,7 @@ async function activateRegistration(registration, origin) { const bootstrapControl = await startProxyBootstrapControl({ runtime, socketPath: process.env.FACTORIZE_PROXY_BOOTSTRAP_SOCKET, - onRegistered: (registration, request) => activateRegistration(registration, request.controlOrigin), + onRegistered: (registration, request) => activateRegistration(registration, request.controlOrigin, request.browserOrigin), issueVmCertificate: ({ vmIdentity, csr }) => { if (!tunnel) throw new Error("proxy tunnel is not registered"); return tlsAuthority.signVmCsr({ vmIdentity, csr }); @@ -93,6 +105,8 @@ async function stop() { controlOrigin = undefined; await revocationMonitor?.close(); revocationMonitor = undefined; + await browserControl?.close().catch(() => {}); + browserControl = undefined; await runtime.shutdown(); await tunnel?.close().catch(() => {}); tunnel = undefined; @@ -129,8 +143,8 @@ lines.on("line", async (line) => { let credential = command.credential; command.credential = undefined; try { - const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, token: credential, bootEpoch: runtime.bootEpoch }); - await activateRegistration(registration, command.controlOrigin); + const registration = await registerProxyBoot({ controlOrigin: command.controlOrigin, browserOrigin: command.browserOrigin, token: credential, bootEpoch: runtime.bootEpoch }); + await activateRegistration(registration, command.controlOrigin, command.browserOrigin); respond({ type: "boot-registered", proxyIdentity: registration.proxyIdentity, bootEpoch: registration.bootEpoch }); } catch { respond({ type: "error", error: "proxy boot registration failed" }); diff --git a/packages/credential-proxy/test/bootstrap-control.test.mjs b/packages/credential-proxy/test/bootstrap-control.test.mjs index 5e7956ac..de122d5f 100644 --- a/packages/credential-proxy/test/bootstrap-control.test.mjs +++ b/packages/credential-proxy/test/bootstrap-control.test.mjs @@ -42,12 +42,12 @@ test("local startup control redeems the one-use credential and leaves a private async value => { calls.push(value); return { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }; }, (value, request) => registered.push({ value, request }), ); - const result = await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + const result = await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch, credential: token }); assert.deepEqual(result, { registered: true, proxyIdentity: "fly:app:machine", bootEpoch: epoch }); - assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch }]); + assert.deepEqual(calls, [{ controlOrigin: "https://factorize.example", token, bootEpoch: epoch, browserOrigin: "https://proxy.example" }]); assert.deepEqual(registered, [{ value: { proxyIdentity: "fly:app:machine", bootEpoch: epoch, controlCredential: token }, - request: { controlOrigin: "https://factorize.example", bootEpoch: epoch }, + request: { controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch }, }]); assert.equal(JSON.stringify(result).includes(token), false, "startup response never returns the proxy control credential"); assert.equal((await stat(socketPath)).mode & 0o777, 0o600); @@ -57,9 +57,9 @@ test("local startup control rejects run credentials, epoch substitution, and ext const calls = []; const socketPath = await fixture(async value => { calls.push(value); return {}; }); for (const payload of [ - { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, - { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, - { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, + { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch, credential: `fzt_run_${"b".repeat(32)}` }, + { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: "22222222-2222-4222-8222-222222222222", credential: token }, + { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch, credential: token, tenantId: "attacker" }, ]) assert.deepEqual(await request(socketPath, payload), { error: "invalid bootstrap request" }); assert.deepEqual(calls, []); }); @@ -77,7 +77,7 @@ test("local provider channel returns only a boot-bound VM certificate and public ); const payload = { type: "issue-vm-certificate", bootEpoch: epoch, vmIdentity: "fly:job-1", csr }; assert.deepEqual(await request(socketPath, payload), { error: "certificate provisioning failed" }); - await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", bootEpoch: epoch, credential: token }); + await request(socketPath, { type: "register-boot", controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", bootEpoch: epoch, credential: token }); assert.deepEqual(await request(socketPath, payload), { issued: true, bootEpoch: epoch, vmIdentity: "fly:job-1", certificate: "cert-for:fly:job-1", trustBundle: "public-ca", }); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index e25342cf..449cf991 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -9,7 +9,7 @@ const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R" test("boot registration uses a one-call control credential, HTTPS, exact path and fresh process epoch", async () => { let request; - const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async (url, init) => { + const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", token, bootEpoch, fetchImpl: async (url, init) => { request = { url: String(url), init }; return Response.json({ registered: true, tenantId: "11111111-1111-4111-8111-111111111111", proxyIdentity: "proxy-fixture", bootEpoch, controlCredential: `fzc_${"c".repeat(43)}` }); } }); @@ -17,7 +17,7 @@ test("boot registration uses a one-call control credential, HTTPS, exact path an assert.equal(request.url, "https://factorize.example/internal/credential-proxy/register-boot"); assert.equal(request.init.headers.Authorization, `Bearer ${token}`); assert.equal(request.init.redirect, "error"); - assert.deepEqual(JSON.parse(request.init.body), { bootEpoch }); + assert.deepEqual(JSON.parse(request.init.body), { bootEpoch, browserOrigin: "https://proxy.example" }); }); test("proxy snapshot reader requests only the bound run revision and validates encrypted records", async () => { @@ -84,9 +84,9 @@ test("proxy revocation monitor applies durable run revocations and tolerates an }); test("boot registration rejects unsafe origins, invalid credentials and mismatched response bindings", async () => { - await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch }), /HTTPS origin/); - await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch }), /credential/); - await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, fetchImpl: async () => Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch: "33333333-3333-4333-8333-333333333333" }) }), /invalid proxy boot registration response/); + await assert.rejects(registerProxyBoot({ controlOrigin: "http://factorize.example", token, bootEpoch, browserOrigin: "https://proxy.example" }), /HTTPS origin/); + await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token: "fzt_run_token", bootEpoch, browserOrigin: "https://proxy.example" }), /credential/); + await assert.rejects(registerProxyBoot({ controlOrigin: "https://factorize.example", token, bootEpoch, browserOrigin: "https://proxy.example", fetchImpl: async () => Response.json({ registered: true, proxyIdentity: "proxy-fixture", bootEpoch: "33333333-3333-4333-8333-333333333333" }) }), /invalid proxy boot registration response/); }); test("proxy control client requests and consumes a one-use enrollment over authenticated HTTPS", async () => { diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index 62e4f652..cb5d0509 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -1806,6 +1806,82 @@ components: schema: $ref: "#/components/schemas/Error" paths: + /api/v1/model-accounts/{accountId}/browser-capability: + post: + summary: Issue a short-lived browser capability for a paired model account + description: |- + Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens. + + Requires the integrations:write scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "200": + description: Paired proxy endpoint and one-use capability + content: + application/json: + schema: + type: object + properties: + endpoint: + type: string + format: uri + capability: + type: string + required: + - endpoint + - capability + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + "415": + $ref: "#/components/responses/Error415" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: post_api_v1_model_accounts_accountId_browser_capability + tags: + - model-accounts + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: integrations:write + x-owner-session-required: true + parameters: + - name: accountId + in: path + required: true + schema: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + action: + type: string + enum: + - authorize + - status + - cancel + required: + - action + additionalProperties: false /api/v1/unlock-key/verification: get: summary: Get the current encrypted unlock-key verification record From 8a6853446d496ee83971949f58d25ba10a8faf3d Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:03:23 +0000 Subject: [PATCH 087/102] GEN-2824 connect model account browser authorization --- packages/api-client/src/schema.d.ts | 60 +++++++++++++ packages/api/src/api-contract.ts | 6 ++ packages/api/src/flow-service.ts | 5 ++ packages/api/test/api-contract.test.ts | 7 ++ .../src/features/settings/integrations.tsx | 3 +- .../src/features/settings/model-accounts.tsx | 90 +++++++++++++++++++ packages/app/test/settings.browser.test.mjs | 29 ++++++ .../credential-proxy/src/browser-control.mjs | 23 ++++- .../src/codex-auth-helper.mjs | 7 +- .../src/model-account-lifecycle.mjs | 2 +- .../test/browser-control.test.mjs | 6 +- .../test/codex-auth-helper.test.mjs | 3 + packages/docs/openapi.yaml | 69 ++++++++++++++ 13 files changed, 300 insertions(+), 10 deletions(-) create mode 100644 packages/app/src/features/settings/model-accounts.tsx diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index e7c92965..56992fde 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -1,5 +1,29 @@ // Generated from packages/docs/openapi.yaml. Do not edit. export interface paths { + "/api/v1/model-accounts": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List model accounts available to the owner + * @description Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned. + * + * Requires the integrations:read scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + get: operations["get_api_v1_model_accounts"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/model-accounts/{accountId}/browser-capability": { parameters: { query?: never; @@ -2707,6 +2731,42 @@ export interface components { } export type $defs = Record; export interface operations { + get_api_v1_model_accounts: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Model account metadata */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + accounts: { + /** Format: uuid */ + id: string; + name: string; + description: string | null; + revision: number; + }[]; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 409: components["responses"]["Error409"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; post_api_v1_model_accounts_accountId_browser_capability: { parameters: { query?: never; diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index aae9e10a..36474829 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -131,6 +131,7 @@ const bearerIntegrationCreate = z.object({ const bearerIntegrationUpdate = bearerIntegrationCreate.omit({ id: true }); const bearerIntegrationDelete = z.object({ expectedRevision: z.number().int().nonnegative(), idempotencyKey: z.uuid() }).strict(); const bearerIntegrationList = z.object({ revision: z.number().int().nonnegative(), integrations: z.array(bearerIntegrationMetadata) }); +const modelAccountMetadata = z.object({ id: z.uuid(), name: z.string(), description: z.string().nullable(), revision: z.number().int().nonnegative() }); const bearerIntegrationWriteResult = z.object({ revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const bearerIntegrationDeleteResult = z.object({ id: z.uuid(), deleted: z.literal(true), revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const unlockKeyVerification = z.object({ ownerId: z.string(), verification: z.object({ generationId: z.uuid(), encryptedVerification: z.string() }).strict().nullable() }).strict(); @@ -214,6 +215,11 @@ function authRoute(method: string, path: string, action: AuthAction, summary: st } export const API_ROUTES: RouteContract[] = [ + route("GET", "/api/v1/model-accounts", "integrations:read", { + summary: "List model accounts available to the owner", + description: "Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned.", + responses: { "200": { description: "Model account metadata", content: { "application/json": { schema: jsonSchema(z.object({ accounts: z.array(modelAccountMetadata) }), "output") } } } }, + }, service => service.listModelAccounts(), { ownerSession: true }), route("POST", "/api/v1/model-accounts/{accountId}/browser-capability", "integrations:write", { summary: "Issue a short-lived browser capability for a paired model account", description: "Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens.", diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index 865bda9f..f8acb68e 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -589,6 +589,11 @@ export class ApiService { }); return { endpoint: `${pairing.browserOrigin}/model-accounts/${accountId}/${action}`, capability }; } + async listModelAccounts() { + await this.authorizeOwnerSession("integrations:read"); + const accounts = await this.encryptedIntegrations().records("model"); + return { accounts: accounts.map(({ id, name, description, revision }) => ({ id, name, description, revision })) }; + } async removeTailIntegration(id: string) { await this.authorize("integrations:write"); if (!await this.connections().delete(`cloudflare-tail:${id}`)) throw new ServiceError(404, "not_found", "Integration not found"); return { id, deleted: true }; } async linearProjects() { await this.authorize("integrations:read"); return publicValue(z.array(namedOption), await this.providers().linearProjects()); } async linearOptions() { await this.authorize("integrations:read"); return publicValue(providerOptions, await this.providers().linearOptions()); } diff --git a/packages/api/test/api-contract.test.ts b/packages/api/test/api-contract.test.ts index 7a803c25..bebc6035 100644 --- a/packages/api/test/api-contract.test.ts +++ b/packages/api/test/api-contract.test.ts @@ -73,6 +73,13 @@ describe("versioned API contract", () => { expect(route?.body?.safeParse({ action: "authorize", accountId: "attacker" }).success).toBe(false); expect(openapi.paths["/api/v1/model-accounts/{accountId}/browser-capability"].post.requestBody).toBeDefined(); }); + it("lists only safe model account metadata for the owner browser", () => { + const route = API_ROUTES.find(item => item.method === "GET" && item.path === "/api/v1/model-accounts"); + expect(route?.scope).toBe("integrations:read"); + expect(route?.ownerSession).toBe(true); + expect(route?.documentation.responses["200"].content["application/json"].schema.properties.accounts.items.properties.encryptedPayload).toBeUndefined(); + expect(openapi.paths["/api/v1/model-accounts"].get).toBeDefined(); + }); it("accepts code jobs without agent prompts or models and rejects missing scripts", () => { const code = { name: "Export", slug: "export", executionType: "code", script: "#!/bin/sh\nprintf '{\"status\":\"succeeded\"}' > \"$FACTORIZE_RESULT_PATH\"", executionConfigId: "exe", triggers: [] }; expect(jobInputSchema.safeParse(code).success).toBe(true); diff --git a/packages/app/src/features/settings/integrations.tsx b/packages/app/src/features/settings/integrations.tsx index dfa08fbc..f8c72b9a 100644 --- a/packages/app/src/features/settings/integrations.tsx +++ b/packages/app/src/features/settings/integrations.tsx @@ -10,6 +10,7 @@ import { IntegrationForm } from "./integration-form"; import { messageOf } from "../auth/session"; import { UnlockKeySetup } from "./unlock-key-setup"; import { BearerIntegrations } from "./bearer-integrations"; +import { ModelAccounts } from "./model-accounts"; const linkClass = "inline-flex min-h-10 items-center rounded-lg border border-stone-200 px-3 py-2 text-sm font-semibold hover:bg-stone-100 dark:border-slate-700 dark:hover:bg-slate-800"; @@ -53,7 +54,7 @@ export function Integrations() { {status.data?.cloudflareTail.installations.map(c =>

Cloudflare Tail · {c.name}

Referenced by {c.referencedJobCount} {c.referencedJobCount === 1 ? "job" : "jobs"}

Connected{buttons("cloudflareTail", c.integrationId)}
{manage(`tail:${c.integrationId}`, )}
)} } - +

Available integrations

Connect a provider to add it to your workspace. Connected providers remain available for additional connections.

setOpenSetup(open ? "linear-new" : null)}>{status.data?.linear ? "Add another Linear account" : "Connect Linear"} setOpenSetup(open ? "clickup-new" : null)}>{status.data?.clickup ? "Add another ClickUp workspace" : "Connect ClickUp"} diff --git a/packages/app/src/features/settings/model-accounts.tsx b/packages/app/src/features/settings/model-accounts.tsx new file mode 100644 index 00000000..b94e0806 --- /dev/null +++ b/packages/app/src/features/settings/model-accounts.tsx @@ -0,0 +1,90 @@ +import { useEffect, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { api } from "factorize-api-client"; +import { Button, Card } from "../../shared/ui"; +import { messageOf } from "../auth/session"; + +type DeviceState = { state: string; verificationUrl?: string; userCode?: string }; +type ModelAccount = { id: string; name: string; description: string | null; revision: number }; +const accountsQuery = { queryKey: ["settings", "model-accounts"], queryFn: async ({ signal }: { signal: AbortSignal }) => { + const { data, error } = await api.GET("/api/v1/model-accounts", { signal }); + if (!data || error) throw new Error(messageOf(error)); + return data.accounts; +} }; + +async function proxyAction(accountId: string, action: "authorize" | "status" | "cancel"): Promise { + const capability = await api.POST("/api/v1/model-accounts/{accountId}/browser-capability", { + params: { path: { accountId } }, body: { action }, + }); + if (!capability.data || capability.error) throw new Error(messageOf(capability.error)); + const response = await fetch(capability.data.endpoint, { + method: "POST", mode: "cors", credentials: "omit", cache: "no-store", + headers: { Authorization: `Bearer ${capability.data.capability}`, "Content-Type": "application/json" }, + body: "{}", + }); + let result: DeviceState; + try { result = await response.json() as DeviceState; } + catch { throw new Error("The credential proxy returned an invalid response."); } + if (!response.ok) { + const messages: Record = { 423: "Unlock the credential proxy before connecting a model account.", 503: "The credential proxy is unavailable. Try again shortly." }; + throw new Error(messages[response.status] ?? "The credential proxy could not complete this request."); + } + return result; +} + +export function ModelAccounts() { + const accounts = useQuery(accountsQuery); + const [active, setActive] = useState(null); + const [states, setStates] = useState>({}); + const [error, setError] = useState(""); + + useEffect(() => { + if (!active) return; + let stopped = false; + const poll = async () => { + try { + const next = await proxyAction(active, "status"); + if (stopped) return; + setStates(current => ({ ...current, [active]: next })); + if (next.state !== "pending") setActive(null); + } catch (reason) { + if (!stopped) setError(reason instanceof Error ? reason.message : "Could not check model account authorization."); + } + }; + const timer = window.setInterval(poll, 2000); + return () => { stopped = true; window.clearInterval(timer); }; + }, [active]); + + const run = async (account: ModelAccount, action: "authorize" | "cancel") => { + setError(""); + try { + const next = await proxyAction(account.id, action); + setStates(current => ({ ...current, [account.id]: next })); + setActive(action === "authorize" && next.state === "pending" ? account.id : null); + } catch (reason) { + setError(reason instanceof Error ? reason.message : "Could not connect the model account."); + } + }; + + return
+

ChatGPT model accounts

Connect each account through Codex device authorization. Credentials stay in the credential proxy.

+ {accounts.isPending &&

Loading model accounts…

} + {accounts.error &&

{accounts.error.message}

} + {error &&

{error}

} + {!accounts.isPending && accounts.data?.length === 0 &&

No model accounts are configured yet.

} + {accounts.data?.map(account => { + const state = states[account.id] ?? { state: "idle" }; + return +

{account.name}

{account.description &&

{account.description}

} + {state.state === "pending" &&

Waiting for device authorization.

{state.userCode &&

Code: {state.userCode}

}{state.verificationUrl && OpenAI device authorization}
} + {state.state === "authorized" &&

ChatGPT account connected.

} + {state.state === "authorized-unsaved" &&

ChatGPT account connected, but its encrypted token cache could not be saved. Keep the proxy running and retry after checking its connection.

} + {state.state === "reauthentication-required" &&

Sign in again to reconnect this account.

} + {state.state === "failed" &&

Device authorization failed. You can try again.

} + {state.state === "canceled" &&

Device authorization canceled.

} +
+ {state.state === "pending" ? : } + ; + })} +
; +} diff --git a/packages/app/test/settings.browser.test.mjs b/packages/app/test/settings.browser.test.mjs index bc3368e5..83db10d0 100644 --- a/packages/app/test/settings.browser.test.mjs +++ b/packages/app/test/settings.browser.test.mjs @@ -18,6 +18,35 @@ test("integration setup tests credentials, submits typed fields, clears secrets await form.getByLabel("exe.dev account token").fill("private-input"); await form.getByRole("button", { name: "Test credentials" }).click(); await form.getByText("✓ Credentials verified.", { exact: true }).waitFor(); await form.getByLabel("VM tags").selectOption(["dev"]); await form.getByRole("button", { name: "Save integration", exact: true }).click(); await form.getByText("Integration saved", { exact: true }).waitFor(); assert.deepEqual(body.tags, ["dev"]); assert.equal(body.agentKind, "codex"); assert.equal(await form.getByLabel("exe.dev account token").inputValue(), ""); assert.equal(await page.evaluate(() => JSON.stringify(localStorage).includes("private-input")), false); await page.getByText("dev", { exact: true }).waitFor(); await context.close(); }); +test("model account settings issue short-lived capabilities and display the bounded device challenge", async () => { + const { context, page } = await setup(); + const accountId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", proxyOrigin = "https://proxy.example"; + await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); + await page.route("**/api/v1/integrations", route => route.fulfill({ json: { linear: null, clickup: null, exe: null, exeConnections: [], ampConnections: [], cloudflareTail: { count: 0, installations: [] } } })); + await page.route("**/api/v1/model-accounts", async route => { + if (route.request().method() === "GET") return route.fulfill({ json: { accounts: [{ id: accountId, name: "Work account", description: null, revision: 0 }] } }); + const action = route.request().postDataJSON().action; + return route.fulfill({ json: { endpoint: `${proxyOrigin}/model-accounts/${accountId}/${action}`, capability: `one-use-${action}` } }); + }); + const proxyCalls = []; + await page.route(`${proxyOrigin}/model-accounts/**`, async route => { + const request = route.request(); + if (request.method() === "OPTIONS") return route.fulfill({ status: 204, headers: { "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Methods": "POST", "Access-Control-Allow-Headers": "Authorization, Content-Type" } }); + proxyCalls.push({ path: new URL(request.url()).pathname, authorization: request.headers().authorization, body: request.postData() }); + const response = request.url().endsWith("/authorize") ? { state: "pending", verificationUrl: "https://auth.openai.com/codex/device", userCode: "ABCD-1234" } : { state: "canceled" }; + return route.fulfill({ json: response, headers: { "Access-Control-Allow-Origin": origin, "Cache-Control": "no-store" } }); + }); + await page.goto(origin + "/settings/integrations"); + const models = page.getByRole("region", { name: "ChatGPT model accounts" }); + await models.getByRole("button", { name: "Connect ChatGPT" }).click(); + await models.getByText("ABCD-1234", { exact: false }).waitFor(); + await models.getByRole("link", { name: "OpenAI device authorization" }).waitFor(); + await models.getByRole("button", { name: "Cancel authorization" }).click(); + await models.getByText("Device authorization canceled.", { exact: true }).waitFor(); + assert.deepEqual(proxyCalls.map(call => call.authorization), ["Bearer one-use-authorize", "Bearer one-use-cancel"]); + assert.deepEqual(proxyCalls.map(call => call.body), ["{}", "{}"]); + await context.close(); +}); test("unlock-key setup requires acknowledgement, stores ciphertext only, verifies entered keys and survives refresh without a saved key", async () => { const { context, page } = await setup(); await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); diff --git a/packages/credential-proxy/src/browser-control.mjs b/packages/credential-proxy/src/browser-control.mjs index e87e1218..930498b8 100644 --- a/packages/credential-proxy/src/browser-control.mjs +++ b/packages/credential-proxy/src/browser-control.mjs @@ -32,6 +32,18 @@ function json(response, status, body, origin) { response.writeHead(status, headers).end(JSON.stringify(body)); } +function publicAuthorization(value) { + if (!value || typeof value !== "object") return { state: "idle" }; + const result = { state: value.state }; + const challenge = value.challenge; + if (challenge?.verificationUrl === "https://auth.openai.com/codex/device" + && typeof challenge.userCode === "string" && (/^[A-Z0-9]{4,6}-[A-Z0-9]{4,6}$/u.test(challenge.userCode) || /^[A-Z0-9]{8,12}$/u.test(challenge.userCode))) { + result.verificationUrl = challenge.verificationUrl; + result.userCode = challenge.userCode; + } + return result; +} + async function readJson(request) { const chunks = []; let size = 0; @@ -119,16 +131,19 @@ export async function startBrowserControl({ if (operation === "authorize") { if (runtime.state !== "unlocked") return json(response, 423, { error: "proxy_locked" }, appOrigin); const login = await lifecycle.deviceLogin(accountId); - const state = { state: "pending" }; + const state = { state: "pending", challenge: login.challenge }; authorizations.set(accountId, state); Promise.resolve(login.completion).then(result => { - if (authorizations.get(accountId) === state) authorizations.set(accountId, { ...result }); + if (authorizations.get(accountId) === state) { + const next = result?.state === "saved" ? "authorized" : result?.state === "unsaved" ? "authorized-unsaved" : result?.state ?? "reauthentication-required"; + authorizations.set(accountId, { state: next }); + } }, () => { if (authorizations.get(accountId) === state) authorizations.set(accountId, { state: "failed" }); }); - return json(response, 202, state, appOrigin); + return json(response, 202, publicAuthorization(state), appOrigin); } - if (operation === "status") return json(response, 200, authorizations.get(accountId) ?? { state: "idle" }, appOrigin); + if (operation === "status") return json(response, 200, publicAuthorization(authorizations.get(accountId)), appOrigin); const result = await lifecycle.cancel(accountId); authorizations.set(accountId, { state: "canceled" }); return json(response, 200, { state: result?.state === "idle" ? "idle" : "canceled" }, appOrigin); diff --git a/packages/credential-proxy/src/codex-auth-helper.mjs b/packages/credential-proxy/src/codex-auth-helper.mjs index 199d906f..8f888875 100644 --- a/packages/credential-proxy/src/codex-auth-helper.mjs +++ b/packages/credential-proxy/src/codex-auth-helper.mjs @@ -99,6 +99,7 @@ export function createCodexAuthHelper({ if (logins.has(accountId)) throw new Error("device authorization is already running for this account"); const home = await prepareHome(accountId); let child; + const challenge = {}; const completion = new Promise((resolveRun, rejectRun) => { child = spawnProcess(codexPath, ["login", "--device-auth"], { env: { PATH: process.env.PATH ?? "", CODEX_HOME: home, HOME: home }, @@ -109,6 +110,10 @@ export function createCodexAuthHelper({ timer.unref?.(); child.stdout.on("data", chunk => { stdout = (stdout + chunk.toString("utf8")).slice(-16_384); + const url = stdout.match(/https:\/\/auth\.openai\.com\/codex\/device(?:\b|\/)/u)?.[0]; + const code = stdout.match(/\b(?:user\s+)?code\s*:\s*([A-Z0-9]{4,6}-[A-Z0-9]{4,6}|[A-Z0-9]{8,12})\b/iu)?.[1]; + if (url) challenge.verificationUrl = "https://auth.openai.com/codex/device"; + if (code) challenge.userCode = code.toUpperCase(); }); child.stderr.on("data", chunk => { // Retain no provider error text, which could include device details. @@ -124,7 +129,7 @@ export function createCodexAuthHelper({ const entry = { child, completion }; logins.set(accountId, entry); completion.finally(() => { if (logins.get(accountId) === entry) logins.delete(accountId); }).catch(() => {}); - return { state: "pending", completion }; + return { state: "pending", challenge, completion }; } if (request.method === "cancel") { const login = logins.get(accountId); diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index 8e13d05f..fc6a1308 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -120,7 +120,7 @@ export function createModelAccountLifecycle({ if (outcome?.state !== "authorized") return { state: outcome?.state ?? "reauthentication-required" }; return persist(id, generation); }); - return { state: "pending", completion }; + return { state: "pending", challenge: result.challenge, completion }; }); } diff --git a/packages/credential-proxy/test/browser-control.test.mjs b/packages/credential-proxy/test/browser-control.test.mjs index 0c7af413..918c6995 100644 --- a/packages/credential-proxy/test/browser-control.test.mjs +++ b/packages/credential-proxy/test/browser-control.test.mjs @@ -22,7 +22,7 @@ test("private browser control requires exact origin and one-use owner/proxy/acti const calls = []; const runtime = { state: "unlocked" }; const lifecycle = { - async deviceLogin(id) { calls.push(["authorize", id]); return { state: "pending", completion: Promise.resolve({ state: "saved" }) }; }, + async deviceLogin(id) { calls.push(["authorize", id]); return { state: "pending", challenge: { verificationUrl: "https://auth.openai.com/codex/device", userCode: "ABCD-1234" }, completion: Promise.resolve({ state: "saved" }) }; }, async cancel(id) { calls.push(["cancel", id]); return { state: "canceled" }; }, }; const server = await startBrowserControl({ runtime, lifecycle, publicKey, proxyId, ownerId, generation, appOrigin, port: 0, host: "127.0.0.1", now: () => now }); @@ -44,13 +44,13 @@ test("private browser control requires exact origin and one-use owner/proxy/acti const authToken = capability({ action: "model-account:authorize" }); const started = await post("model-account:authorize", authToken); assert.equal(started.status, 202); - assert.deepEqual(await started.json(), { state: "pending" }); + assert.deepEqual(await started.json(), { state: "pending", verificationUrl: "https://auth.openai.com/codex/device", userCode: "ABCD-1234" }); assert.deepEqual(calls[0], ["authorize", accountId]); const replay = await post("model-account:authorize", authToken); assert.equal(replay.status, 401); const status = await post("model-account:status", capability({ action: "model-account:status" })); - assert.deepEqual(await status.json(), { state: "saved" }); + assert.deepEqual(await status.json(), { state: "authorized" }); const wrongAudience = await post("model-account:cancel", capability({ action: "model-account:cancel", aud: "fly:other:machine" })); assert.equal(wrongAudience.status, 401); const expired = await post("model-account:cancel", capability({ action: "model-account:cancel", exp: now - 1 })); diff --git a/packages/credential-proxy/test/codex-auth-helper.test.mjs b/packages/credential-proxy/test/codex-auth-helper.test.mjs index 1b386c1d..9528f76f 100644 --- a/packages/credential-proxy/test/codex-auth-helper.test.mjs +++ b/packages/credential-proxy/test/codex-auth-helper.test.mjs @@ -63,8 +63,11 @@ test("device authorization runs as a separate cancellable login process", async const helper = createCodexAuthHelper({ accountHomeRoot: directory, spawnProcess: fake.spawnProcess }); const result = await helper.invoke({ method: "device-login", accountId: accountA }); assert.equal(result.state, "pending"); + assert.deepEqual(result.challenge, {}); assert.deepEqual(fake.calls.map(call => call.args), [["--version"], ["login", "--device-auth"]]); assert.equal(fake.calls[1].options.env.CODEX_HOME, join(directory, accountA)); + fake.children[1].stdout.write("Open this URL in your browser: https://auth.openai.com/codex/device\nEnter user code: abcd-1234\n"); + assert.deepEqual(result.challenge, { verificationUrl: "https://auth.openai.com/codex/device", userCode: "ABCD-1234" }); assert.deepEqual(await helper.invoke({ method: "cancel", accountId: accountA }), { state: "canceled" }); await assert.rejects(result.completion, /stopped/); assert.deepEqual(await helper.invoke({ method: "cancel", accountId: accountA }), { state: "idle" }); diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index cb5d0509..6089764f 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -1806,6 +1806,75 @@ components: schema: $ref: "#/components/schemas/Error" paths: + /api/v1/model-accounts: + get: + summary: List model accounts available to the owner + description: |- + Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned. + + Requires the integrations:read scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "200": + description: Model account metadata + content: + application/json: + schema: + type: object + properties: + accounts: + type: array + items: + type: object + properties: + id: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + name: + type: string + description: + type: + - string + - "null" + revision: + type: integer + minimum: 0 + maximum: 9007199254740991 + required: + - id + - name + - description + - revision + additionalProperties: false + required: + - accounts + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: get_api_v1_model_accounts + tags: + - model-accounts + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: integrations:read + x-owner-session-required: true /api/v1/model-accounts/{accountId}/browser-capability: post: summary: Issue a short-lived browser capability for a paired model account From 6e81b33bcccaaa31ff43b75795c5e9315d829f85 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:07:01 +0000 Subject: [PATCH 088/102] GEN-2824 cover capability endpoint in browser test --- packages/app/test/settings.browser.test.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/app/test/settings.browser.test.mjs b/packages/app/test/settings.browser.test.mjs index 83db10d0..81903180 100644 --- a/packages/app/test/settings.browser.test.mjs +++ b/packages/app/test/settings.browser.test.mjs @@ -28,6 +28,10 @@ test("model account settings issue short-lived capabilities and display the boun const action = route.request().postDataJSON().action; return route.fulfill({ json: { endpoint: `${proxyOrigin}/model-accounts/${accountId}/${action}`, capability: `one-use-${action}` } }); }); + await page.route("**/api/v1/model-accounts/*/browser-capability", async route => { + const action = route.request().postDataJSON().action; + return route.fulfill({ json: { endpoint: `${proxyOrigin}/model-accounts/${accountId}/${action}`, capability: `one-use-${action}` } }); + }); const proxyCalls = []; await page.route(`${proxyOrigin}/model-accounts/**`, async route => { const request = route.request(); From 834e2b326a69ee9e16383fac3980651f44997d92 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:29:09 +0000 Subject: [PATCH 089/102] GEN-2824 add model account create and delete --- packages/api-client/src/schema.d.ts | 130 +++++++++++- packages/api/src/api-contract.ts | 21 +- packages/api/src/flow-service.ts | 46 ++++- .../encrypted-integration-repository.ts | 27 +++ packages/api/test/api-contract.test.ts | 9 + ...encrypted-integration-api.postgres.test.ts | 37 ++++ .../src/features/settings/model-accounts.tsx | 70 ++++++- packages/app/src/lib/unlock-crypto.ts | 16 ++ packages/app/test/settings.browser.test.mjs | 66 +++++- packages/app/test/unlock-crypto.test.mjs | 21 +- packages/docs/openapi.yaml | 190 +++++++++++++++++- 11 files changed, 615 insertions(+), 18 deletions(-) diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index 56992fde..1e4bd5e7 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -9,7 +9,7 @@ export interface paths { }; /** * List model accounts available to the owner - * @description Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned. + * @description Requires an interactive owner browser session. Returns account metadata and the collection revision only; encrypted account caches and credentials are never returned. * * Requires the integrations:read scope. * @@ -17,13 +17,45 @@ export interface paths { */ get: operations["get_api_v1_model_accounts"]; put?: never; - post?: never; + /** + * Create a model account + * @description Requires an interactive owner browser session and the current collection revision. Accepts only user-encrypted initial account state; credentials are established directly with the credential proxy. + * + * Requires the integrations:write scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + post: operations["post_api_v1_model_accounts"]; delete?: never; options?: never; head?: never; patch?: never; trace?: never; }; + "/api/v1/model-accounts/{accountId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + /** + * Delete a model account + * @description Requires an interactive owner browser session and the current collection revision. Deletion is recorded as a tombstone so future snapshots and unlock reconciliation cannot restore the account. + * + * Requires the integrations:write scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + delete: operations["delete_api_v1_model_accounts_accountId"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/model-accounts/{accountId}/browser-capability": { parameters: { query?: never; @@ -2740,13 +2772,14 @@ export interface operations { }; requestBody?: never; responses: { - /** @description Model account metadata */ + /** @description Model account metadata and collection revision */ 200: { headers: { [name: string]: unknown; }; content: { "application/json": { + revision: number; accounts: { /** Format: uuid */ id: string; @@ -2767,6 +2800,97 @@ export interface operations { 503: components["responses"]["Error503"]; }; }; + post_api_v1_model_accounts: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + /** Format: uuid */ + id: string; + expectedRevision: number; + /** Format: uuid */ + idempotencyKey: string; + name: string; + description: string | null; + encryptedPayload: string; + }; + }; + }; + responses: { + /** @description Created model account */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + revision: number; + duplicate: boolean; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 409: components["responses"]["Error409"]; + 415: components["responses"]["Error415"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; + delete_api_v1_model_accounts_accountId: { + parameters: { + query?: never; + header?: never; + path: { + accountId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + expectedRevision: number; + /** Format: uuid */ + idempotencyKey: string; + }; + }; + }; + responses: { + /** @description Deleted model account */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** Format: uuid */ + id: string; + /** @constant */ + deleted: true; + revision: number; + duplicate: boolean; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 409: components["responses"]["Error409"]; + 415: components["responses"]["Error415"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; post_api_v1_model_accounts_accountId_browser_capability: { parameters: { query?: never; diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index 36474829..6ea38cdc 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -132,6 +132,13 @@ const bearerIntegrationUpdate = bearerIntegrationCreate.omit({ id: true }); const bearerIntegrationDelete = z.object({ expectedRevision: z.number().int().nonnegative(), idempotencyKey: z.uuid() }).strict(); const bearerIntegrationList = z.object({ revision: z.number().int().nonnegative(), integrations: z.array(bearerIntegrationMetadata) }); const modelAccountMetadata = z.object({ id: z.uuid(), name: z.string(), description: z.string().nullable(), revision: z.number().int().nonnegative() }); +const modelAccountCreate = z.object({ + id: z.uuid(), expectedRevision: z.number().int().nonnegative(), idempotencyKey: z.uuid(), + name: z.string().trim().min(1).max(120), description: z.string().max(500).nullable(), + encryptedPayload: z.string().min(1).max(65_536), +}).strict(); +const modelAccountDelete = z.object({ expectedRevision: z.number().int().nonnegative(), idempotencyKey: z.uuid() }).strict(); +const modelAccountList = z.object({ revision: z.number().int().nonnegative(), accounts: z.array(modelAccountMetadata) }); const bearerIntegrationWriteResult = z.object({ revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const bearerIntegrationDeleteResult = z.object({ id: z.uuid(), deleted: z.literal(true), revision: z.number().int().nonnegative(), duplicate: z.boolean() }); const unlockKeyVerification = z.object({ ownerId: z.string(), verification: z.object({ generationId: z.uuid(), encryptedVerification: z.string() }).strict().nullable() }).strict(); @@ -217,9 +224,19 @@ function authRoute(method: string, path: string, action: AuthAction, summary: st export const API_ROUTES: RouteContract[] = [ route("GET", "/api/v1/model-accounts", "integrations:read", { summary: "List model accounts available to the owner", - description: "Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned.", - responses: { "200": { description: "Model account metadata", content: { "application/json": { schema: jsonSchema(z.object({ accounts: z.array(modelAccountMetadata) }), "output") } } } }, + description: "Requires an interactive owner browser session. Returns account metadata and the collection revision only; encrypted account caches and credentials are never returned.", + responses: { "200": { description: "Model account metadata and collection revision", content: { "application/json": { schema: jsonSchema(modelAccountList, "output") } } } }, }, service => service.listModelAccounts(), { ownerSession: true }), + route("POST", "/api/v1/model-accounts", "integrations:write", { + summary: "Create a model account", + description: "Requires an interactive owner browser session and the current collection revision. Accepts only user-encrypted initial account state; credentials are established directly with the credential proxy.", + responses: { "201": { description: "Created model account", content: { "application/json": { schema: jsonSchema(bearerIntegrationWriteResult, "output") } } } }, + }, (service, { body }) => service.createModelAccount(body), { body: modelAccountCreate, maxBodyBytes: 70_000, ownerSession: true }), + route("DELETE", "/api/v1/model-accounts/{accountId}", "integrations:write", { + summary: "Delete a model account", + description: "Requires an interactive owner browser session and the current collection revision. Deletion is recorded as a tombstone so future snapshots and unlock reconciliation cannot restore the account.", + responses: { "200": { description: "Deleted model account", content: { "application/json": { schema: jsonSchema(bearerIntegrationDeleteResult, "output") } } } }, + }, (service, { params, body }) => service.deleteModelAccount(params.accountId, body), { body: modelAccountDelete, maxBodyBytes: 4096, ownerSession: true, parameters: z.object({ accountId: z.uuid() }) }), route("POST", "/api/v1/model-accounts/{accountId}/browser-capability", "integrations:write", { summary: "Issue a short-lived browser capability for a paired model account", description: "Requires an interactive owner browser session. Returns a single-use, 60-second capability scoped to the account, action, proxy boot and exact application origin. The browser sends it directly to the paired proxy; Factorize never receives model-account tokens.", diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index f8acb68e..9275b638 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -591,8 +591,50 @@ export class ApiService { } async listModelAccounts() { await this.authorizeOwnerSession("integrations:read"); - const accounts = await this.encryptedIntegrations().records("model"); - return { accounts: accounts.map(({ id, name, description, revision }) => ({ id, name, description, revision })) }; + const repository = this.encryptedIntegrations(); + const [revision, accounts] = await Promise.all([repository.revision(), repository.records("model")]); + return { revision, accounts: accounts.map(({ id, name, description, revision }) => ({ id, name, description, revision })) }; + } + private async modelAccountWrite(input: { id: string; expectedRevision: number; idempotencyKey: string; name: string; description: string | null; encryptedPayload?: string; operation: "upsert" | "delete" }) { + try { return await this.encryptedIntegrations().write({ ...input, kind: "model", origin: null }); } + catch (error) { + if (error instanceof StaleIntegrationRevisionError) throw new ServiceError(409, "stale_revision", `The model account collection changed. Current revision is ${error.currentRevision}.`); + if (error instanceof IntegrationIdempotencyConflictError) throw new ServiceError(409, "conflict", "The idempotency key was already used for a different model account change."); + if (error instanceof IntegrationValidationError) throw new ServiceError(400, "invalid_request", error.message); + throw error; + } + } + async createModelAccount(input: { id: string; expectedRevision: number; idempotencyKey: string; name: string; description: string | null; encryptedPayload: string }) { + await this.authorizeOwnerSession("integrations:write"); + const repository = this.encryptedIntegrations(); + try { + const replay = await repository.replay({ ...input, kind: "model", origin: null, operation: "upsert" }); + if (replay) return replay; + } catch (error) { + if (error instanceof IntegrationIdempotencyConflictError) throw new ServiceError(409, "conflict", "The idempotency key was already used for a different model account change."); + if (error instanceof IntegrationValidationError) throw new ServiceError(400, "invalid_request", error.message); + throw error; + } + if ((await repository.records()).some(record => record.id === input.id)) throw new ServiceError(409, "conflict", "An integration with this id already exists."); + return this.modelAccountWrite({ ...input, operation: "upsert" }); + } + async deleteModelAccount(id: string, input: { expectedRevision: number; idempotencyKey: string }) { + await this.authorizeOwnerSession("integrations:write"); + const repository = this.encryptedIntegrations(); + const existing = (await repository.records("model")).find(record => record.id === id); + if (!existing) { + try { + const replay = await repository.replayDelete(id, "model", input.idempotencyKey); + if (replay) return { id, deleted: true as const, ...replay }; + } catch (error) { + if (error instanceof IntegrationIdempotencyConflictError) throw new ServiceError(409, "conflict", "The idempotency key was already used for a different model account change."); + if (error instanceof IntegrationValidationError) throw new ServiceError(400, "invalid_request", error.message); + throw error; + } + throw new ServiceError(404, "not_found", "Model account not found."); + } + const result = await this.modelAccountWrite({ id, name: existing.name, description: existing.description, ...input, operation: "delete" }); + return { id, deleted: true as const, ...result }; } async removeTailIntegration(id: string) { await this.authorize("integrations:write"); if (!await this.connections().delete(`cloudflare-tail:${id}`)) throw new ServiceError(404, "not_found", "Integration not found"); return { id, deleted: true }; } async linearProjects() { await this.authorize("integrations:read"); return publicValue(z.array(namedOption), await this.providers().linearProjects()); } diff --git a/packages/api/src/postgres/encrypted-integration-repository.ts b/packages/api/src/postgres/encrypted-integration-repository.ts index 3451086a..36b3a504 100644 --- a/packages/api/src/postgres/encrypted-integration-repository.ts +++ b/packages/api/src/postgres/encrypted-integration-repository.ts @@ -77,6 +77,33 @@ export class EncryptedIntegrationRepository { return rows.length ? safeRevision(rows[0]!.revision) : 0; } + /** Recognize an exact retry before collection-level create checks run. */ + async replay(input: IntegrationWrite): Promise<{ revision: number; duplicate: true } | null> { + validate(input); + const { rows } = await this.database.pool.query("SELECT revision,operation,record_id,kind,name,origin,description,encrypted_payload FROM app.integration_changes WHERE tenant_id=$1 AND idempotency_key=$2", [this.tenantId, input.idempotencyKey]); + const row = rows[0]; + if (!row) return null; + const sameCiphertext = row.encrypted_payload === null + ? input.operation === "delete" && input.encryptedPayload === undefined + : input.operation === "upsert" && typeof input.encryptedPayload === "string" && await decrypt(row.encrypted_payload, this.serverEncryptionKey) === input.encryptedPayload; + if (row.record_id !== input.id || row.kind !== input.kind || row.name !== input.name || row.origin !== input.origin + || row.description !== input.description || row.operation !== input.operation || !sameCiphertext) { + throw new IntegrationIdempotencyConflictError(); + } + return { revision: safeRevision(row.revision), duplicate: true }; + } + + /** Recognize a delete retry after its active record has already become a tombstone. */ + async replayDelete(id: string, kind: EncryptedIntegrationKind, idempotencyKey: string): Promise<{ revision: number; duplicate: true } | null> { + const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (!uuid.test(id) || !uuid.test(idempotencyKey)) throw new IntegrationValidationError("id and idempotencyKey must be UUIDs"); + const { rows } = await this.database.pool.query("SELECT revision,operation,record_id,kind,name,origin,description,encrypted_payload FROM app.integration_changes WHERE tenant_id=$1 AND idempotency_key=$2", [this.tenantId, idempotencyKey]); + const row = rows[0]; + if (!row) return null; + if (row.record_id !== id || row.kind !== kind || row.operation !== "delete" || row.encrypted_payload !== null) throw new IntegrationIdempotencyConflictError(); + return { revision: safeRevision(row.revision), duplicate: true }; + } + async write(input: IntegrationWrite): Promise<{ revision: number; duplicate: boolean }> { validate(input); const encryptedPayload = input.operation === "upsert" ? await encrypt(input.encryptedPayload!, this.serverEncryptionKey) : null; diff --git a/packages/api/test/api-contract.test.ts b/packages/api/test/api-contract.test.ts index bebc6035..abfbcb01 100644 --- a/packages/api/test/api-contract.test.ts +++ b/packages/api/test/api-contract.test.ts @@ -75,9 +75,18 @@ describe("versioned API contract", () => { }); it("lists only safe model account metadata for the owner browser", () => { const route = API_ROUTES.find(item => item.method === "GET" && item.path === "/api/v1/model-accounts"); + const create = API_ROUTES.find(item => item.method === "POST" && item.path === "/api/v1/model-accounts"); + const remove = API_ROUTES.find(item => item.method === "DELETE" && item.path === "/api/v1/model-accounts/{accountId}"); expect(route?.scope).toBe("integrations:read"); expect(route?.ownerSession).toBe(true); expect(route?.documentation.responses["200"].content["application/json"].schema.properties.accounts.items.properties.encryptedPayload).toBeUndefined(); + expect(create?.scope).toBe("integrations:write"); expect(create?.ownerSession).toBe(true); + expect(create?.body?.safeParse({ id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", expectedRevision: 0, idempotencyKey: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", name: "Account", description: null, encryptedPayload: "cipher" }).success).toBe(true); + expect(create?.body?.safeParse({ id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", expectedRevision: 0, idempotencyKey: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", name: "Account", description: null, encryptedPayload: "cipher", unlockKey: "must-not-be-accepted" }).success).toBe(false); + expect(remove?.scope).toBe("integrations:write"); expect(remove?.ownerSession).toBe(true); + expect(remove?.parameters.shape.accountId).toBeDefined(); + expect(openapi.paths["/api/v1/model-accounts"].post).toBeDefined(); + expect(openapi.paths["/api/v1/model-accounts/{accountId}"].delete).toBeDefined(); expect(openapi.paths["/api/v1/model-accounts"].get).toBeDefined(); }); it("accepts code jobs without agent prompts or models and rejects missing scripts", () => { diff --git a/packages/api/test/encrypted-integration-api.postgres.test.ts b/packages/api/test/encrypted-integration-api.postgres.test.ts index 7aa42ad5..4a5622bd 100644 --- a/packages/api/test/encrypted-integration-api.postgres.test.ts +++ b/packages/api/test/encrypted-integration-api.postgres.test.ts @@ -76,6 +76,43 @@ describe.skipIf(!databaseUrl)("browser-only encrypted bearer integration API (Po } finally { await f.cleanup(); } }); + it("creates and deletes model accounts through owner-only revision-checked writes", async () => { + const f = await loadingFixture(databaseUrl!); + try { + const path = f.env.APP_ORIGIN + "/api/v1/model-accounts", id = crypto.randomUUID(); + const request = (url: string, method = "GET", body?: unknown, cookie = f.cookie) => worker.fetch(new Request(url, { + method, headers: { Cookie: cookie, Origin: f.env.APP_ORIGIN, "Content-Type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }), f.env, { waitUntil() {} } as any); + const ciphertext = await userEncrypt("{}"), createBody = { + id, expectedRevision: 0, idempotencyKey: crypto.randomUUID(), name: "Work account", + description: "Personal subscription", encryptedPayload: ciphertext, + }; + const created = await request(path, "POST", createBody); + expect(created.status).toBe(201); + expect(await created.json()).toEqual({ revision: 1, duplicate: false }); + expect(await (await request(path, "POST", createBody)).json()).toEqual({ revision: 1, duplicate: true }); + + const listed = await request(path); + expect(listed.status).toBe(200); + expect(await listed.json()).toEqual({ revision: 1, accounts: [{ id, name: "Work account", description: "Personal subscription", revision: 1 }] }); + + const stale = await request(path + `/${id}`, "DELETE", { expectedRevision: 0, idempotencyKey: crypto.randomUUID() }); + expect(stale.status).toBe(409); + expect((await stale.json() as any).error.code).toBe("stale_revision"); + + const deleteBody = { expectedRevision: 1, idempotencyKey: crypto.randomUUID() }; + const deleted = await request(path + `/${id}`, "DELETE", deleteBody); + expect(deleted.status).toBe(200); + expect(await deleted.json()).toEqual({ id, deleted: true, revision: 2, duplicate: false }); + expect(await (await request(path + `/${id}`, "DELETE", deleteBody)).json()).toEqual({ id, deleted: true, revision: 2, duplicate: true }); + expect(await (await request(path)).json()).toEqual({ revision: 2, accounts: [] }); + + const agent = await protectedApiFetch(new Request(path, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ ...createBody, id: crypto.randomUUID() }) }), f.env, { tenantId: f.tenantId, userId: f.userId, sessionVersion: 0, scopes: ["integrations:write"], authMethod: "access_token" } as any, {} as any); + expect(agent.status).toBe(403); + } finally { await f.cleanup(); } + }); + it("requires a browser session and isolates records across tenants", async () => { const f = await loadingFixture(databaseUrl!); try { diff --git a/packages/app/src/features/settings/model-accounts.tsx b/packages/app/src/features/settings/model-accounts.tsx index b94e0806..3283878f 100644 --- a/packages/app/src/features/settings/model-accounts.tsx +++ b/packages/app/src/features/settings/model-accounts.tsx @@ -1,7 +1,9 @@ import { useEffect, useState } from "react"; -import { useQuery } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { api } from "factorize-api-client"; -import { Button, Card } from "../../shared/ui"; +import { Button, Card, Input, Label } from "../../shared/ui"; +import { encryptModelAccountCacheRecord, verifyUnlockKey } from "../../lib/unlock-crypto"; +import { getUnlockVerification } from "../../lib/unlock-key"; import { messageOf } from "../auth/session"; type DeviceState = { state: string; verificationUrl?: string; userCode?: string }; @@ -9,8 +11,9 @@ type ModelAccount = { id: string; name: string; description: string | null; revi const accountsQuery = { queryKey: ["settings", "model-accounts"], queryFn: async ({ signal }: { signal: AbortSignal }) => { const { data, error } = await api.GET("/api/v1/model-accounts", { signal }); if (!data || error) throw new Error(messageOf(error)); - return data.accounts; + return data; } }; +const verificationQuery = { queryKey: ["unlock-key-verification"], queryFn: ({ signal }: { signal: AbortSignal }) => getUnlockVerification(signal), retry: false as const }; async function proxyAction(accountId: string, action: "authorize" | "status" | "cancel"): Promise { const capability = await api.POST("/api/v1/model-accounts/{accountId}/browser-capability", { @@ -33,10 +36,53 @@ async function proxyAction(accountId: string, action: "authorize" | "status" | " } export function ModelAccounts() { - const accounts = useQuery(accountsQuery); + const cache = useQueryClient(), accounts = useQuery(accountsQuery), verification = useQuery(verificationQuery); const [active, setActive] = useState(null); const [states, setStates] = useState>({}); const [error, setError] = useState(""); + const [name, setName] = useState(""), [description, setDescription] = useState(""), [key, setKey] = useState(""); + + const create = useMutation({ retry: false, mutationFn: async () => { + const cleanName = name.trim(), cleanDescription = description.trim(); + if (!cleanName || cleanName.length > 120) throw new Error("Enter a name up to 120 characters."); + if (cleanDescription.length > 500) throw new Error("Description must be at most 500 characters."); + const setup = verification.data?.verification; + if (!setup || !verification.data) throw new Error("Set up an unlock key before adding a model account."); + if (!accounts.data) throw new Error("Model accounts are still loading. Try again shortly."); + try { await verifyUnlockKey(key, verification.data.ownerId, setup.generationId, JSON.parse(setup.encryptedVerification)); } + catch { throw new Error("Unlock key could not be verified for this account and generation."); } + const id = crypto.randomUUID(), expectedRevision = accounts.data.revision; + const encryptedPayload = await encryptModelAccountCacheRecord(new TextEncoder().encode("{}"), key, { + ownerId: verification.data.ownerId, recordId: id, recordType: "model", destination: "factorize", + keyGeneration: setup.generationId, revision: expectedRevision + 1, + }); + const { data, error } = await api.POST("/api/v1/model-accounts", { body: { + id, expectedRevision, idempotencyKey: crypto.randomUUID(), name: cleanName, + description: cleanDescription || null, encryptedPayload, + } }); + if (!data || error) throw new Error(messageOf(error)); + return data; + }, onSuccess: async () => { + setName(""); setDescription(""); setKey(""); setError(""); + await cache.invalidateQueries({ queryKey: accountsQuery.queryKey }); + }, onError: async reason => { + setKey(""); setError(reason.message); + if (reason.message.includes("changed") || reason.message.includes("revision")) await cache.invalidateQueries({ queryKey: accountsQuery.queryKey }); + } }); + + const remove = useMutation({ retry: false, mutationFn: async (account: ModelAccount) => { + if (!accounts.data) throw new Error("Model accounts are still loading. Try again shortly."); + const { data, error } = await api.DELETE("/api/v1/model-accounts/{accountId}", { + params: { path: { accountId: account.id } }, + body: { expectedRevision: accounts.data.revision, idempotencyKey: crypto.randomUUID() }, + }); + if (!data || error) throw new Error(messageOf(error)); + return data; + }, onSuccess: async () => { + setError(""); await cache.invalidateQueries({ queryKey: accountsQuery.queryKey }); + }, onError: async reason => { + setError(reason.message); await cache.invalidateQueries({ queryKey: accountsQuery.queryKey }); + } }); useEffect(() => { if (!active) return; @@ -71,8 +117,8 @@ export function ModelAccounts() { {accounts.isPending &&

Loading model accounts…

} {accounts.error &&

{accounts.error.message}

} {error &&

{error}

} - {!accounts.isPending && accounts.data?.length === 0 &&

No model accounts are configured yet.

} - {accounts.data?.map(account => { + {!accounts.isPending && accounts.data?.accounts.length === 0 &&

No model accounts are configured yet.

} + {accounts.data?.accounts.map(account => { const state = states[account.id] ?? { state: "idle" }; return

{account.name}

{account.description &&

{account.description}

} @@ -83,8 +129,18 @@ export function ModelAccounts() { {state.state === "failed" &&

Device authorization failed. You can try again.

} {state.state === "canceled" &&

Device authorization canceled.

}
- {state.state === "pending" ? : } + {state.state === "pending" ? : } + ; })} +

Add a model account

+ {!verification.data?.verification && verification.data &&

Set up your unlock key above before adding a model account.

} +
{ event.preventDefault(); setError(""); create.mutate(); }}> + + + + +
+
; } diff --git a/packages/app/src/lib/unlock-crypto.ts b/packages/app/src/lib/unlock-crypto.ts index 900aabcf..7648d973 100644 --- a/packages/app/src/lib/unlock-crypto.ts +++ b/packages/app/src/lib/unlock-crypto.ts @@ -91,6 +91,22 @@ export async function encryptRecord(plaintext: Uint8Array, key: string, binding: return { version: VERSION, algorithm: "A256GCM", nonce: encodeBase64Url(nonce), ciphertext: encodeBase64Url(new Uint8Array(ciphertext)) }; } +/** Encrypt an initial model-account cache with the same key-generation-bound AAD used by the proxy. */ +export async function encryptModelAccountCacheRecord(plaintext: Uint8Array, key: string, binding: RecordBinding & { keyGeneration: string }): Promise { + if (!(plaintext instanceof Uint8Array) || !binding || typeof binding.ownerId !== "string" || !binding.ownerId + || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(binding.recordId) + || typeof binding.keyGeneration !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(binding.keyGeneration) + || !Number.isSafeInteger(binding.revision) || binding.revision < 1) { + throw new TypeError("Malformed model-account encryption binding"); + } + const aad = encoder.encode(JSON.stringify(["factorize-record-v1", binding.ownerId, binding.recordId, "model", "factorize", binding.keyGeneration, binding.revision])); + const nonce = crypto.getRandomValues(new Uint8Array(NONCE_BYTES)); + const aesKey = await deriveAesKey(key, ["encrypt"]); + const ciphertext = await crypto.subtle.encrypt({ name: "AES-GCM", iv: toArrayBuffer(nonce), additionalData: toArrayBuffer(aad), tagLength: 128 }, aesKey, toArrayBuffer(plaintext)); + const encode = (bytes: Uint8Array) => btoa(Array.from(bytes, byte => String.fromCharCode(byte)).join("")); + return `${encode(nonce)}.${encode(new Uint8Array(ciphertext))}`; +} + /** Decrypt a supported envelope; malformed metadata, bindings and authentication fail closed. */ export async function decryptRecord(envelope: unknown, key: string, binding: RecordBinding): Promise { if (!envelope || typeof envelope !== "object" || Array.isArray(envelope)) throw new TypeError("Malformed encrypted envelope"); diff --git a/packages/app/test/settings.browser.test.mjs b/packages/app/test/settings.browser.test.mjs index 81903180..0d5ca6f0 100644 --- a/packages/app/test/settings.browser.test.mjs +++ b/packages/app/test/settings.browser.test.mjs @@ -24,7 +24,7 @@ test("model account settings issue short-lived capabilities and display the boun await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); await page.route("**/api/v1/integrations", route => route.fulfill({ json: { linear: null, clickup: null, exe: null, exeConnections: [], ampConnections: [], cloudflareTail: { count: 0, installations: [] } } })); await page.route("**/api/v1/model-accounts", async route => { - if (route.request().method() === "GET") return route.fulfill({ json: { accounts: [{ id: accountId, name: "Work account", description: null, revision: 0 }] } }); + if (route.request().method() === "GET") return route.fulfill({ json: { revision: 1, accounts: [{ id: accountId, name: "Work account", description: null, revision: 1 }] } }); const action = route.request().postDataJSON().action; return route.fulfill({ json: { endpoint: `${proxyOrigin}/model-accounts/${accountId}/${action}`, capability: `one-use-${action}` } }); }); @@ -51,6 +51,70 @@ test("model account settings issue short-lived capabilities and display the boun assert.deepEqual(proxyCalls.map(call => call.body), ["{}", "{}"]); await context.close(); }); +test("model account create/delete encrypts initial state and uses the current collection revision", async () => { + const { context, page } = await setup(); + await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); + await page.route("**/api/v1/integrations", route => route.fulfill({ json: { linear: null, clickup: null, exe: null, exeConnections: [], ampConnections: [], cloudflareTail: { count: 0, installations: [] } } })); + let verification = null, revision = 0, account = null, createBody = null, deleteBodies = [], staleOnce = true; + await page.route("**/api/v1/unlock-key/verification", async route => { + if (route.request().method() === "GET") return route.fulfill({ json: { ownerId: "owner", verification } }); + const body = route.request().postDataJSON(); verification = { generationId: body.generationId, encryptedVerification: body.encryptedVerification }; + return route.fulfill({ json: { generationId: body.generationId } }); + }); + await page.route("**/api/v1/model-accounts", async route => { + if (route.request().method() === "GET") return route.fulfill({ json: { revision, accounts: account ? [account] : [] } }); + createBody = route.request().postDataJSON(); + assert.deepEqual(Object.keys(createBody).sort(), ["description", "encryptedPayload", "expectedRevision", "id", "idempotencyKey", "name"]); + assert.equal(createBody.expectedRevision, revision); + assert.equal(createBody.name, "Work account"); + assert.equal(createBody.encryptedPayload.includes("{}"), false); + assert.equal(createBody.encryptedPayload.includes("fzk1."), false); + assert.match(createBody.encryptedPayload, /^[A-Za-z0-9+/]+=*\.[A-Za-z0-9+/]+=*$/u); + revision++; account = { id: createBody.id, name: createBody.name, description: createBody.description, revision }; + return route.fulfill({ status: 201, json: { revision, duplicate: false } }); + }); + await page.route("**/api/v1/model-accounts/*", async route => { + assert.equal(route.request().method(), "DELETE"); + const body = route.request().postDataJSON(); deleteBodies.push(body); + assert.deepEqual(Object.keys(body).sort(), ["expectedRevision", "idempotencyKey"]); + if (staleOnce) { + staleOnce = false; revision++; + return route.fulfill({ status: 409, json: { error: { code: "stale_revision", message: "The model account collection changed." } } }); + } + assert.equal(body.expectedRevision, revision); + revision++; account = null; + return route.fulfill({ json: { id: createBody.id, deleted: true, revision, duplicate: false } }); + }); + + await page.goto(origin + "/settings/integrations"); + await page.getByRole("button", { name: "Generate unlock key" }).click(); + const unlockKey = await page.getByLabel("New unlock key").inputValue(); + await page.getByRole("checkbox", { name: "I saved this key somewhere safe." }).check(); + await page.getByRole("button", { name: "I saved my key" }).click(); + await page.getByText("Unlock key verification is set up.", { exact: false }).waitFor(); + + const models = page.getByRole("region", { name: "ChatGPT model accounts" }); + await models.getByLabel("Name").fill("Work account"); + await models.getByLabel("Description (optional)").fill("Personal account"); + await models.getByLabel("Unlock key").fill(unlockKey); + await models.getByRole("button", { name: "Add model account" }).click(); + await models.getByText("Work account", { exact: true }).waitFor(); + assert.equal(createBody.expectedRevision, 0); + assert.equal(createBody.description, "Personal account"); + + page.once("dialog", dialog => dialog.accept()); + const refreshAfterConflict = page.waitForResponse(response => response.url().includes("/api/v1/model-accounts") && response.request().method() === "GET" && response.status() === 200); + await models.getByRole("button", { name: "Delete account" }).click(); + await models.getByRole("alert").filter({ hasText: "collection changed" }).waitFor(); + await refreshAfterConflict; + page.once("dialog", dialog => dialog.accept()); + await models.getByRole("button", { name: "Delete account" }).click(); + await models.getByText("No model accounts are configured yet.", { exact: true }).waitFor(); + assert.equal(deleteBodies.length, 2); + assert.equal(deleteBodies[0].expectedRevision, 1); + assert.equal(deleteBodies[1].expectedRevision, 2); + await context.close(); +}); test("unlock-key setup requires acknowledgement, stores ciphertext only, verifies entered keys and survives refresh without a saved key", async () => { const { context, page } = await setup(); await page.route("**/api/v1/providers/github/installations", route => route.fulfill({ json: [] })); diff --git a/packages/app/test/unlock-crypto.test.mjs b/packages/app/test/unlock-crypto.test.mjs index 2ac54691..ab04eb4a 100644 --- a/packages/app/test/unlock-crypto.test.mjs +++ b/packages/app/test/unlock-crypto.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { createUnlockVerification, decryptRecord, encryptAfterUnlockVerification, encryptRecord, generateUnlockKey, verifyUnlockKey } from "../src/lib/unlock-crypto.ts"; +import { createUnlockVerification, decryptRecord, encryptAfterUnlockVerification, encryptModelAccountCacheRecord, encryptRecord, generateUnlockKey, verifyUnlockKey } from "../src/lib/unlock-crypto.ts"; const key = "fzk1.AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8"; const binding = { ownerId: "owner-vector", recordId: "record-vector", recordType: "integration", destination: "https://api.example.test", revision: 7 }; @@ -25,6 +25,25 @@ test("fresh nonces make repeated encryptions differ", async () => { assert.notEqual(first.ciphertext, second.ciphertext); }); +test("initial model-account cache uses the proxy key-generation and record-revision binding", async () => { + const unlockKey = generateUnlockKey(), modelBinding = { + ownerId: "owner-a", recordId: "11111111-1111-4111-8111-111111111111", recordType: "model", + destination: "factorize", keyGeneration: "22222222-2222-4222-8222-222222222222", revision: 4, + }; + const encrypted = await encryptModelAccountCacheRecord(encoder.encode("{}"), unlockKey, modelBinding); + const [nonce, ciphertext] = encrypted.split(".").map(value => Uint8Array.from(atob(value), char => char.charCodeAt(0))); + const raw = Uint8Array.from(atob(unlockKey.slice(5).replaceAll("-", "+").replaceAll("_", "/") + "="), char => char.charCodeAt(0)); + try { + const material = await crypto.subtle.importKey("raw", raw, "HKDF", false, ["deriveKey"]); + const aes = await crypto.subtle.deriveKey({ name: "HKDF", hash: "SHA-256", salt: encoder.encode("factorize.unlock-key.v1"), info: encoder.encode("factorize.record.aes-256-gcm.v1") }, material, { name: "AES-GCM", length: 256 }, false, ["decrypt"]); + const aad = encoder.encode(JSON.stringify(["factorize-record-v1", modelBinding.ownerId, modelBinding.recordId, "model", "factorize", modelBinding.keyGeneration, modelBinding.revision])); + const plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv: nonce, additionalData: aad, tagLength: 128 }, aes, ciphertext); + assert.equal(new TextDecoder().decode(plain), "{}"); + const staleAad = encoder.encode(JSON.stringify(["factorize-record-v1", modelBinding.ownerId, modelBinding.recordId, "model", "factorize", modelBinding.keyGeneration, modelBinding.revision + 1])); + await assert.rejects(crypto.subtle.decrypt({ name: "AES-GCM", iv: nonce, additionalData: staleAad, tagLength: 128 }, aes, ciphertext)); + } finally { raw.fill(0); } +}); + test("wrong key and every authenticated binding change fail closed", async () => { await assert.rejects(decryptRecord(vector, "fzk1.AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE", binding)); for (const field of Object.keys(binding)) { diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index 6089764f..32c63f8c 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -1810,19 +1810,23 @@ paths: get: summary: List model accounts available to the owner description: |- - Requires an interactive owner browser session. Returns account metadata only; encrypted account caches and credentials are never returned. + Requires an interactive owner browser session. Returns account metadata and the collection revision only; encrypted account caches and credentials are never returned. Requires the integrations:read scope. An interactive owner session is required; bearer tokens cannot call this operation. responses: "200": - description: Model account metadata + description: Model account metadata and collection revision content: application/json: schema: type: object properties: + revision: + type: integer + minimum: 0 + maximum: 9007199254740991 accounts: type: array items: @@ -1849,6 +1853,7 @@ paths: - revision additionalProperties: false required: + - revision - accounts additionalProperties: false "400": @@ -1875,6 +1880,187 @@ paths: - cookieAuth: [] x-required-scope: integrations:read x-owner-session-required: true + post: + summary: Create a model account + description: |- + Requires an interactive owner browser session and the current collection revision. Accepts only user-encrypted initial account state; credentials are established directly with the credential proxy. + + Requires the integrations:write scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "201": + description: Created model account + content: + application/json: + schema: + type: object + properties: + revision: + type: integer + minimum: 0 + maximum: 9007199254740991 + duplicate: + type: boolean + required: + - revision + - duplicate + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + "415": + $ref: "#/components/responses/Error415" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: post_api_v1_model_accounts + tags: + - model-accounts + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: integrations:write + x-owner-session-required: true + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + id: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + expectedRevision: + type: integer + minimum: 0 + maximum: 9007199254740991 + idempotencyKey: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + name: + type: string + minLength: 1 + maxLength: 120 + description: + anyOf: + - type: string + maxLength: 500 + - type: "null" + encryptedPayload: + type: string + minLength: 1 + maxLength: 65536 + required: + - id + - expectedRevision + - idempotencyKey + - name + - description + - encryptedPayload + additionalProperties: false + /api/v1/model-accounts/{accountId}: + delete: + summary: Delete a model account + description: |- + Requires an interactive owner browser session and the current collection revision. Deletion is recorded as a tombstone so future snapshots and unlock reconciliation cannot restore the account. + + Requires the integrations:write scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "200": + description: Deleted model account + content: + application/json: + schema: + type: object + properties: + id: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + deleted: + type: boolean + const: true + revision: + type: integer + minimum: 0 + maximum: 9007199254740991 + duplicate: + type: boolean + required: + - id + - deleted + - revision + - duplicate + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + "415": + $ref: "#/components/responses/Error415" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: delete_api_v1_model_accounts_accountId + tags: + - model-accounts + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: integrations:write + x-owner-session-required: true + parameters: + - name: accountId + in: path + required: true + schema: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + expectedRevision: + type: integer + minimum: 0 + maximum: 9007199254740991 + idempotencyKey: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + required: + - expectedRevision + - idempotencyKey + additionalProperties: false /api/v1/model-accounts/{accountId}/browser-capability: post: summary: Issue a short-lived browser capability for a paired model account From b99aa4260efd29dd991dde425c2ad8ea3f216012 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:48:17 +0000 Subject: [PATCH 090/102] GEN-2824 recover stale model account cache writes --- .../credential-proxy/src/control-client.mjs | 10 ++- .../src/model-account-lifecycle.mjs | 86 +++++++++++++------ .../test/model-account-lifecycle.test.mjs | 42 ++++++++- 3 files changed, 109 insertions(+), 29 deletions(-) diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 9b4656c9..0a4c889e 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -39,7 +39,15 @@ async function proxyControlRequest({ controlOrigin, token, path, body, fetchImpl method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(10_000), redirect: "error", }); - if (!response.ok) throw new Error("proxy control request rejected"); + if (!response.ok) { + let code; + try { code = (await response.clone().json())?.error; } catch { /* Keep error handling bounded to the HTTP status. */ } + const error = new Error("proxy control request rejected"); + error.name = "ProxyControlRequestError"; + error.status = response.status; + if (typeof code === "string") error.code = code; + throw error; + } return response.json(); } diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index fc6a1308..cb5af18e 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -25,6 +25,7 @@ export function createModelAccountLifecycle({ persistEncrypted, onUnsaved = () => {}, retryDelaysMs = [100, 500, 2_000], + staleRevisionRetries = 3, sleep = ms => new Promise(resolveSleep => setTimeout(resolveSleep, ms)), removeCache = rm, readCache = readFile, @@ -37,7 +38,8 @@ export function createModelAccountLifecycle({ if (!authHelper || typeof authHelper.invoke !== "function" || typeof getUnlockKey !== "function" || typeof getAccount !== "function" || typeof encryptCache !== "function" || typeof persistEncrypted !== "function" || !Array.isArray(retryDelaysMs) || retryDelaysMs.length > 5 - || retryDelaysMs.some(ms => !Number.isSafeInteger(ms) || ms < 0 || ms > 30_000)) { + || retryDelaysMs.some(ms => !Number.isSafeInteger(ms) || ms < 0 || ms > 30_000) + || !Number.isSafeInteger(staleRevisionRetries) || staleRevisionRetries < 0 || staleRevisionRetries > 5) { throw new TypeError("model account lifecycle configuration is invalid"); } if ((getEncryptedCache === undefined) !== (decryptCache === undefined) @@ -59,7 +61,7 @@ export function createModelAccountLifecycle({ } async function persist(id, generation) { - const account = await getAccount(id); + let account = await getAccount(id); if (!account || account.id !== id || account.kind !== "model" || !Number.isSafeInteger(account.revision) || typeof account.ownerId !== "string" || !account.ownerId || typeof account.keyGeneration !== "string" || !account.keyGeneration) { throw new Error("model account persistence binding is unavailable"); @@ -68,36 +70,66 @@ export function createModelAccountLifecycle({ if (typeof unlockKey !== "string" || !unlockKey) throw new Error("proxy is locked"); const plain = await readCache(cachePath(accountHomeRoot, id)); if (!Buffer.isBuffer(plain) || plain.byteLength === 0 || plain.byteLength > 65_536) throw new Error("Codex token cache is missing or exceeds its limit"); - let encrypted; + let staleRetries = 0; + let attempts = 0; try { - encrypted = await encryptCache(new Uint8Array(plain), unlockKey, { - ownerId: account.ownerId, - recordId: id, - recordType: "model", - destination: "factorize", - revision: account.revision + 1, - keyGeneration: account.keyGeneration, - }); - } finally { - plain.fill(0); - } - if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); - const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted, idempotencyKey: randomUUID() }; - for (let attempt = 0; ; attempt++) { - if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); - try { - await persistEncrypted(request); - unsaved.delete(id); - onUnsaved({ accountId: id, unsaved: false }); - return { state: "saved" }; - } catch (error) { - if (attempt >= retryDelaysMs.length) { + for (;;) { + let encrypted; + try { + encrypted = await encryptCache(new Uint8Array(plain), unlockKey, { + ownerId: account.ownerId, + recordId: id, + recordType: "model", + destination: "factorize", + revision: account.revision + 1, + keyGeneration: account.keyGeneration, + }); + } catch (error) { + throw error; + } + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + const request = { id, expectedRevision: account.revision, keyGeneration: account.keyGeneration, encryptedPayload: encrypted, idempotencyKey: randomUUID() }; + let retryWithLatestRevision = false; + for (let attempt = 0; ; attempt++) { + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + attempts++; + try { + await persistEncrypted(request); + unsaved.delete(id); + onUnsaved({ accountId: id, unsaved: false }); + return { state: "saved" }; + } catch (error) { + if (error?.code === "stale_revision") { + if (staleRetries >= staleRevisionRetries) break; + let latest; + try { latest = await getAccount(id); } + catch { throw new Error("model account changed before token persistence"); } + if (!latest || latest.id !== id || latest.kind !== "model" || !Number.isSafeInteger(latest.revision) + || latest.revision <= account.revision || latest.ownerId !== account.ownerId + || latest.keyGeneration !== account.keyGeneration) { + throw new Error("model account changed before token persistence"); + } + account = latest; + staleRetries++; + retryWithLatestRevision = true; + break; + } + if (attempt >= retryDelaysMs.length) { + unsaved.add(id); + onUnsaved({ accountId: id, unsaved: true }); + return { state: "unsaved", attempts }; + } + await sleep(retryDelaysMs[attempt]); + } + } + if (!retryWithLatestRevision) { unsaved.add(id); onUnsaved({ accountId: id, unsaved: true }); - return { state: "unsaved", attempts: attempt + 1 }; + return { state: "unsaved", attempts }; } - await sleep(retryDelaysMs[attempt]); } + } finally { + plain.fill(0); } } diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 37ab14e4..2028d069 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -18,7 +18,7 @@ async function fixture(t, overrides = {}) { const helper = { invoke: async request => { methods.push(request.method); if (request.method === "device-login") return { state: "pending", completion: overrides.completion ?? Promise.resolve({ state: "authorized" }) }; - if (request.method === "refresh") { calls.push("refresh"); return { state: "refreshed" }; } + if (request.method === "refresh") { calls.push("refresh"); return overrides.refreshResult ?? { state: "refreshed" }; } if (request.method === "account-status") return { state: "authenticated" }; return { state: request.method === "cancel" ? "canceled" : "logged-out" }; } }; @@ -67,6 +67,12 @@ test("denied or expired device authorization does not persist token state", asyn assert.equal(f.saves, 0); }); +test("expired refresh explicitly requires reauthentication without saving stale tokens", async t => { + const f = await fixture(t, { refreshResult: { state: "reauthentication-required" } }); + assert.deepEqual(await f.lifecycle.refresh(accountId), { state: "reauthentication-required" }); + assert.equal(f.saves, 0); +}); + test("refresh keeps the working local cache and reports bounded encrypted-save failures", async t => { const f = await fixture(t, { retryDelaysMs: [0, 0], persistEncrypted: async () => { throw new Error("fixture outage"); } }); assert.deepEqual(await f.lifecycle.refresh(accountId), { state: "unsaved", attempts: 3 }); @@ -93,6 +99,28 @@ test("refresh requests for one account serialize and use the latest revision", a assert.equal(f.saves, 2); }); +test("refresh re-encrypts against a newer account revision after a concurrent edit", async t => { + let revision = 4; + const f = await fixture(t, { + getAccount: () => ({ ...account, revision }), + persistEncrypted: async request => { + if (request.expectedRevision !== revision || f.saves === 1) { + revision++; + const error = new Error("stale model account revision"); + error.code = "stale_revision"; + throw error; + } + revision++; + }, + }); + assert.deepEqual(await f.lifecycle.refresh(accountId), { state: "saved" }); + const writes = f.events.filter(event => event.type === "persist").map(event => event.request); + assert.deepEqual(writes.map(request => request.expectedRevision), [4, 5]); + assert.notEqual(writes[0].idempotencyKey, writes[1].idempotencyKey); + const encryptions = f.events.filter(event => event.type === "encrypt"); + assert.deepEqual(encryptions.map(event => event.binding.revision), [5, 6]); +}); + test("deletion invalidates a retry and removes the account cache", async t => { let release; const waiting = new Promise(resolve => { release = resolve; }); @@ -112,6 +140,18 @@ test("deletion invalidates a retry and removes the account cache", async t => { assert.ok(f.methods.includes("logout")); }); +test("deletion racing pending device login prevents a late authorization from restoring its cache", async t => { + let complete; + const f = await fixture(t, { completion: new Promise(resolve => { complete = resolve; }) }); + const login = await f.lifecycle.deviceLogin(accountId); + const deleting = f.lifecycle.deleteAccount(accountId); + complete({ state: "authorized" }); + assert.deepEqual(await login.completion, { state: "deleted" }); + assert.deepEqual(await deleting, { state: "deleted" }); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.equal(f.saves, 0); +}); + test("unlock reconciliation restores the encrypted Codex cache before checking account status", async t => { const persisted = new TextEncoder().encode(JSON.stringify({ tokens: { access_token: "restored-access", refresh_token: "restored-refresh" } })); const encryptedPayload = `cache.${Buffer.from(persisted).toString("base64")}`; From 4883c887ce7473edb7d206b29c5c520126fc3e78 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:55:44 +0000 Subject: [PATCH 091/102] GEN-2824 clear deleted model caches on unlock --- packages/credential-proxy/src/main.mjs | 9 +++- .../src/model-account-lifecycle.mjs | 49 ++++++++++++++----- .../test/model-account-lifecycle.test.mjs | 25 ++++++++++ 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 016c80a3..50f1ba2c 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -132,8 +132,13 @@ lines.on("line", async (line) => { else if (command?.type === "unlock") { runtime.unlock(command.key); if (modelAccountLifecycle && modelAccounts) { - const accounts = await modelAccounts.listAccounts(); - for (const account of accounts) await modelAccountLifecycle.reconcile(account.id); + try { + const accounts = await modelAccounts.listAccounts(); + await modelAccountLifecycle.reconcileAll(accounts); + } catch (error) { + await runtime.lock(); + throw error; + } } respondStatus(); } else if (command?.type === "lock") { diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index cb5af18e..14179ce8 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -1,4 +1,4 @@ -import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; import { randomUUID } from "node:crypto"; import { join, resolve, sep } from "node:path"; @@ -31,6 +31,7 @@ export function createModelAccountLifecycle({ readCache = readFile, writeCache = writeFile, makeDirectory = mkdir, + readDirectory = readdir, replaceFile = rename, getEncryptedCache, decryptCache, @@ -220,15 +221,8 @@ export function createModelAccountLifecycle({ return { state: "restored", revision: encrypted.revision }; } - return Object.freeze({ - deviceLogin, - cancel, - refresh, - deleteAccount, - restore, - isUnsaved: id => unsaved.has(id), - /** Invoke after encrypted snapshots are reconciled at unlock. */ - reconcile: async id => serialize(id, async () => { + async function reconcile(id) { + return serialize(id, async () => { if (!await getAccount(id)) { await removeCache(resolve(accountHomeRoot, id), { recursive: true, force: true }); unsaved.delete(id); @@ -240,7 +234,40 @@ export function createModelAccountLifecycle({ } const result = await authHelper.invoke({ method: "account-status", accountId: id }); return result.state === "authenticated" ? persist(id, currentGeneration(id)) : { state: "reauthentication-required" }; - }), + }); + } + + return Object.freeze({ + deviceLogin, + cancel, + refresh, + deleteAccount, + restore, + reconcile, + isUnsaved: id => unsaved.has(id), + /** Reconcile a complete server snapshot after unlock. A reset removes every + * prior model account from that snapshot, so stale local Codex homes must + * be logged out and removed before any current account is admitted. */ + reconcileAll: async accounts => { + if (!Array.isArray(accounts) || accounts.some(account => !account || !UUID.test(account.id ?? "") || account.kind !== "model") + || new Set(accounts.map(account => account.id)).size !== accounts.length) { + throw new TypeError("model account snapshot is invalid"); + } + const activeIds = new Set(accounts.map(account => account.id)); + const entries = await readDirectory(accountHomeRoot, { withFileTypes: true }).catch(error => { + if (error?.code === "ENOENT") return []; + throw error; + }); + const removed = []; + for (const entry of entries) { + if (!UUID.test(entry.name) || activeIds.has(entry.name)) continue; + await deleteAccount(entry.name); + removed.push(entry.name); + } + const states = []; + for (const account of accounts) states.push({ accountId: account.id, ...(await reconcile(account.id)) }); + return { removed, accounts: states }; + }, }); } diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 2028d069..342886ed 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -6,6 +6,7 @@ import { join } from "node:path"; import { createModelAccountLifecycle } from "../src/model-account-lifecycle.mjs"; const accountId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const resetAccountId = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; const account = { id: accountId, kind: "model", ownerId: "owner", keyGeneration: "generation", revision: 4 }; async function fixture(t, overrides = {}) { @@ -190,3 +191,27 @@ test("unlock reconciliation removes stale cache when the account key generation await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); assert.equal(f.methods.includes("account-status"), false); }); + +test("unlock snapshot reconciliation logs out and removes cached accounts deleted by reset", async t => { + const f = await fixture(t); + const staleHome = join(f.root, resetAccountId); + await mkdir(staleHome, { recursive: true }); + await writeFile(join(staleHome, "auth.json"), JSON.stringify({ tokens: { access_token: "stale-reset-token" } })); + + assert.deepEqual(await f.lifecycle.reconcileAll([account]), { + removed: [resetAccountId], + accounts: [{ accountId, state: "saved" }], + }); + await assert.rejects(readFile(join(staleHome, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); + assert.equal(f.saves, 1); +}); + +test("unlock snapshot rejects duplicate account identities before touching local state", async t => { + const f = await fixture(t); + await assert.rejects(f.lifecycle.reconcileAll([account, account]), /snapshot is invalid/); + assert.deepEqual(JSON.parse(await readFile(join(f.home, "auth.json"), "utf8")), { + tokens: { access_token: "synthetic-access", refresh_token: "synthetic-refresh" }, + }); + assert.equal(f.methods.length, 0); +}); From 1d56983755e75e981b9917616a771b9b705537d6 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 04:19:13 +0000 Subject: [PATCH 092/102] GEN-2824 cover model cache key rotation race --- .../test/model-account-lifecycle.test.mjs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 342886ed..8d90fa1d 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -122,6 +122,23 @@ test("refresh re-encrypts against a newer account revision after a concurrent ed assert.deepEqual(encryptions.map(event => event.binding.revision), [5, 6]); }); +test("refresh does not retry a stale cache write after the unlock-key generation rotates", async t => { + let generation = "generation"; + const f = await fixture(t, { + getAccount: () => ({ ...account, keyGeneration: generation }), + persistEncrypted: async () => { + generation = "rotated-generation"; + const error = new Error("stale model account revision"); + error.code = "stale_revision"; + throw error; + }, + }); + + await assert.rejects(f.lifecycle.refresh(accountId), /changed before token persistence/); + assert.equal(f.saves, 1); + assert.deepEqual(f.events.filter(event => event.type === "encrypt").map(event => event.binding.keyGeneration), ["generation"]); +}); + test("deletion invalidates a retry and removes the account cache", async t => { let release; const waiting = new Promise(resolve => { release = resolve; }); From 6ed5e760fc87816dc48f9dd66b21b54bfaee7c76 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 04:34:34 +0000 Subject: [PATCH 093/102] GEN-2824 cover model reset refresh race --- .../test/model-account-lifecycle.test.mjs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 8d90fa1d..0438a445 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -158,6 +158,34 @@ test("deletion invalidates a retry and removes the account cache", async t => { assert.ok(f.methods.includes("logout")); }); +test("reset snapshot invalidates a refresh cache write already in flight", async t => { + let release; + let entered; + const waiting = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const f = await fixture(t, { + persistEncrypted: async () => { + entered(); + await waiting; + const error = new Error("stale model account revision"); + error.code = "stale_revision"; + throw error; + }, + }); + + const refreshing = f.lifecycle.refresh(accountId); + await started; + const resetting = f.lifecycle.reconcileAll([]); + await new Promise(resolve => setImmediate(resolve)); + release(); + + await assert.rejects(refreshing, /changed before token persistence/); + assert.deepEqual(await resetting, { removed: [accountId], accounts: [] }); + assert.equal(f.saves, 1); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); +}); + test("deletion racing pending device login prevents a late authorization from restoring its cache", async t => { let complete; const f = await fixture(t, { completion: new Promise(resolve => { complete = resolve; }) }); From 4b67c2adce810ca648e4b237e0765751a11d6bba Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 04:47:45 +0000 Subject: [PATCH 094/102] GEN-2824 cover reset during refresh retry --- .../test/model-account-lifecycle.test.mjs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index 0438a445..f188b7bb 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -186,6 +186,29 @@ test("reset snapshot invalidates a refresh cache write already in flight", async assert.ok(f.methods.includes("logout")); }); +test("deletion during refresh persistence backoff prevents a delayed retry", async t => { + let releaseBackoff; + let enteredBackoff; + const waiting = new Promise(resolve => { releaseBackoff = resolve; }); + const started = new Promise(resolve => { enteredBackoff = resolve; }); + const f = await fixture(t, { + retryDelaysMs: [1], + options: { sleep: async () => { enteredBackoff(); await waiting; } }, + persistEncrypted: async () => { throw new Error("temporary persistence failure"); }, + }); + + const refreshing = f.lifecycle.refresh(accountId); + await started; + const deleting = f.lifecycle.deleteAccount(accountId); + releaseBackoff(); + + await assert.rejects(refreshing, /changed before token persistence/); + assert.deepEqual(await deleting, { state: "deleted" }); + assert.equal(f.saves, 1); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); +}); + test("deletion racing pending device login prevents a late authorization from restoring its cache", async t => { let complete; const f = await fixture(t, { completion: new Promise(resolve => { complete = resolve; }) }); From 3981a80ea6b25614224d0409a5b6aae892d5539a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 05:09:24 +0000 Subject: [PATCH 095/102] GEN-2824 invalidate refresh retries after model rotation --- .../src/model-account-lifecycle.mjs | 8 ++++ .../test/model-account-lifecycle.test.mjs | 47 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/packages/credential-proxy/src/model-account-lifecycle.mjs b/packages/credential-proxy/src/model-account-lifecycle.mjs index 14179ce8..565abde8 100644 --- a/packages/credential-proxy/src/model-account-lifecycle.mjs +++ b/packages/credential-proxy/src/model-account-lifecycle.mjs @@ -121,6 +121,14 @@ export function createModelAccountLifecycle({ return { state: "unsaved", attempts }; } await sleep(retryDelaysMs[attempt]); + if (currentGeneration(id) !== generation) throw new Error("model account changed before token persistence"); + let latest; + try { latest = await getAccount(id); } + catch { throw new Error("model account changed before token persistence"); } + if (!latest || latest.id !== id || latest.kind !== "model" || latest.ownerId !== account.ownerId + || latest.keyGeneration !== account.keyGeneration) { + throw new Error("model account changed before token persistence"); + } } } if (!retryWithLatestRevision) { diff --git a/packages/credential-proxy/test/model-account-lifecycle.test.mjs b/packages/credential-proxy/test/model-account-lifecycle.test.mjs index f188b7bb..310b4dfb 100644 --- a/packages/credential-proxy/test/model-account-lifecycle.test.mjs +++ b/packages/credential-proxy/test/model-account-lifecycle.test.mjs @@ -139,6 +139,20 @@ test("refresh does not retry a stale cache write after the unlock-key generation assert.deepEqual(f.events.filter(event => event.type === "encrypt").map(event => event.binding.keyGeneration), ["generation"]); }); +test("refresh does not retry a transient cache write after the key generation rotates during backoff", async t => { + let generation = "generation"; + const f = await fixture(t, { + retryDelaysMs: [1], + getAccount: () => ({ ...account, keyGeneration: generation }), + options: { sleep: async () => { generation = "rotated-generation"; } }, + persistEncrypted: async () => { throw new Error("temporary persistence failure"); }, + }); + + await assert.rejects(f.lifecycle.refresh(accountId), /changed before token persistence/); + assert.equal(f.saves, 1); + assert.deepEqual(f.events.filter(event => event.type === "encrypt").map(event => event.binding.keyGeneration), ["generation"]); +}); + test("deletion invalidates a retry and removes the account cache", async t => { let release; const waiting = new Promise(resolve => { release = resolve; }); @@ -209,6 +223,39 @@ test("deletion during refresh persistence backoff prevents a delayed retry", asy assert.ok(f.methods.includes("logout")); }); +test("reset during refresh persistence backoff prevents a delayed retry and clears the cache", async t => { + let releaseBackoff; + let enteredBackoff; + let releaseSnapshot; + let enteredSnapshot; + const waiting = new Promise(resolve => { releaseBackoff = resolve; }); + const started = new Promise(resolve => { enteredBackoff = resolve; }); + const snapshotWaiting = new Promise(resolve => { releaseSnapshot = resolve; }); + const snapshotStarted = new Promise(resolve => { enteredSnapshot = resolve; }); + const f = await fixture(t, { + retryDelaysMs: [1], + options: { + sleep: async () => { enteredBackoff(); await waiting; }, + readDirectory: async () => { enteredSnapshot(); await snapshotWaiting; return [{ name: accountId }]; }, + }, + persistEncrypted: async () => { throw new Error("temporary persistence failure"); }, + }); + + const refreshing = f.lifecycle.refresh(accountId); + await started; + const resetting = f.lifecycle.reconcileAll([]); + await snapshotStarted; + releaseSnapshot(); + await new Promise(resolve => setImmediate(resolve)); + releaseBackoff(); + + await assert.rejects(refreshing, /changed before token persistence/); + assert.deepEqual(await resetting, { removed: [accountId], accounts: [] }); + assert.equal(f.saves, 1); + await assert.rejects(readFile(join(f.home, "auth.json")), { code: "ENOENT" }); + assert.ok(f.methods.includes("logout")); +}); + test("deletion racing pending device login prevents a late authorization from restoring its cache", async t => { let complete; const f = await fixture(t, { completion: new Promise(resolve => { complete = resolve; }) }); From 3414c7b618ed40b0b1cdb009cdbf02777cc02728 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 05:24:08 +0000 Subject: [PATCH 096/102] GEN-2824 define portable proxy launch gates --- packages/api/src/proxy-launch-policy.ts | 50 +++++++++++++++++++ packages/api/test/proxy-launch-policy.test.ts | 36 +++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 packages/api/src/proxy-launch-policy.ts create mode 100644 packages/api/test/proxy-launch-policy.test.ts diff --git a/packages/api/src/proxy-launch-policy.ts b/packages/api/src/proxy-launch-policy.ts new file mode 100644 index 00000000..0987c6f3 --- /dev/null +++ b/packages/api/src/proxy-launch-policy.ts @@ -0,0 +1,50 @@ +/** Inputs required to decide whether a queued execution may create a guest VM. */ +export type ProxyLaunchState = { + enabled: boolean; + running: boolean; + unlocked: boolean; + pendingChanges: boolean; +}; + +export type ProxyLaunchJob = { + executionProvider: "exe" | "fly"; + executionType: "agent" | "code"; + portableIntegrationIds?: readonly string[]; + modelAccess?: "native" | "portable"; + modelAccountId?: string; +}; + +export type ProxyLaunchDecision = + | { kind: "allow" } + | { kind: "start_locked" } + | { kind: "blocked"; reason: "proxy_disabled" | "proxy_unavailable" | "proxy_locked" | "pending_changes"; retry: "manual" }; + +/** + * Encodes the GEN-2824 launch gate. This function is deliberately side-effect + * free: the scheduler must persist a blocked decision and wait for an explicit + * retry instead of treating a later unlock as a queued launch signal. + */ +export function proxyLaunchDecision(job: ProxyLaunchJob, proxy: ProxyLaunchState): ProxyLaunchDecision { + const needsUserCredentials = Boolean(job.portableIntegrationIds?.length) + || job.modelAccess === "portable" + || Boolean(job.modelAccountId); + + // exe.dev-native integrations keep their existing launch behavior. + if (job.executionProvider === "exe" && !needsUserCredentials) return { kind: "allow" }; + + if (!proxy.enabled) return { kind: "blocked", reason: "proxy_disabled", retry: "manual" }; + + // Only code jobs with Factorize access may automatically boot a stopped + // proxy, and that boot is always locked. + if (!needsUserCredentials && job.executionType === "code") { + if (!proxy.running) return { kind: "start_locked" }; + return { kind: "allow" }; + } + + // User credential grants require an already-running proxy with edits + // applied and a user unlock. Unlocking alone does not resume this run. + if (!proxy.running) return { kind: "blocked", reason: "proxy_unavailable", retry: "manual" }; + if (proxy.pendingChanges) return { kind: "blocked", reason: "pending_changes", retry: "manual" }; + if (!proxy.unlocked) return { kind: "blocked", reason: "proxy_locked", retry: "manual" }; + return { kind: "allow" }; +} diff --git a/packages/api/test/proxy-launch-policy.test.ts b/packages/api/test/proxy-launch-policy.test.ts new file mode 100644 index 00000000..81544828 --- /dev/null +++ b/packages/api/test/proxy-launch-policy.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { proxyLaunchDecision, type ProxyLaunchJob, type ProxyLaunchState } from "../src/proxy-launch-policy"; + +const stopped: ProxyLaunchState = { enabled: true, running: false, unlocked: false, pendingChanges: false }; +const locked: ProxyLaunchState = { enabled: true, running: true, unlocked: false, pendingChanges: false }; +const unlocked: ProxyLaunchState = { enabled: true, running: true, unlocked: true, pendingChanges: false }; +const flyCode: ProxyLaunchJob = { executionProvider: "fly", executionType: "code" }; + +describe("GEN-2824 proxy launch policy", () => { + it("starts an enabled proxy locked for Fly code jobs with no user credential grants", () => { + expect(proxyLaunchDecision(flyCode, stopped)).toEqual({ kind: "start_locked" }); + }); + + it("allows code-only Factorize access while the proxy is running and locked", () => { + expect(proxyLaunchDecision(flyCode, locked)).toEqual({ kind: "allow" }); + }); + + it.each([ + { label: "portable tools", job: { ...flyCode, portableIntegrationIds: ["integration"] } }, + { label: "portable model account", job: { ...flyCode, executionType: "agent" as const, modelAccess: "portable" as const, modelAccountId: "model" } }, + ])("requires an unlocked proxy with applied updates for $label", ({ job }) => { + expect(proxyLaunchDecision(job, stopped)).toEqual({ kind: "blocked", reason: "proxy_unavailable", retry: "manual" }); + expect(proxyLaunchDecision(job, locked)).toEqual({ kind: "blocked", reason: "proxy_locked", retry: "manual" }); + expect(proxyLaunchDecision(job, { ...unlocked, pendingChanges: true })).toEqual({ kind: "blocked", reason: "pending_changes", retry: "manual" }); + expect(proxyLaunchDecision(job, unlocked)).toEqual({ kind: "allow" }); + }); + + it("never auto-starts a disabled proxy and preserves native exe.dev behavior", () => { + expect(proxyLaunchDecision(flyCode, { ...stopped, enabled: false })).toEqual({ kind: "blocked", reason: "proxy_disabled", retry: "manual" }); + expect(proxyLaunchDecision({ executionProvider: "exe", executionType: "agent" }, { ...stopped, enabled: false })).toEqual({ kind: "allow" }); + }); + + it("does not auto-start a stopped proxy for credential-dependent jobs", () => { + expect(proxyLaunchDecision({ ...flyCode, portableIntegrationIds: ["integration"] }, stopped)).toMatchObject({ kind: "blocked", retry: "manual" }); + }); +}); From 6d293218e815403cc40dcfc1bfcce9b36b0c866a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 05:50:29 +0000 Subject: [PATCH 097/102] GEN-2824 disable Fly job machine restarts --- packages/api/src/fly-machines-client.ts | 4 ++++ packages/api/test/fly-machines-client.test.ts | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/api/src/fly-machines-client.ts b/packages/api/src/fly-machines-client.ts index a7139e37..f4742050 100644 --- a/packages/api/src/fly-machines-client.ts +++ b/packages/api/src/fly-machines-client.ts @@ -136,6 +136,10 @@ export function createFlyMachinesClient(options: FlyMachinesClientOptions) { const body = { config: { ...config, + // Job Machines are supervised by Factorize. A provider restart can + // replay a script after an ambiguous guest exit, duplicating side + // effects, so execution guests must stop with their process. + restart: { policy: "no" }, metadata: { ...config.metadata, [OPERATION_METADATA_KEY]: operationId }, }, }; diff --git a/packages/api/test/fly-machines-client.test.ts b/packages/api/test/fly-machines-client.test.ts index 9b5bf80c..60ab6d26 100644 --- a/packages/api/test/fly-machines-client.test.ts +++ b/packages/api/test/fly-machines-client.test.ts @@ -42,6 +42,7 @@ describe("bounded Fly Machines client", () => { let raw = ""; for await (const chunk of req) raw += chunk; const body = JSON.parse(raw); expect(body.config.metadata.factorize_operation_id).toBe("op-1"); + expect(body.config.restart).toEqual({ policy: "no" }); return json(res, 200, machine); } if (req.method === "GET" && req.url === "/v1/apps/app-a/machines/m-1") return json(res, 200, machine); @@ -53,7 +54,7 @@ describe("bounded Fly Machines client", () => { json(res, 404, { error: "not found" }); }); const client = createFlyMachinesClient({ token, apiOrigin: origin, wait: async () => undefined }); - const created = await client.createMachine("app-a", { image: "example/image", metadata: { keep: "yes" } }, "op-1"); + const created = await client.createMachine("app-a", { image: "example/image", restart: { policy: "always" }, metadata: { keep: "yes" } }, "op-1"); expect(created.id).toBe("m-1"); expect((await client.inspectMachine("app-a", "m-1")).id).toBe("m-1"); await client.startMachine("app-a", "m-1"); From a5c6d8dc3c9057043e53b36f8bc4846283ae1b96 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 06:03:33 +0000 Subject: [PATCH 098/102] GEN-2824 persist proxy launch blocks and manual retry --- packages/api-client/src/schema.d.ts | 61 +++++++++++++++++++ packages/api/src/api-contract.ts | 9 +++ packages/api/src/flow-service.ts | 6 ++ packages/api/src/postgres/run-repository.ts | 29 ++++++++- packages/api/src/postgres/run-scheduler.ts | 8 ++- packages/api/src/proxy-launch-policy.ts | 11 ++++ packages/api/test/proxy-launch-policy.test.ts | 8 ++- packages/api/test/run-launch-block.test.ts | 25 ++++++++ packages/docs/openapi.yaml | 61 +++++++++++++++++++ 9 files changed, 215 insertions(+), 3 deletions(-) create mode 100644 packages/api/test/run-launch-block.test.ts diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index 1e4bd5e7..858fd9ce 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -2068,6 +2068,30 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/runs/{runId}/retry-launch": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Manually retry a blocked launch + * @description Retries only a pre-launch proxy-gated run after its prerequisites have been resolved. The run remains blocked until an interactive owner explicitly retries it; unlock or status changes do not automatically launch it. + * + * Requires the runs:cancel scope. + * + * An interactive owner session is required; bearer tokens cannot call this operation. + */ + post: operations["post_api_v1_runs_runId_retry_launch"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/runs/{runId}/kill": { parameters: { query?: never; @@ -8206,6 +8230,43 @@ export interface operations { 503: components["responses"]["Error503"]; }; }; + post_api_v1_runs_runId_retry_launch: { + parameters: { + query?: never; + header?: never; + path: { + runId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Manual retry queued */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** Format: uuid */ + runId: string; + /** @constant */ + retryRequested: true; + }; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + /** @description Interactive owner session required */ + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + /** @description Run is not waiting for a manual launch retry */ + 409: components["responses"]["Error409"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; post_api_v1_runs_runId_kill: { parameters: { query?: never; diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index 6ea38cdc..8d646991 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -1127,6 +1127,15 @@ export const API_ROUTES: RouteContract[] = [ { summary: "Stop run", responses: { "200": { description: "Stopped" } } }, (service, { params }) => service.stopRun(params.runId), ), + route("POST", "/api/v1/runs/{runId}/retry-launch", "runs:cancel", { + summary: "Manually retry a blocked launch", + description: "Retries only a pre-launch proxy-gated run after its prerequisites have been resolved. The run remains blocked until an interactive owner explicitly retries it; unlock or status changes do not automatically launch it.", + responses: { + "200": { description: "Manual retry queued", content: { "application/json": { schema: { type: "object", required: ["runId", "retryRequested"], properties: { runId: { type: "string", format: "uuid" }, retryRequested: { type: "boolean", const: true } }, additionalProperties: false } } } }, + "403": { description: "Interactive owner session required" }, + "409": { description: "Run is not waiting for a manual launch retry" }, + }, + }, (service, { params }) => service.retryBlockedLaunch(params.runId), { ownerSession: true, parameters: z.object({ runId: z.uuid() }) }), route( "POST", "/api/v1/runs/{runId}/kill", diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index 9275b638..44e7f067 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -322,6 +322,12 @@ export class ApiService { catch (error) { if (error instanceof TraceReplayError) throw new ServiceError(error.status, error.code, error.message); throw error; } } async stopRun(runId: string) { await this.authorize("runs:cancel"); const stopped = await new RunQueryRepository(databaseFor(this.env), this.auth.tenantId).stop(runId); if (!stopped) throw new ServiceError(409, "operation_failed", "Run is not active"); if (this.env.SCHEDULER) await this.env.SCHEDULER.get(this.env.SCHEDULER.idFromName("global")).fetch("https://scheduler/wake", { method: "POST" }); return stopped; } + async retryBlockedLaunch(runId: string) { + await this.authorizeOwnerSession("runs:cancel"); + if (!await new RunRepository(databaseFor(this.env)).retryBlockedLaunch(this.auth.tenantId, runId)) throw new ServiceError(409, "operation_failed", "Run is not waiting for a manual launch retry"); + if (this.env.SCHEDULER) await this.env.SCHEDULER.get(this.env.SCHEDULER.idFromName("global")).fetch("https://scheduler/wake", { method: "POST" }); + return { runId, retryRequested: true }; + } killRun(runId: string) { return this.stopRun(runId); } async listJobSummaries(input: JobPageQuery, selector = false) { await this.authorize("jobs:read"); try { return await new JobSummaryRepository(databaseFor(this.env), this.auth.tenantId).page(input, selector); } catch (error) { if (error instanceof InvalidJobSummaryCursor) throw new ServiceError(400, "invalid_request", error.message); throw error; } } async listJobs() { await this.authorize("jobs:read"); const jobs = await this.jobs().list(), stats = await this.jobStatistics(jobs.map(job => job.id)); return Promise.all(jobs.map(job => this.presentJob(job, stats.get(job.id) ?? { running_count: "0", last_run_state: null }))); } diff --git a/packages/api/src/postgres/run-repository.ts b/packages/api/src/postgres/run-repository.ts index 3e23cd77..60f5d824 100644 --- a/packages/api/src/postgres/run-repository.ts +++ b/packages/api/src/postgres/run-repository.ts @@ -51,7 +51,7 @@ export class RunRepository { async dueForPoll(limit = 40): Promise { const result = await this.database.pool.query(`SELECT r.tenant_id,r.id,r.job_id,r.invocation_id,r.state,r.encrypted_prompt,x.execution_config,x.trace_sources,x.execution_handle,x.execution_backend_kind,x.execution_capabilities,x.execution_diagnostics,x.failure_code,x.failure_reason,x.vm_cleanup_attempt,x.artifact_state,x.result_protocol_version,r.created_at,r.updated_at FROM app.job_runs r JOIN app.jobs j ON j.tenant_id=r.tenant_id AND j.id=r.job_id JOIN app.runs x ON x.tenant_id=r.tenant_id AND x.id=r.id - WHERE r.state IN ('running','blocked','stopping') AND (r.next_poll_at IS NULL OR r.next_poll_at<=now()) ORDER BY r.next_poll_at NULLS FIRST,r.updated_at LIMIT $1`, [limit]); + WHERE r.state IN ('running','blocked','stopping') AND x.execution_handle IS NOT NULL AND (r.next_poll_at IS NULL OR r.next_poll_at<=now()) ORDER BY r.next_poll_at NULLS FIRST,r.updated_at LIMIT $1`, [limit]); return result.rows.map(mapped); } @@ -117,6 +117,33 @@ export class RunRepository { WHERE tenant_id=$1 AND run_id=$2`, [tenantId, runId, seconds, safeDiagnosticText(error instanceof Error ? error.message : "Execution provider cleanup failed")]); } + /** Persist a pre-provisioning gate without creating a VM or releasing the run's queue slot. */ + async blockLaunch(run: PersistedRun, reason: string): Promise { + const safeReason = reason.slice(0, 500); + await this.database.transaction(async client => { + const changed = await client.query(`UPDATE app.job_runs SET state='blocked',launch_lease_expires_at=NULL,next_poll_at=NULL,updated_at=now() + WHERE tenant_id=$1 AND id=$2 AND state='starting' RETURNING id`, [run.tenantId, run.id]); + if (!changed.rowCount) return; + await client.query(`UPDATE app.runs SET state='blocked',failure_source='system',failure_code='proxy_launch_blocked',failure_reason=$3,updated_at=now() + WHERE tenant_id=$1 AND id=$2 AND execution_handle IS NULL`, [run.tenantId, run.id, safeReason]); + await client.query("INSERT INTO app.run_activity(tenant_id,id,run_id,action,detail) VALUES ($1,$2,$3,'launch_blocked',$4)", [run.tenantId, crypto.randomUUID(), run.id, safeReason]); + }); + } + + /** A blocked pre-launch run resumes only after an explicit owner retry. */ + async retryBlockedLaunch(tenantId: string, runId: string): Promise { + return this.database.transaction(async client => { + const changed = await client.query(`UPDATE app.job_runs r SET state='reserved',launch_lease_expires_at=NULL,next_poll_at=NULL,updated_at=now() + FROM app.runs x WHERE r.tenant_id=$1 AND r.id=$2 AND r.tenant_id=x.tenant_id AND r.id=x.id + AND r.state='blocked' AND x.execution_handle IS NULL AND x.failure_code='proxy_launch_blocked' RETURNING r.id`, [tenantId, runId]); + if (!changed.rowCount) return false; + await client.query("UPDATE app.runs SET state='reserved',failure_source=NULL,failure_code=NULL,failure_reason=NULL,updated_at=now() WHERE tenant_id=$1 AND id=$2", [tenantId, runId]); + await client.query("INSERT INTO app.run_activity(tenant_id,id,run_id,action,detail) VALUES ($1,$2,$3,'launch_retry_requested','Owner requested a manual retry')", [tenantId, crypto.randomUUID(), runId]); + await client.query("INSERT INTO app.wake_hints(key,not_before) VALUES ('global',now()) ON CONFLICT (key) DO UPDATE SET not_before=least(app.wake_hints.not_before,excluded.not_before),updated_at=now()"); + return true; + }); + } + /** Persist cleanup ownership before provisioning any external resources. */ async recordLaunchHandle(run: PersistedRun, handle: RunHandle): Promise { const result = await this.database.pool.query(`UPDATE app.runs x SET execution_handle=$3,updated_at=now() diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index b2df5c47..0529c19d 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -22,6 +22,7 @@ import { TraceProjectionRepository, TRACE_PARSER_VERSION } from "./trace-project import { LiveTraceRepository } from "./live-trace-repository"; import { JOB_RESULT_INSTRUCTIONS, parseJobResult } from "../job-result"; import { issueRunCredential, runProxyInstructions } from "../run-credentials"; +import { ProxyLaunchBlockedError } from "../proxy-launch-policy"; export class RunScheduler { private runs: RunRepository; @@ -31,7 +32,12 @@ export class RunScheduler { await this.runs.consumeWakeHint(); await new AutomationScheduler(this.database, this.env).process(); for (const run of await this.runs.dueForPoll()) await this.poll(run).catch(error => this.fail(run, error, "poll")); - for (let count = 0; count < 20; count++) { const run = await this.runs.claimNext(); if (!run) break; await this.launch(run).catch(error => this.fail(run, error, "launch")); } + for (let count = 0; count < 20; count++) { + const run = await this.runs.claimNext(); if (!run) break; + await this.launch(run).catch(error => error instanceof ProxyLaunchBlockedError + ? this.runs.blockLaunch(run, `Credential proxy ${error.reason}; resolve the prerequisite and manually retry this run.`) + : this.fail(run, error, "launch")); + } const retention = new RunRetentionRepository(this.database); await this.reconcileProviderCleanup(); await this.runRetentionMaintenance(retention); diff --git a/packages/api/src/proxy-launch-policy.ts b/packages/api/src/proxy-launch-policy.ts index 0987c6f3..da8cb5ab 100644 --- a/packages/api/src/proxy-launch-policy.ts +++ b/packages/api/src/proxy-launch-policy.ts @@ -19,6 +19,17 @@ export type ProxyLaunchDecision = | { kind: "start_locked" } | { kind: "blocked"; reason: "proxy_disabled" | "proxy_unavailable" | "proxy_locked" | "pending_changes"; retry: "manual" }; +export class ProxyLaunchBlockedError extends Error { + constructor(readonly reason: Extract["reason"]) { + super(`Job launch is blocked until proxy prerequisite is resolved: ${reason}`); + this.name = "ProxyLaunchBlockedError"; + } +} + +export function requireProxyLaunch(decision: ProxyLaunchDecision): void { + if (decision.kind === "blocked") throw new ProxyLaunchBlockedError(decision.reason); +} + /** * Encodes the GEN-2824 launch gate. This function is deliberately side-effect * free: the scheduler must persist a blocked decision and wait for an explicit diff --git a/packages/api/test/proxy-launch-policy.test.ts b/packages/api/test/proxy-launch-policy.test.ts index 81544828..48952ca5 100644 --- a/packages/api/test/proxy-launch-policy.test.ts +++ b/packages/api/test/proxy-launch-policy.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { proxyLaunchDecision, type ProxyLaunchJob, type ProxyLaunchState } from "../src/proxy-launch-policy"; +import { proxyLaunchDecision, ProxyLaunchBlockedError, requireProxyLaunch, type ProxyLaunchJob, type ProxyLaunchState } from "../src/proxy-launch-policy"; const stopped: ProxyLaunchState = { enabled: true, running: false, unlocked: false, pendingChanges: false }; const locked: ProxyLaunchState = { enabled: true, running: true, unlocked: false, pendingChanges: false }; @@ -33,4 +33,10 @@ describe("GEN-2824 proxy launch policy", () => { it("does not auto-start a stopped proxy for credential-dependent jobs", () => { expect(proxyLaunchDecision({ ...flyCode, portableIntegrationIds: ["integration"] }, stopped)).toMatchObject({ kind: "blocked", retry: "manual" }); }); + + it("turns a blocked decision into a scheduler-recognized launch outcome", () => { + expect(() => requireProxyLaunch(proxyLaunchDecision({ ...flyCode, portableIntegrationIds: ["integration"] }, locked))) + .toThrow(new ProxyLaunchBlockedError("proxy_locked")); + expect(() => requireProxyLaunch(proxyLaunchDecision(flyCode, locked))).not.toThrow(); + }); }); diff --git a/packages/api/test/run-launch-block.test.ts b/packages/api/test/run-launch-block.test.ts new file mode 100644 index 00000000..5dc0670e --- /dev/null +++ b/packages/api/test/run-launch-block.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { RunRepository } from "../src/postgres/run-repository"; + +describe("durable proxy-gated launch retry", () => { + it("persists a blocked pre-launch run without polling it as a live execution", async () => { + const statements: string[] = []; + const client = { query: async (sql: string) => { statements.push(sql); return { rowCount: 1, rows: [{ id: "run" }] }; } }; + const database = { transaction: async (fn: (client: any) => Promise) => fn(client), pool: { query: client.query } }; + await new RunRepository(database as any).blockLaunch({ tenantId: "tenant", id: "run" } as any, "proxy_locked"); + expect(statements.some(sql => sql.includes("state='blocked'") && sql.includes("launch_lease_expires_at=NULL"))).toBe(true); + expect(statements.some(sql => sql.includes("execution_handle IS NULL"))).toBe(true); + expect(statements.some(sql => sql.includes("'launch_blocked'"))).toBe(true); + }); + + it("only manually requeues a proxy-gated run that has no execution handle", async () => { + const statements: string[] = []; + const client = { query: async (sql: string) => { statements.push(sql); return { rowCount: 1, rows: [{ id: "run" }] }; } }; + const database = { transaction: async (fn: (client: any) => Promise) => fn(client), pool: { query: client.query } }; + expect(await new RunRepository(database as any).retryBlockedLaunch("tenant", "run")).toBe(true); + expect(statements.some(sql => sql.includes("state='blocked'") && sql.includes("failure_code='proxy_launch_blocked'") && sql.includes("execution_handle IS NULL"))).toBe(true); + expect(statements.some(sql => sql.includes("state='reserved'"))).toBe(true); + expect(statements.some(sql => sql.includes("'launch_retry_requested'"))).toBe(true); + expect(statements.some(sql => sql.includes("wake_hints"))).toBe(true); + }); +}); diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index 32c63f8c..9e04c8a0 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -13417,6 +13417,67 @@ paths: schema: type: string minLength: 1 + /api/v1/runs/{runId}/retry-launch: + post: + summary: Manually retry a blocked launch + description: |- + Retries only a pre-launch proxy-gated run after its prerequisites have been resolved. The run remains blocked until an interactive owner explicitly retries it; unlock or status changes do not automatically launch it. + + Requires the runs:cancel scope. + + An interactive owner session is required; bearer tokens cannot call this operation. + responses: + "200": + description: Manual retry queued + content: + application/json: + schema: + type: object + required: + - runId + - retryRequested + properties: + runId: + type: string + format: uuid + retryRequested: + type: boolean + const: true + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + description: Interactive owner session required + "404": + $ref: "#/components/responses/Error404" + "409": + $ref: "#/components/responses/Error409" + description: Run is not waiting for a manual launch retry + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: post_api_v1_runs_runId_retry_launch + tags: + - runs + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: runs:cancel + x-owner-session-required: true + parameters: + - name: runId + in: path + required: true + schema: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ /api/v1/runs/{runId}/kill: post: summary: Kill run and immediately release its concurrency slot From 28dd751ca62df794faa07866586019b64626af36 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 06:39:02 +0000 Subject: [PATCH 099/102] GEN-2824 report proxy unlock state to scheduler --- .../migrations/0047_proxy_launch_status.sql | 2 ++ .../src/postgres/proxy-control-repository.ts | 12 +++++++- .../src/postgres/run-enrollment-repository.ts | 2 +- packages/api/src/postgres/run-scheduler.ts | 19 +++++++++++- packages/api/src/postgres/schema.ts | 2 +- packages/api/src/proxy-control.ts | 9 +++++- packages/api/test/proxy-control.test.ts | 10 +++++++ packages/api/test/run-proxy-launch.test.ts | 12 ++++++++ .../credential-proxy/src/control-client.mjs | 22 ++++++++++++++ packages/credential-proxy/src/main.mjs | 11 ++++++- .../test/control-client.test.mjs | 29 ++++++++++++++++++- 11 files changed, 123 insertions(+), 7 deletions(-) create mode 100644 packages/api/migrations/0047_proxy_launch_status.sql diff --git a/packages/api/migrations/0047_proxy_launch_status.sql b/packages/api/migrations/0047_proxy_launch_status.sql new file mode 100644 index 00000000..d7bf1f7a --- /dev/null +++ b/packages/api/migrations/0047_proxy_launch_status.sql @@ -0,0 +1,2 @@ +ALTER TABLE app.credential_proxy_states + ADD COLUMN unlocked boolean NOT NULL DEFAULT false; diff --git a/packages/api/src/postgres/proxy-control-repository.ts b/packages/api/src/postgres/proxy-control-repository.ts index 047d0837..84a7be38 100644 --- a/packages/api/src/postgres/proxy-control-repository.ts +++ b/packages/api/src/postgres/proxy-control-repository.ts @@ -37,7 +37,7 @@ export class ProxyControlRepository { const identity = credential.rows[0]; if (!identity) return null; const saved = await client.query(`INSERT INTO app.credential_proxy_states(tenant_id,proxy_identity,boot_epoch) - VALUES ($1,$2,$3) ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity,boot_epoch=excluded.boot_epoch,updated_at=now() + VALUES ($1,$2,$3) ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity,boot_epoch=excluded.boot_epoch,unlocked=false,updated_at=now() WHERE app.credential_proxy_states.proxy_identity=excluded.proxy_identity AND app.credential_proxy_states.disabled_at IS NULL AND app.credential_proxy_states.revoked_at IS NULL RETURNING tenant_id`, [identity.tenant_id, identity.proxy_identity, bootEpoch]); @@ -75,6 +75,16 @@ export class ProxyControlRepository { return row ? { tenantId: row.tenant_id, proxyIdentity: row.proxy_identity, bootEpoch: row.boot_epoch } : null; } + async reportRuntimeStatus(token: string, bootEpoch: string, unlocked: boolean): Promise { + if (!uuid.test(bootEpoch) || typeof unlocked !== "boolean") throw new TypeError("Invalid proxy runtime status"); + const identity = await this.authenticateProxy(token); + if (!identity || identity.bootEpoch !== bootEpoch) return false; + const result = await this.database.pool.query(`UPDATE app.credential_proxy_states SET unlocked=$3,updated_at=now() + WHERE tenant_id=$1 AND proxy_identity=$2 AND boot_epoch=$4 AND disabled_at IS NULL AND revoked_at IS NULL`, + [identity.tenantId, identity.proxyIdentity, unlocked, bootEpoch]); + return Boolean(result.rowCount); + } + async revokeCredential(tenantId: string, proxyIdentity: string): Promise { const result = await this.database.pool.query(`UPDATE app.credential_proxy_control_credentials SET revoked_at=now() WHERE tenant_id=$1 AND proxy_identity=$2 AND revoked_at IS NULL`, [tenantId, proxyIdentity]); diff --git a/packages/api/src/postgres/run-enrollment-repository.ts b/packages/api/src/postgres/run-enrollment-repository.ts index 328ad692..3b4095e5 100644 --- a/packages/api/src/postgres/run-enrollment-repository.ts +++ b/packages/api/src/postgres/run-enrollment-repository.ts @@ -30,7 +30,7 @@ export class RunEnrollmentRepository { if (!input.proxyIdentity.trim() || input.proxyIdentity.length > 255) throw new TypeError("proxyIdentity must contain 1 to 255 characters"); await this.database.pool.query(`INSERT INTO app.credential_proxy_states(tenant_id,proxy_identity,boot_epoch,disabled_at,revoked_at) VALUES ($1,$2,$3,CASE WHEN $4 THEN now() END,CASE WHEN $5 THEN now() END) - ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity,boot_epoch=excluded.boot_epoch,disabled_at=excluded.disabled_at,revoked_at=excluded.revoked_at,updated_at=now()`, + ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity,boot_epoch=excluded.boot_epoch,disabled_at=excluded.disabled_at,revoked_at=excluded.revoked_at,unlocked=false,updated_at=now()`, [input.tenantId, input.proxyIdentity, input.bootEpoch, input.disabled ?? false, input.revoked ?? false]); } diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index 0529c19d..b735147e 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -22,7 +22,7 @@ import { TraceProjectionRepository, TRACE_PARSER_VERSION } from "./trace-project import { LiveTraceRepository } from "./live-trace-repository"; import { JOB_RESULT_INSTRUCTIONS, parseJobResult } from "../job-result"; import { issueRunCredential, runProxyInstructions } from "../run-credentials"; -import { ProxyLaunchBlockedError } from "../proxy-launch-policy"; +import { ProxyLaunchBlockedError, proxyLaunchDecision, requireProxyLaunch } from "../proxy-launch-policy"; export class RunScheduler { private runs: RunRepository; @@ -127,6 +127,15 @@ export class RunScheduler { } private async launch(run: PersistedRun) { + const config = run.executionConfig; + const decision = proxyLaunchDecision({ + executionProvider: config.executionProvider === "fly" ? "fly" : "exe", + executionType: config.executionType === "code" ? "code" : "agent", + portableIntegrationIds: config.portableIntegrationIds, + modelAccess: config.modelAccess, + modelAccountId: config.modelAccountId, + }, await this.proxyLaunchState(run.tenantId)); + requireProxyLaunch(decision); if (run.executionConfig.executionType === "code") { if (typeof run.executionConfig.script !== "string" || !run.executionConfig.script.trim()) throw new Error("The code job has no saved script"); const backend = await this.backend(run), context = run.executionConfig.inputContext; @@ -152,6 +161,14 @@ export class RunScheduler { if (launched.observation.state === "failed") await this.runs.recordObservation(run, launched.observation); } + private async proxyLaunchState(tenantId: string) { + const result = await this.database.pool.query<{ enabled: boolean; unlocked: boolean }>(`SELECT disabled_at IS NULL AND revoked_at IS NULL AS enabled, + unlocked AND updated_at > now() - interval '30 seconds' AS unlocked + FROM app.credential_proxy_states WHERE tenant_id=$1`, [tenantId]); + const state = result.rows[0]; + return { enabled: Boolean(state?.enabled), running: Boolean(state), unlocked: Boolean(state?.unlocked), pendingChanges: false }; + } + private async poll(run: PersistedRun) { if (!run.executionHandle && run.state === "stopping") return this.runs.schedulePoll(run, "stopping"); const backend = await this.backend(run); if (!run.executionHandle) throw new Error("Execution handle is missing"); diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 89dc11ce..880433c4 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -51,7 +51,7 @@ export const integrationChanges = app.table("integration_changes", { tenantId: uuid("tenant_id").notNull().references(() => integrationCollections.tenantId, { onDelete: "cascade" }), revision: bigint("revision", { mode: "number" }).notNull(), id: uuid("id").notNull(), idempotencyKey: uuid("idempotency_key").notNull(), operation: text("operation").$type<"upsert" | "delete">().notNull(), recordId: uuid("record_id").notNull(), kind: text("kind").$type<"bearer" | "model">().notNull(), name: text("name").notNull(), origin: text("origin"), description: text("description"), encryptedPayload: text("encrypted_payload"), createdAt: createdAt(), }, t => [primaryKey({ columns: [t.tenantId, t.revision] }), unique().on(t.tenantId, t.idempotencyKey), index("integration_changes_page").on(t.tenantId, t.revision), check("integration_changes_operation_check", sql`${t.operation} IN ('upsert','delete')`), check("integration_changes_kind_check", sql`${t.kind} IN ('bearer','model')`), check("integration_changes_origin_check", sql`(${t.kind} = 'bearer' AND ${t.origin} IS NOT NULL) OR (${t.kind} = 'model' AND ${t.origin} IS NULL)`), check("integration_changes_payload_check", sql`(${t.operation} = 'upsert' AND ${t.encryptedPayload} IS NOT NULL) OR (${t.operation} = 'delete' AND ${t.encryptedPayload} IS NULL)`)]); export const credentialProxyStates = app.table("credential_proxy_states", { - tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), disabledAt: timestamp("disabled_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), updatedAt: updatedAt(), + tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), proxyIdentity: text("proxy_identity").notNull(), bootEpoch: uuid("boot_epoch").notNull(), disabledAt: timestamp("disabled_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), unlocked: boolean("unlocked").notNull().default(false), updatedAt: updatedAt(), }); export const credentialProxyBrowserPairings = app.table("credential_proxy_browser_pairings", { tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), diff --git a/packages/api/src/proxy-control.ts b/packages/api/src/proxy-control.ts index 64db6ad6..1e136ffd 100644 --- a/packages/api/src/proxy-control.ts +++ b/packages/api/src/proxy-control.ts @@ -9,7 +9,7 @@ const json = (body: unknown, status = 200) => Response.json(body, { status, head /** Dedicated service-to-service endpoint. The run API credential is intentionally not accepted here. */ export async function proxyControlFetch(request: Request, env: Parameters[0]): Promise { const url = new URL(request.url); - if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot", "/internal/credential-proxy/model-account-cache", "/internal/credential-proxy/revocations"].includes(url.pathname)) return json({ error: "not_found" }, 404); + if (!["/internal/credential-proxy/register-boot", "/internal/credential-proxy/status", "/internal/credential-proxy/enroll", "/internal/credential-proxy/consume-enrollment", "/internal/credential-proxy/integration-snapshot", "/internal/credential-proxy/model-account-cache", "/internal/credential-proxy/revocations"].includes(url.pathname)) return json({ error: "not_found" }, 404); if (request.method !== "POST") return json({ error: "method_not_allowed" }, 405); if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return json({ error: "unsupported_media_type" }, 415); const authorization = request.headers.get("authorization") ?? ""; @@ -40,6 +40,13 @@ export async function proxyControlFetch(request: Request, env: Parameters sql.includes("proxy_token_digest") ? { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", proxy_identity: "proxy-fixture", boot_epoch: bootEpoch }] : [], rowCount: credentialFound ? 1 : 0 } + : sql.includes("SET unlocked=$3") ? { rows: [], rowCount: credentialFound ? 1 : 0 } : sql.includes("FROM app.run_credentials") ? { rows: credentialFound ? [{ tenant_id: "11111111-1111-4111-8111-111111111111", user_id: "owner-fixture", session_version: 1, job_id: "job-fixture", scopes: ["integrations:read"] }] : [], rowCount: credentialFound ? 1 : 0 } : { rows: [], rowCount: 0 }); @@ -44,6 +45,15 @@ describe("dedicated proxy boot registration endpoint", () => { expect(env.transactionQuery).not.toHaveBeenCalled(); }); + it("accepts runtime status only from the current registered proxy credential", async () => { + const env = envFor(); + const response = await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/status", body: { bootEpoch, unlocked: true } }), env); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ updated: true }); + expect(env.query.mock.calls.some((call: any[]) => call[0].includes("SET unlocked=$3") && call[0].includes("boot_epoch=$4"))).toBe(true); + expect((await proxyControlFetch(request({ authorization: `Bearer ${token}`, path: "/internal/credential-proxy/status", body: { bootEpoch, unlocked: "yes" } }), env)).status).toBe(400); + }); + it("rejects revoked credentials, disabled proxy identities and tenant or proxy substitution", async () => { for (const env of [envFor({ credentialFound: false }), envFor({ stateSaved: false })]) { const response = await proxyControlFetch(request({ authorization: `Bearer ${token}` }), env); diff --git a/packages/api/test/run-proxy-launch.test.ts b/packages/api/test/run-proxy-launch.test.ts index 7c23e64d..299f3e6e 100644 --- a/packages/api/test/run-proxy-launch.test.ts +++ b/packages/api/test/run-proxy-launch.test.ts @@ -4,10 +4,21 @@ import { RunRepository } from "../src/postgres/run-repository"; import { ExeVmBackend } from "../src/exe-vm-backend"; import { encrypt } from "../src/crypto"; import { runProxyName } from "../src/run-credentials"; +import { ProxyLaunchBlockedError } from "../src/proxy-launch-policy"; afterEach(() => vi.restoreAllMocks()); describe("control-plane run credential delivery", () => { + it("persists a pre-launch block when a credential-dependent job has no fresh unlocked proxy status", async () => { + const database = { pool: { query: vi.fn() } }; + const scheduler: any = new RunScheduler(database as any, {} as any); + vi.spyOn(scheduler, "proxyLaunchState").mockResolvedValue({ enabled: true, running: true, unlocked: false, pendingChanges: false }); + await expect(scheduler.launch({ + id: "00000000-0000-4000-8000-000000000001", tenantId: "tenant", + executionConfig: { executionProvider: "fly", executionType: "agent", modelAccess: "portable", modelAccountId: "account" }, + })).rejects.toEqual(new ProxyLaunchBlockedError("proxy_locked")); + }); + it.each(["codex", "claude", "pi", "code"])("keeps %s proxy credentials out of the guest environment and prompt", async agentKind => { const runId = "00000000-0000-4000-8000-000000000001", key = btoa("a".repeat(32)); let issuedScopes: unknown; @@ -18,6 +29,7 @@ describe("control-plane run credential delivery", () => { } } }; const env = { DATABASE: database, APP_ORIGIN: "https://app.test", CREDENTIAL_ENCRYPTION_KEY: key } as any; const scheduler: any = new RunScheduler(database as any, env); + vi.spyOn(scheduler, "proxyLaunchState").mockResolvedValue({ enabled: true, running: true, unlocked: true, pendingChanges: false }); const backend = new ExeVmBackend({ apiToken: "control-plane-token", agentKind: (agentKind === "code" ? "codex" : agentKind) as any, tags: [] }); vi.spyOn(scheduler, "backend").mockResolvedValue(backend); const launch = vi.spyOn(backend, "launch").mockResolvedValue({ handle: { backendKind: "exe-vm", id: "factorize-run" }, observation: { state: "running" }, capabilities: [], destinationUrl: "https://vm.test" }); diff --git a/packages/credential-proxy/src/control-client.mjs b/packages/credential-proxy/src/control-client.mjs index 0a4c889e..0d74e3fd 100644 --- a/packages/credential-proxy/src/control-client.mjs +++ b/packages/credential-proxy/src/control-client.mjs @@ -63,6 +63,28 @@ export async function consumeProxyEnrollment(options) { return result.binding; } +export async function reportProxyRuntimeStatus({ controlOrigin, token, bootEpoch, unlocked, fetchImpl = fetch } = {}) { + if (!bootEpochPattern.test(bootEpoch ?? "") || typeof unlocked !== "boolean") throw new TypeError("invalid proxy runtime status"); + const result = await proxyControlRequest({ controlOrigin, token, path: "/internal/credential-proxy/status", body: { bootEpoch, unlocked }, fetchImpl }); + if (result?.updated !== true) throw new Error("invalid proxy runtime status response"); + return result; +} + +export function startProxyRuntimeStatusReporter({ controlOrigin, token, bootEpoch, runtime, intervalMs = 10_000, fetchImpl = fetch } = {}) { + if (!runtime || typeof runtime.state !== "string" || !Number.isSafeInteger(intervalMs) || intervalMs < 1_000 || intervalMs > 60_000) { + throw new TypeError("invalid proxy runtime status reporter"); + } + let closed = false, pending = Promise.resolve(); + const report = () => { + if (closed) return; + pending = pending.then(() => reportProxyRuntimeStatus({ controlOrigin, token, bootEpoch, unlocked: runtime.state === "unlocked", fetchImpl })).catch(() => {}); + }; + report(); + const timer = setInterval(report, intervalMs); + timer.unref?.(); + return { async close() { closed = true; clearInterval(timer); await pending; } }; +} + /** Read the current user-encrypted integration records selected by an active run. * The control plane decrypts only Factorize's outer encryption layer. */ export async function readProxyIntegrationSnapshot({ controlOrigin, token, binding, fetchImpl = fetch } = {}) { diff --git a/packages/credential-proxy/src/main.mjs b/packages/credential-proxy/src/main.mjs index 50f1ba2c..773f1d3b 100644 --- a/packages/credential-proxy/src/main.mjs +++ b/packages/credential-proxy/src/main.mjs @@ -1,7 +1,7 @@ import { createInterface } from "node:readline"; import { randomUUID } from "node:crypto"; import { createProxyRuntime } from "./runtime.mjs"; -import { registerProxyBoot, startProxyRunRevocationMonitor } from "./control-client.mjs"; +import { registerProxyBoot, startProxyRunRevocationMonitor, startProxyRuntimeStatusReporter } from "./control-client.mjs"; import { startProxyBootstrapControl } from "./bootstrap-control.mjs"; import { createProxyIntegrationGrantResolver } from "./decrypted-grants.mjs"; import { startAuthenticatedProxyTunnel } from "./tunnel-transport.mjs"; @@ -25,6 +25,7 @@ let controlCredential; let controlOrigin; let tunnel; let revocationMonitor; +let statusReporter; let modelAccounts; let modelAccountLifecycle; let browserControl; @@ -75,6 +76,12 @@ async function activateRegistration(registration, origin, browserOrigin) { bootEpoch: runtime.bootEpoch, runtime, }); + statusReporter = startProxyRuntimeStatusReporter({ + controlOrigin: origin, + token: registration.controlCredential, + bootEpoch: runtime.bootEpoch, + runtime, + }); } const bootstrapControl = await startProxyBootstrapControl({ @@ -105,6 +112,8 @@ async function stop() { controlOrigin = undefined; await revocationMonitor?.close(); revocationMonitor = undefined; + await statusReporter?.close(); + statusReporter = undefined; await browserControl?.close().catch(() => {}); browserControl = undefined; await runtime.shutdown(); diff --git a/packages/credential-proxy/test/control-client.test.mjs b/packages/credential-proxy/test/control-client.test.mjs index 449cf991..fad390cb 100644 --- a/packages/credential-proxy/test/control-client.test.mjs +++ b/packages/credential-proxy/test/control-client.test.mjs @@ -1,12 +1,39 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { consumeProxyEnrollment, createModelAccountPersistenceAdapter, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, readProxyRunRevocations, registerProxyBoot, startProxyRunRevocationMonitor } from "../src/control-client.mjs"; +import { consumeProxyEnrollment, createModelAccountPersistenceAdapter, createProxyEnrollmentAuthenticator, issueProxyEnrollment, readProxyIntegrationSnapshot, readProxyRunRevocations, registerProxyBoot, reportProxyRuntimeStatus, startProxyRuntimeStatusReporter, startProxyRunRevocationMonitor } from "../src/control-client.mjs"; const token = `fzc_${"A".repeat(43)}`; const bootEpoch = "22222222-2222-4222-8222-222222222222"; const runId = "33333333-3333-4333-8333-333333333333"; const runCredential = `fzt_run_${Buffer.from(runId).toString("base64url")}_${"R".repeat(43)}`; +test("proxy runtime status reports the current boot and runtime unlock state", async () => { + const requests = []; + await reportProxyRuntimeStatus({ controlOrigin: "https://factorize.example", token, bootEpoch, unlocked: true, fetchImpl: async (url, init) => { + requests.push({ url: String(url), init, body: JSON.parse(init.body) }); + return Response.json({ updated: true }); + } }); + assert.equal(requests[0].url, "https://factorize.example/internal/credential-proxy/status"); + assert.equal(requests[0].init.headers.Authorization, `Bearer ${token}`); + assert.deepEqual(requests[0].body, { bootEpoch, unlocked: true }); + await assert.rejects(reportProxyRuntimeStatus({ controlOrigin: "https://factorize.example", token, bootEpoch: "invalid", unlocked: false }), /runtime status/); +}); + +test("runtime status reporter periodically refreshes authenticated status", async () => { + const requests = []; + const runtime = { state: "locked" }; + const monitor = startProxyRuntimeStatusReporter({ controlOrigin: "https://factorize.example", token, bootEpoch, runtime, intervalMs: 1_000, fetchImpl: async (url, init) => { + requests.push({ url: String(url), body: JSON.parse(init.body) }); + return Response.json({ updated: true }); + } }); + await new Promise(resolve => setTimeout(resolve, 5)); + runtime.state = "unlocked"; + await new Promise(resolve => setTimeout(resolve, 5)); + await monitor.close(); + assert.equal(requests[0].body.unlocked, false); + assert.ok(requests.every(request => request.url.endsWith("/internal/credential-proxy/status"))); +}); + test("boot registration uses a one-call control credential, HTTPS, exact path and fresh process epoch", async () => { let request; const result = await registerProxyBoot({ controlOrigin: "https://factorize.example", browserOrigin: "https://proxy.example", token, bootEpoch, fetchImpl: async (url, init) => { From f2ac3f91b1a00a0a59580ff2f5a1b5fccf59846a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 06:56:50 +0000 Subject: [PATCH 100/102] GEN-2824 persist blocked proxy launch state --- packages/api/src/postgres/run-repository.ts | 2 +- .../test/run-proxy-launch.postgres.test.ts | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 packages/api/test/run-proxy-launch.postgres.test.ts diff --git a/packages/api/src/postgres/run-repository.ts b/packages/api/src/postgres/run-repository.ts index 029d0946..a1c28cda 100644 --- a/packages/api/src/postgres/run-repository.ts +++ b/packages/api/src/postgres/run-repository.ts @@ -62,7 +62,7 @@ export class RunRepository { const changed = await client.query(`UPDATE app.job_runs SET state='blocked',launch_lease_expires_at=NULL,next_poll_at=NULL,updated_at=now() WHERE tenant_id=$1 AND id=$2 AND state='starting' RETURNING id`, [run.tenantId, run.id]); if (!changed.rowCount) return; - await client.query(`UPDATE app.runs SET state='blocked',failure_source='system',failure_code='proxy_launch_blocked',failure_reason=$3,updated_at=now() + await client.query(`UPDATE app.runs SET state='blocked',failure_code='proxy_launch_blocked',failure_reason=$3,updated_at=now() WHERE tenant_id=$1 AND id=$2 AND execution_handle IS NULL`, [run.tenantId, run.id, safeReason]); await client.query("INSERT INTO app.run_activity(tenant_id,id,run_id,action,detail) VALUES ($1,$2,$3,'launch_blocked',$4)", [run.tenantId, crypto.randomUUID(), run.id, safeReason]); }); diff --git a/packages/api/test/run-proxy-launch.postgres.test.ts b/packages/api/test/run-proxy-launch.postgres.test.ts new file mode 100644 index 00000000..fffc058c --- /dev/null +++ b/packages/api/test/run-proxy-launch.postgres.test.ts @@ -0,0 +1,51 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { loadingFixture } from "./helpers/loading-fixture"; +import { RunScheduler } from "../src/postgres/run-scheduler"; +import { protectedApiFetch } from "../src/protected-api"; + +vi.mock("cloudflare:workers", () => ({ WorkerEntrypoint: class {}, DurableObject: class {} })); +vi.mock("@cloudflare/workers-oauth-provider", () => ({ OAuthProvider: class {}, AuthorizationError: class extends Error {}, getOAuthApi: vi.fn() })); + +const url = process.env.AUTH_TEST_DATABASE_URL; +describe.skipIf(!url)("proxy-gated scheduler lifecycle (PostgreSQL/API)", () => { + let fixture: Awaited>; + beforeAll(async () => { fixture = await loadingFixture(url!); }); + afterAll(async () => { await fixture?.cleanup(); }); + + it("keeps a blocked launch parked after unlock until the owner retries through the API", async () => { + const { db, env, auth, tenantId, runId } = fixture; + const proxyIdentity = `proxy-${crypto.randomUUID()}`; + await db.pool.query(`INSERT INTO app.credential_proxy_states(tenant_id,proxy_identity,boot_epoch,unlocked) + VALUES ($1,$2,$3,false)`, [tenantId, proxyIdentity, crypto.randomUUID()]); + await db.pool.query(`UPDATE app.runs SET execution_config=$3 + WHERE tenant_id=$1 AND id=$2`, [tenantId, runId, { + agentKind: "codex", connectionId: "local", executionProvider: "fly", executionType: "agent", + modelAccess: "portable", modelAccountId: crypto.randomUUID(), portableIntegrationIds: [], runScopes: [], + }]); + + await new RunScheduler(db, env).process(); + const blocked = (await db.pool.query(`SELECT jr.state job_state,r.state run_state,r.failure_code,r.failure_reason, + r.execution_handle,jr.launch_lease_expires_at + FROM app.job_runs jr JOIN app.runs r USING (tenant_id,id) WHERE jr.tenant_id=$1 AND jr.id=$2`, [tenantId, runId])).rows[0]; + expect(blocked).toMatchObject({ job_state: "blocked", run_state: "blocked", failure_code: "proxy_launch_blocked", execution_handle: null, launch_lease_expires_at: null }); + expect(blocked.failure_reason).toContain("proxy_locked"); + expect((await db.pool.query("SELECT action FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2 ORDER BY created_at", [tenantId, runId])).rows).toEqual([{ action: "launch_blocked" }]); + + await db.pool.query("UPDATE app.credential_proxy_states SET unlocked=true,updated_at=now() WHERE tenant_id=$1", [tenantId]); + await new RunScheduler(db, env).process(); + expect((await db.pool.query("SELECT state FROM app.job_runs WHERE tenant_id=$1 AND id=$2", [tenantId, runId])).rows[0].state).toBe("blocked"); + expect((await db.pool.query("SELECT action FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2 ORDER BY created_at", [tenantId, runId])).rows).toEqual([{ action: "launch_blocked" }]); + + const request = () => protectedApiFetch(new Request(`${env.APP_ORIGIN}/api/v1/runs/${runId}/retry-launch`, { + method: "POST", headers: { Origin: env.APP_ORIGIN }, + }), env, { ...auth, authMethod: "session", scopes: ["runs:cancel"] }, { waitUntil() {} } as any); + const retried = await request(); + expect(retried.status).toBe(200); + expect(await retried.json()).toEqual({ runId, retryRequested: true }); + expect((await db.pool.query(`SELECT jr.state job_state,r.state run_state,r.failure_code,r.execution_handle + FROM app.job_runs jr JOIN app.runs r USING (tenant_id,id) WHERE jr.tenant_id=$1 AND jr.id=$2`, [tenantId, runId])).rows[0]).toMatchObject({ job_state: "reserved", run_state: "reserved", failure_code: null, execution_handle: null }); + expect((await db.pool.query("SELECT action FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2 ORDER BY created_at", [tenantId, runId])).rows.map((row: any) => row.action)).toEqual(["launch_blocked", "launch_retry_requested"]); + expect((await db.pool.query("SELECT key FROM app.wake_hints WHERE key='global'")).rowCount).toBe(1); + expect((await request()).status).toBe(409); + }, 30_000); +}); From d529ab1d839cf8224ec69c548624f9af85b13a2c Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 07:07:14 +0000 Subject: [PATCH 101/102] GEN-2824 cover proxy launch lifecycle states --- .../test/run-proxy-launch.postgres.test.ts | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/api/test/run-proxy-launch.postgres.test.ts b/packages/api/test/run-proxy-launch.postgres.test.ts index fffc058c..3386193c 100644 --- a/packages/api/test/run-proxy-launch.postgres.test.ts +++ b/packages/api/test/run-proxy-launch.postgres.test.ts @@ -12,30 +12,50 @@ describe.skipIf(!url)("proxy-gated scheduler lifecycle (PostgreSQL/API)", () => beforeAll(async () => { fixture = await loadingFixture(url!); }); afterAll(async () => { await fixture?.cleanup(); }); - it("keeps a blocked launch parked after unlock until the owner retries through the API", async () => { + it("keeps launches blocked for missing, disabled, revoked, stale, or locked proxy state until the owner retries", async () => { const { db, env, auth, tenantId, runId } = fixture; const proxyIdentity = `proxy-${crypto.randomUUID()}`; - await db.pool.query(`INSERT INTO app.credential_proxy_states(tenant_id,proxy_identity,boot_epoch,unlocked) - VALUES ($1,$2,$3,false)`, [tenantId, proxyIdentity, crypto.randomUUID()]); await db.pool.query(`UPDATE app.runs SET execution_config=$3 WHERE tenant_id=$1 AND id=$2`, [tenantId, runId, { agentKind: "codex", connectionId: "local", executionProvider: "fly", executionType: "agent", modelAccess: "portable", modelAccountId: crypto.randomUUID(), portableIntegrationIds: [], runScopes: [], }]); - await new RunScheduler(db, env).process(); - const blocked = (await db.pool.query(`SELECT jr.state job_state,r.state run_state,r.failure_code,r.failure_reason, - r.execution_handle,jr.launch_lease_expires_at - FROM app.job_runs jr JOIN app.runs r USING (tenant_id,id) WHERE jr.tenant_id=$1 AND jr.id=$2`, [tenantId, runId])).rows[0]; - expect(blocked).toMatchObject({ job_state: "blocked", run_state: "blocked", failure_code: "proxy_launch_blocked", execution_handle: null, launch_lease_expires_at: null }); - expect(blocked.failure_reason).toContain("proxy_locked"); - expect((await db.pool.query("SELECT action FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2 ORDER BY created_at", [tenantId, runId])).rows).toEqual([{ action: "launch_blocked" }]); + const cases = [ + { label: "no registered proxy", state: "missing", reason: "proxy_disabled" }, + { label: "disabled proxy", state: "disabled", reason: "proxy_disabled" }, + { label: "revoked proxy", state: "revoked", reason: "proxy_disabled" }, + { label: "stale unlock heartbeat", state: "stale", reason: "proxy_locked" }, + { label: "locked proxy", state: "locked", reason: "proxy_locked" }, + ] as const; + for (const scenario of cases) { + await db.pool.query("DELETE FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2", [tenantId, runId]); + await db.pool.query(`UPDATE app.job_runs SET state='reserved',launch_lease_expires_at=NULL,next_poll_at=NULL WHERE tenant_id=$1 AND id=$2`, [tenantId, runId]); + await db.pool.query(`UPDATE app.runs SET state='reserved',failure_source=NULL,failure_code=NULL,failure_reason=NULL,execution_handle=NULL WHERE tenant_id=$1 AND id=$2`, [tenantId, runId]); + if (scenario.state === "missing") { + await db.pool.query("DELETE FROM app.credential_proxy_states WHERE tenant_id=$1", [tenantId]); + } else { + const disabled = scenario.state === "disabled" ? "now()" : "NULL"; + const revoked = scenario.state === "revoked" ? "now()" : "NULL"; + const unlocked = scenario.state === "stale" ? "true" : "false"; + const heartbeat = scenario.state === "stale" ? "now()-interval '1 minute'" : "now()"; + await db.pool.query(`INSERT INTO app.credential_proxy_states(tenant_id,proxy_identity,boot_epoch,disabled_at,revoked_at,unlocked,updated_at) + VALUES ($1,$2,$3,${disabled},${revoked},${unlocked},${heartbeat}) + ON CONFLICT (tenant_id) DO UPDATE SET proxy_identity=excluded.proxy_identity,boot_epoch=excluded.boot_epoch, + disabled_at=excluded.disabled_at,revoked_at=excluded.revoked_at,unlocked=excluded.unlocked,updated_at=excluded.updated_at`, + [tenantId, proxyIdentity, crypto.randomUUID()]); + } - await db.pool.query("UPDATE app.credential_proxy_states SET unlocked=true,updated_at=now() WHERE tenant_id=$1", [tenantId]); - await new RunScheduler(db, env).process(); - expect((await db.pool.query("SELECT state FROM app.job_runs WHERE tenant_id=$1 AND id=$2", [tenantId, runId])).rows[0].state).toBe("blocked"); - expect((await db.pool.query("SELECT action FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2 ORDER BY created_at", [tenantId, runId])).rows).toEqual([{ action: "launch_blocked" }]); + await new RunScheduler(db, env).process(); + const blocked = (await db.pool.query(`SELECT jr.state job_state,r.state run_state,r.failure_code,r.failure_reason, + r.execution_handle,jr.launch_lease_expires_at + FROM app.job_runs jr JOIN app.runs r USING (tenant_id,id) WHERE jr.tenant_id=$1 AND jr.id=$2`, [tenantId, runId])).rows[0]; + expect(blocked, scenario.label).toMatchObject({ job_state: "blocked", run_state: "blocked", failure_code: "proxy_launch_blocked", execution_handle: null, launch_lease_expires_at: null }); + expect(blocked.failure_reason, scenario.label).toContain(scenario.reason); + expect((await db.pool.query("SELECT action FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2 ORDER BY created_at", [tenantId, runId])).rows, scenario.label).toEqual([{ action: "launch_blocked" }]); + } + await db.pool.query("UPDATE app.credential_proxy_states SET unlocked=true,updated_at=now(),disabled_at=NULL,revoked_at=NULL WHERE tenant_id=$1", [tenantId]); const request = () => protectedApiFetch(new Request(`${env.APP_ORIGIN}/api/v1/runs/${runId}/retry-launch`, { method: "POST", headers: { Origin: env.APP_ORIGIN }, }), env, { ...auth, authMethod: "session", scopes: ["runs:cancel"] }, { waitUntil() {} } as any); From 542a82c66992b291f546300a3280d9edc78c521b Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 07:27:45 +0000 Subject: [PATCH 102/102] GEN-2824 recheck proxy gate after manual retry --- packages/api/test/run-proxy-launch.postgres.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/api/test/run-proxy-launch.postgres.test.ts b/packages/api/test/run-proxy-launch.postgres.test.ts index 3386193c..3bfbf500 100644 --- a/packages/api/test/run-proxy-launch.postgres.test.ts +++ b/packages/api/test/run-proxy-launch.postgres.test.ts @@ -67,5 +67,13 @@ describe.skipIf(!url)("proxy-gated scheduler lifecycle (PostgreSQL/API)", () => expect((await db.pool.query("SELECT action FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2 ORDER BY created_at", [tenantId, runId])).rows.map((row: any) => row.action)).toEqual(["launch_blocked", "launch_retry_requested"]); expect((await db.pool.query("SELECT key FROM app.wake_hints WHERE key='global'")).rowCount).toBe(1); expect((await request()).status).toBe(409); + + // The explicit retry only requeues the run. If proxy state changes before + // the scheduler claims it, the launch gate must block it again. + await db.pool.query("UPDATE app.credential_proxy_states SET unlocked=false,updated_at=now() WHERE tenant_id=$1", [tenantId]); + await new RunScheduler(db, env).process(); + expect((await db.pool.query(`SELECT jr.state job_state,r.state run_state,r.failure_code,r.execution_handle + FROM app.job_runs jr JOIN app.runs r USING (tenant_id,id) WHERE jr.tenant_id=$1 AND jr.id=$2`, [tenantId, runId])).rows[0]).toMatchObject({ job_state: "blocked", run_state: "blocked", failure_code: "proxy_launch_blocked", execution_handle: null }); + expect((await db.pool.query("SELECT action FROM app.run_activity WHERE tenant_id=$1 AND run_id=$2 ORDER BY created_at", [tenantId, runId])).rows.map((row: any) => row.action)).toEqual(["launch_blocked", "launch_retry_requested", "launch_blocked"]); }, 30_000); });