diff --git a/src/ml/session/onnx-node.ts b/src/ml/session/onnx-node.ts index 44f6870..a33b577 100644 --- a/src/ml/session/onnx-node.ts +++ b/src/ml/session/onnx-node.ts @@ -44,6 +44,20 @@ async function loadOrt(): Promise { } } +/** + * float16 feeds cannot cross the onnxruntime-node binding on runtimes + * with native Float16Array (Node >= 24): ORT's JS Tensor converts every + * float16 feed into a Float16Array view, and the N-API binding reads + * that buffer as zero bytes, so session.run fails with the opaque + * "not enough space: expected N, got 0" (issue #111). On pre-Float16Array + * runtimes the same feed stays Uint16Array-backed and is consumed fine + * (verified Node 20/22 ok, Node 24/26 broken). Artifacts with float16 + * graph I/O (the pre-WO79 fp16 exports) must be IO-normalized instead: + * interscript-models tools/normalize_fp16_io.py. + */ +const FLOAT16_FEEDS_CONSUMABLE = + typeof (globalThis as { Float16Array?: unknown }).Float16Array === "undefined" + class NodeInferenceSession implements InferenceSession { private readonly session: OnnxNodeSession private readonly ort: OnnxNodeModule @@ -76,6 +90,15 @@ class NodeInferenceSession implements InferenceSession { async run(inputs: InferenceInputs): Promise { const feeds: Record = {} for (const [name, tensor] of Object.entries(inputs)) { + if (tensor.type === "float16" && !FLOAT16_FEEDS_CONSUMABLE) { + throw new Error( + `input '${name}' is float16: float16 graph I/O cannot cross the onnxruntime-node ` + + `binding on this runtime (the feed is converted to a Float16Array the binding reads ` + + `as zero bytes, surfacing as "not enough space: expected N, got 0"). The artifact's ` + + `graphs must expose fp32 graph I/O — re-export with interscript-models ` + + `tools/normalize_fp16_io.py.`, + ) + } feeds[name] = new this.ort.Tensor(tensor.type, tensor.data, tensor.dims) } const outputs = await this.session.run(feeds) diff --git a/src/ml/types.ts b/src/ml/types.ts index 19459d2..78785df 100644 --- a/src/ml/types.ts +++ b/src/ml/types.ts @@ -27,13 +27,21 @@ export interface ModelRef { * Record of named tensors. */ export type TensorData = - Int8Array | Uint8Array | Int16Array | Int32Array | Float32Array | Float64Array | BigInt64Array + | Int8Array + | Uint8Array + | Int16Array + | Uint16Array + | Int32Array + | Float32Array + | Float64Array + | BigInt64Array export interface Tensor { readonly name: string readonly data: TensorData readonly dims: readonly number[] - readonly type: "int8" | "uint8" | "int16" | "int32" | "int64" | "float32" | "float64" + readonly type: + "int8" | "uint8" | "int16" | "uint16" | "int32" | "int64" | "float32" | "float64" | "float16" } export interface InferenceInputs { diff --git a/test/fixtures/tiny-fp16io.zip b/test/fixtures/tiny-fp16io.zip new file mode 100644 index 0000000..d3d7c7a Binary files /dev/null and b/test/fixtures/tiny-fp16io.zip differ diff --git a/test/imf-fp16io.test.ts b/test/imf-fp16io.test.ts new file mode 100644 index 0000000..a6ed7ac --- /dev/null +++ b/test/imf-fp16io.test.ts @@ -0,0 +1,56 @@ +/** + * The fp16 graph-I/O contract guard (issue #111). The original fp16 + * exports shipped float16 graph I/O (last_hidden_state / + * encoder_hidden_states / logits all float16). Python ORT consumes + * fp16 I/O natively, so the reference decoder and the frozen vectors + * never saw it — but onnxruntime-node cannot: on runtimes with native + * Float16Array (Node >= 24) every float16 feed is converted to a + * Float16Array view whose bytes the N-API binding reads as zero, and + * session.run fails with the opaque "not enough space: expected N, + * got 0". On pre-Float16Array runtimes the same feed stays + * Uint16Array-backed and decodes fine (verified: Node 20/22 ok, Node + * 24/26 broken). + * + * The runtime therefore rejects the unconsumable condition explicitly, + * naming the artifact contract: graphs must expose fp32 I/O + * (interscript-models tools/normalize_fp16_io.py, WO79 — the shipped + * fp16 artifacts have been IO-normalized since 2026-10-21). + * + * The fixture is tiny-imf.zip with float16 wrapped across both graph + * boundaries (Cast nodes, recomputed member digests) — the pre-WO79 + * export shape in ~1KB instead of 700MB. + */ + +import { describe, expect, it } from "vitest" +import { IMFModel } from "../src/ml/imf/index.js" + +const fixtureZip = "test/fixtures/tiny-fp16io.zip" + +/** Node >= 24 ships native Float16Array; ORT converts fp16 feeds to + * Float16Array views the onnxruntime-node binding cannot consume. */ +const float16FeedsUnconsumable = + typeof (globalThis as { Float16Array?: unknown }).Float16Array !== "undefined" + +describe("fp16 graph-I/O artifacts (issue #111)", () => { + it( + float16FeedsUnconsumable + ? "rejects with the artifact-contract error, not the opaque binding failure" + : "decodes on runtimes whose binding consumes Uint16Array-backed fp16 feeds", + async () => { + const model = await IMFModel.load(fixtureZip) + try { + if (float16FeedsUnconsumable) { + await expect(model.translate("rok", 64)).rejects.toThrow( + /float16 graph I\/O.*fp32 graph I\/O.*normalize_fp16_io/s, + ) + } else { + const out = await model.translate("rok", 64) + expect(typeof out).toBe("string") + } + } finally { + await model.dispose() + } + }, + 60_000, + ) +})