Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions src/ml/session/onnx-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,20 @@ async function loadOrt(): Promise<OnnxNodeModule> {
}
}

/**
* float16 feeds cannot cross the onnxruntime-node binding on runtimes
* with native Float16Array (Node >= 24): ORT's JS Tensor converts every
* float16 feed into a Float16Array view, and the N-API binding reads
* that buffer as zero bytes, so session.run fails with the opaque
* "not enough space: expected N, got 0" (issue #111). On pre-Float16Array
* runtimes the same feed stays Uint16Array-backed and is consumed fine
* (verified Node 20/22 ok, Node 24/26 broken). Artifacts with float16
* graph I/O (the pre-WO79 fp16 exports) must be IO-normalized instead:
* interscript-models tools/normalize_fp16_io.py.
*/
const FLOAT16_FEEDS_CONSUMABLE =
typeof (globalThis as { Float16Array?: unknown }).Float16Array === "undefined"

class NodeInferenceSession implements InferenceSession {
private readonly session: OnnxNodeSession
private readonly ort: OnnxNodeModule
Expand Down Expand Up @@ -76,6 +90,15 @@ class NodeInferenceSession implements InferenceSession {
async run(inputs: InferenceInputs): Promise<InferenceOutputs> {
const feeds: Record<string, unknown> = {}
for (const [name, tensor] of Object.entries(inputs)) {
if (tensor.type === "float16" && !FLOAT16_FEEDS_CONSUMABLE) {
throw new Error(
`input '${name}' is float16: float16 graph I/O cannot cross the onnxruntime-node ` +
`binding on this runtime (the feed is converted to a Float16Array the binding reads ` +
`as zero bytes, surfacing as "not enough space: expected N, got 0"). The artifact's ` +
`graphs must expose fp32 graph I/O — re-export with interscript-models ` +
`tools/normalize_fp16_io.py.`,
)
}
feeds[name] = new this.ort.Tensor(tensor.type, tensor.data, tensor.dims)
}
const outputs = await this.session.run(feeds)
Expand Down
12 changes: 10 additions & 2 deletions src/ml/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,21 @@ export interface ModelRef {
* Record of named tensors.
*/
export type TensorData =
Int8Array | Uint8Array | Int16Array | Int32Array | Float32Array | Float64Array | BigInt64Array
| Int8Array
| Uint8Array
| Int16Array
| Uint16Array
| Int32Array
| Float32Array
| Float64Array
| BigInt64Array

export interface Tensor {
readonly name: string
readonly data: TensorData
readonly dims: readonly number[]
readonly type: "int8" | "uint8" | "int16" | "int32" | "int64" | "float32" | "float64"
readonly type:
"int8" | "uint8" | "int16" | "uint16" | "int32" | "int64" | "float32" | "float64" | "float16"
}

export interface InferenceInputs {
Expand Down
Binary file added test/fixtures/tiny-fp16io.zip
Binary file not shown.
56 changes: 56 additions & 0 deletions test/imf-fp16io.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/**
* The fp16 graph-I/O contract guard (issue #111). The original fp16
* exports shipped float16 graph I/O (last_hidden_state /
* encoder_hidden_states / logits all float16). Python ORT consumes
* fp16 I/O natively, so the reference decoder and the frozen vectors
* never saw it — but onnxruntime-node cannot: on runtimes with native
* Float16Array (Node >= 24) every float16 feed is converted to a
* Float16Array view whose bytes the N-API binding reads as zero, and
* session.run fails with the opaque "not enough space: expected N,
* got 0". On pre-Float16Array runtimes the same feed stays
* Uint16Array-backed and decodes fine (verified: Node 20/22 ok, Node
* 24/26 broken).
*
* The runtime therefore rejects the unconsumable condition explicitly,
* naming the artifact contract: graphs must expose fp32 I/O
* (interscript-models tools/normalize_fp16_io.py, WO79 — the shipped
* fp16 artifacts have been IO-normalized since 2026-10-21).
*
* The fixture is tiny-imf.zip with float16 wrapped across both graph
* boundaries (Cast nodes, recomputed member digests) — the pre-WO79
* export shape in ~1KB instead of 700MB.
*/

import { describe, expect, it } from "vitest"
import { IMFModel } from "../src/ml/imf/index.js"

const fixtureZip = "test/fixtures/tiny-fp16io.zip"

/** Node >= 24 ships native Float16Array; ORT converts fp16 feeds to
* Float16Array views the onnxruntime-node binding cannot consume. */
const float16FeedsUnconsumable =
typeof (globalThis as { Float16Array?: unknown }).Float16Array !== "undefined"

describe("fp16 graph-I/O artifacts (issue #111)", () => {
it(
float16FeedsUnconsumable
? "rejects with the artifact-contract error, not the opaque binding failure"
: "decodes on runtimes whose binding consumes Uint16Array-backed fp16 feeds",
async () => {
const model = await IMFModel.load(fixtureZip)
try {
if (float16FeedsUnconsumable) {
await expect(model.translate("rok", 64)).rejects.toThrow(
/float16 graph I\/O.*fp32 graph I\/O.*normalize_fp16_io/s,
)
} else {
const out = await model.translate("rok", 64)
expect(typeof out).toBe("string")
}
} finally {
await model.dispose()
}
},
60_000,
)
})
Loading