From 1c1374622d7b01ed04085921bc8d24fda7a8e869 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 13:14:59 +0000 Subject: [PATCH 01/13] docs: inventory ndarray::simd vs wide 1.7.1 vs pulp 0.23 Feature matrix, ten tech-debt items and an ideas-to-steal table from six Haiku substrate-mapper runs, counts re-run by the orchestrator. Headline: AVX2 F64x8 compare/select are scalar loops (simd_avx2.rs:982, :1153); the root cause is a single u8 mask shape on every tier; F64x4 has no compares on any stable arm (F64Mask4 exists only in simd_nightly). Mandelbrot bench is prepared with pre-registered predictions, not yet run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 28 ++++ ...d-crate-comparison-wide-pulp-2026-11-17.md | 142 ++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 .claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md diff --git a/.claude/blackboard.md b/.claude/blackboard.md index f2265e69..194ca47f 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,31 @@ +## 2026-11-17 — INVENTORY: ndarray::simd vs wide 1.7.1 vs pulp 0.23 (doc: `.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md`) + +Feature matrix, 10 tech-debt items, an ideas-to-steal table. Headline debt, read +from source: AVX2 `F64x8::simd_le` and `F64Mask8::select` are scalar loops +(`simd_avx2.rs:982`, `:1153`). The root cause is the one-shape `u8` mask. pulp makes the +mask an associated type that differs per tier (vector on V3, `b8` on V4); wide makes it +the vector type. `F64x4` has no compares on any stable arm, and `F64Mask4` exists +only in `simd_nightly/masks.rs:148`, so the backends offer different APIs. + +Mandelbrot bench prepared on `AdaWorldAPI/mandelbrot-simd` branch +`claude/practical-newton-u8lir8` (local, uncommitted: stable toolchain, `ndarray` and +`wide8` arms, pulp `x86-v4`, in-process timer). NOT RUN: `cargo build` there was denied +by the permission classifier. Predictions P1–P4 are pre-registered in the doc. + +Haiku card log (substrate-mapper, 6 units; orchestrator re-ran counts): +| unit | runs | checked | wrong | missed | +|---|---|---|---|---| +| wide traits | 1 | trait/method counts read via its own map | 0 | 0 | +| wide per-type | 1 | brief pattern missed `pub(crate)`; card self-corrected the pattern; hand-tallied table not used | 0 | 0 | +| pulp core | 1 | 2 items Read (`first_true_m64s` :819, `select_u64s` :1447) | 1 (trait line 607 vs 608) | 0 | +| pulp backends | 1 | 4 counts re-run (v1 109, v3 215, v4 370, aarch64 138) | 0 | 0 | +| ndarray facade | 2 (1st BRIEF-INCOMPLETE) | 22 structs, 291 fns re-run | 0 | 0 | +| ndarray extras | 2 (1st BRIEF-INCOMPLETE) | 5 counts re-run | 0 | 0 | +Two briefs came back BRIEF-INCOMPLETE because they gave no caller space. The next brief must state `caller space: none` explicitly. + +Loose ends: run the bench (needs permission), objdump get_count per arm per tier, +then decide on TD 1–3 (tier-native mask + F64x4 compares). + ## 2026-11-15 — RESULT: D-ADD-PM, Codec9 `P × M` lanes added in place `examples/add_pm/` + `tests/add_pm.rs`; doc `.claude/knowledge/add-pm-probe.md`. diff --git a/.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md b/.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md new file mode 100644 index 00000000..3183cd47 --- /dev/null +++ b/.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md @@ -0,0 +1,142 @@ +# ndarray::simd vs `wide` vs `pulp` — inventory, tech debt, ideas to steal (2026-11-17) + +Sources: `Lokathor/wide` @ `1b4559c` (v1.7.1, 2026-11-02), `sarah-quinones/pulp` @ +`601de0c` (v0.23.0, 2026-09-08), this tree. Raw fact maps (Haiku `substrate-mapper`, +six runs) and the name sets behind every count below are in the session scratchpad +(`maps/*.md`, `sets/*.txt`); every claim with a `file:line` here was Read by the +orchestrator. Benchmark: `AdaWorldAPI/mandelbrot-simd` branch +`claude/practical-newton-u8lir8` — **not yet run** (see § 5). + +## 1. Shape of each crate + +| | `wide` | `pulp` | `ndarray::simd` | +|---|---|---|---| +| Model | concrete lane types (`f64x4`, `u8x16`, …), operator overloading | one `Simd` trait with associated types (`S::f64s`, `S::m64s`, `S::c64s`); you write a generic fn over `S` | concrete lane types (`F64x8`, `U8x64`, …), operator overloading | +| Tier choice | compile time (`cfg(target_feature)` inside every method) | **run time**: `Arch::new()` tries V4 → V3 → scalar (`pulp/src/x86.rs:317-336`), `dispatch(WithSimd)` enters a `#[target_feature]` trampoline (`vectorize`) | compile time (`simd.rs` module routing), `cpu_guard` refuses a binary the CPU can't run; `simd_caps` detects but does not dispatch | +| Width | fixed by the type name you pick | native per tier: `f64s` = `f64x4` on V3, `f64x8` on V4 (`x86/v3.rs:350`, `x86/v4.rs:476`) | fixed by type name; on AVX2 a 512-bit type is two 256-bit halves or a `[T; N]` array | +| Mask type | the vector itself (all-ones lanes) | **tier-native**: `m64s` = `m64x4` vector on V3, `b8` bitmask on V4 (`x86/v3.rs:357`, `x86/v4.rs:483`) | `F64Mask8(u8)` bitmask **on every tier** (`simd_avx512.rs:507` `__mmask8`, `simd_avx2.rs:1145` plain `u8`) | +| Types | 30 types: 128/256/512-bit × f32 f64 i8–i64 u8–u64 | 16 associated types per tier, incl. complex `c32s` `c64s` | 22 structs on the AVX-512 arm (`simd_avx512.rs`), incl. `BF16x16/8`, `F16x16`; 13 AVX2 integer types array-backed | +| Backends | SSE2/SSE4.1/AVX/AVX2/AVX-512 (f, vl, bw, ifma), NEON, wasm simd128, scalar | x86 V1–V4 (+ `V4_256`), aarch64 NEON (+ fcma), wasm (+ relaxed), aarch32, scalar | AVX-512, AVX2, AVX (no AVX2), v2→scalar, NEON (+dotprod, bf16, A72), wasm, wasm-scalar, AMX, nightly `core::simd` | +| Size | 357 distinct fn names (`wide/src/simd*.rs` + backends) | `Simd` trait 414 items incl. macro-generated (mapper, `lib.rs:608`), 1,042 backend pub fns | 175 type-method names + 248 in the op modules (160 in `simd_masking_ops.rs`) | + +## 2. Feature inventory (concept → who has it) + +Counts are distinct fn names matching the concept in each crate's name set +(lower bounds for pulp: much of its trait is macro-generated). + +| Concept | wide | pulp | ndarray | Note | +|---|---|---|---|---| +| Transcendentals (sin/cos/exp/ln/pow/atan2/tanh/cbrt) | **40**, all float types | ~0 | `simd_exp_f32`, `simd_ln_f32` only (f32) | `hpc/vml.rs` exists but is not part of the type API | +| floor/ceil/trunc/round/round_ties_even | 28 | 12 (backend) | `floor`, `round` | | +| recip / recip_sqrt | yes | yes (backend) | none | | +| is_nan / is_finite / is_inf / copysign / flip_signs | yes | partial | none | | +| Mask any/all/none | `any`, `all`, `none` | `first_true_m*s` | only `to_bitmask() == 0` | | +| Masked / partial load-store (tails) | none (`from_slice_unaligned`) | **36**: `partial_load_*`, `partial_store_*`, `mask_load_ptr_*`, `mask_store_ptr_*`, `MemMask` | `mask_store` on one type; tail logic per caller | | +| Saturating / overflowing / wrapping ints | **43** incl. mixed signed/unsigned | wrapping only | 7 (`saturating_add/sub/abs`, `narrow_saturating_*`) | | +| Widening mul / mul-high | `widening_mul`, `mul_keep_high`, `mul_widen` | `widening_mul_u32s` | none at type level | `mul_lo32`, `mullo` only | +| Interleave / transpose | 7 | 4 (shfl) | 5 (`interleave_*_u32/u64`, `transpose8`) | | +| Gather | none | backend only | `gather`, `gather_u16`, codebook gathers | | +| Complex c32/c64 (mul_add, conj, abs2, swap_re_im) | none | **40** | none | relevant to `hpc/fft.rs` | +| Fast vs correct variants (`fast_min`, `fast_round_int`, …) | yes, explicit | `mul_add_e` (exact) vs `mul_add` | no split | | +| f16 / bf16 | none | fp16 intrinsics in `core_arch` | **full**: RNE conversions, `F16x16`, `BF16x16/8`, Kahan f16 dot | ndarray-only | +| Bit tricks: ternlog, popcount, hamming, nibble ops, GF2 affine | none | none | **yes** | ndarray-only | +| Int8 dot (VNNI `dot4_u8_i8`), fixed-point (`saturating_rounding_doubling_high_mul`) | none | none | yes | ndarray-only | +| AMX tile GEMM | none | none | yes (`simd_amx.rs`) | ndarray-only | +| Masking-op library (`*_to_mask`, `mask_*`, `masked_*`) | none | none | 160 fns | ndarray-only | +| Runtime dispatch, one binary for v3+v4 | no | **yes** | no (by design) | | +| serde / bytemuck Pod | both | bytemuck | none found in `simd.rs` | | + +## 3. ndarray tech debt found by this comparison + +Ordered by how directly it costs speed in real loops. + +1. **AVX2 `F64x8` compares are scalar loops.** `simd_le` and its siblings convert to arrays + and compare lane by lane (`simd_avx2.rs:982`, the f32 twin at `:644`), and + `F64Mask8::select` does a per-lane `if` (`simd_avx2.rs:1153`). wide does the same op + with one `_mm256_cmp_pd` + `_mm256_blendv_pd` (`wide/src/f64x4_backend.rs:79,107`). + This sits in the Mandelbrot inner loop. It is also an R2 facet-stride violation. +2. **The root cause is the mask type, not the compare.** `F64Mask8(u8)` is AVX-512's + shape. On AVX2 the natural result of `vcmppd` is a vector mask, so a `u8` mask + forces movemask out and per-lane expansion back in for every `select`. pulp avoids this by + making the mask an associated type that differs per tier; wide avoids it by making + the mask the vector type itself. +3. **`F64x4` has no compare, mask or select on any stable arm.** The AVX-512 arm's + `impl F64x4` (`simd_avx512.rs:3078`) has load/store, `reduce_sum`, `abs`, `mul_add` + and arithmetic only. The AVX2 arm reuses that type (`simd_avx2.rs:9`). `F64Mask4` is + exported by `simd.rs:263` but defined only in `simd_nightly/masks.rs:148`. So the + **backends offer different APIs**: code using `F64x4::simd_le` compiles on + `nightly-simd` and on no stable arm. Fixing this also fixes item 1, because + AVX2 `F64x8` is two `F64x4` halves. +4. **13 AVX2 integer types are `[T; N]` arrays with scalar ops** (`avx2_int_type!`, + `simd_avx2.rs:1332-1705, 2439-2441`): `U8x64 I8x64 I16x32 I32x16 I64x8 U16x32 U32x16 + U64x8 U32x8 U64x4 I32x8 …`. This is already in the burn-down list (81 of its R2 hits); the + comparison confirms that both wide and pulp realize the same widths as vector pairs. +5. **No mask predicates** (`any`/`all`/`none`): every caller writes `to_bitmask() == 0`. +6. **No float classification or reciprocal**: `is_nan`, `is_finite`, `copysign`, `recip`, + `recip_sqrt` are missing everywhere. +7. **Transcendentals only for f32 exp/ln.** wide ships sin_cos/exp/exp2/ln/log2/pow/atan2/ + tanh/cbrt on f32 and f64 at every width. +8. **No generic tail handling.** pulp's `partial_load_*`/`partial_store_*` and + `MemMask` give every type a masked head/tail; ndarray callers hand-roll remainders. +9. **No widening multiply or mul-high on the integer types.** +10. **No 128-bit float types on the x86 facade** (`F32x4`/`F64x2` are not exported by + `simd.rs`; only the wasm path mentions f32x4 lanes, `simd.rs:198`). + +## 4. Ideas to steal + +| Idea | From | Why it fits ndarray | Cost | +|---|---|---|---| +| **Tier-native mask type**: AVX2 `F64Mask8` = two `__m256d` lane masks, AVX-512 = `__mmask8`; `to_bitmask()` stays as the common exit | pulp (`m64s`), wide | Removes items 1–2 at the root; R1/R2-clean (one `vcmppd`, one `vblendvpd`, no shuffles) | Medium: mask API is public; `to_bitmask`/`select` signatures stay | +| **Complete `F64x4`/`F32x8` compare + mask + min/max on every stable arm** | wide | Closes the gap between backends (item 3), and AVX2 512-bit types then compose from halves | Small | +| **`any()` / `all()` / `none()` on every mask** | wide | Readability; one `vmovmskpd`/`kortest` | Trivial | +| **`partial_load` / `partial_store` / `MemMask` per type** | pulp | Uniform tails using AVX-512 `k`-masked loads and AVX2 `vmaskmov`; most hand-rolled remainders go away | Medium | +| **Explicit fast/correct split** (`fast_min` vs `min`, `mul_add_e` vs `mul_add`) | wide, pulp | Matches the W1a contract's "document saturating/overflow semantics"; makes NaN/FMA choices visible at the call site | Small per op | +| **Integer semantics breadth**: `overflowing_*`, `saturating_*` incl. mixed-sign, `widening_mul`, `mul_keep_high` | wide | The W1a contract already requires documented overflow semantics; wide's names are a ready vocabulary | Medium | +| **f32/f64 transcendentals on the types** (sin_cos, exp, ln, pow, atan2) | wide | `hpc/vml.rs` and activations could call type methods instead of duplicating polynomial code | Medium | +| **Complex `c32s`/`c64s` ops** (mul_add, conj_mul, abs2, swap_re_im) | pulp | `hpc/fft.rs` (Cooley-Tukey) is the consumer | Medium | +| **Opt-in runtime dispatch layer** (`Arch` + `WithSimd` + `#[target_feature]` trampoline) | pulp | One distribution binary that uses v4 when present. Conflicts with the compile-time doctrine, so only as an opt-in wrapper *above* the facade, never inside it | Large; decision for the operator | + +What **not** to copy: wide's general `shuffle`/`swizzle`/`swizzle_half` family and +pulp's `interleave_shfl`/`deinterleave_shfl` used as compute. Both are banned in this tree by the +FACET STRIDE rule R1. ndarray is ahead on quantized/bit-level work (VNNI, ternlog, +popcount, AMX, f16/bf16 RNE, the masking library) and on the number of backends it +covers and checks for parity against each other. + +## 5. Mandelbrot benchmark — prepared, not run + +Fork: `AdaWorldAPI/mandelbrot-simd`, branch `claude/practical-newton-u8lir8` (local, +uncommitted). Changes: stable toolchain 1.99.0; `std::simd` arm behind a `nightly` +feature; `wide` 0.7.28 → 1.7.1; `pulp` 0.20 → 0.23 with `x86-v4` (its default only has +v3, so without it pulp silently runs AVX2 on an AVX-512 host); new arms `wide8` +(f64x8) and `ndarray` (`ndarray::simd::F64x8`, `src/ndarray_par.rs`); in-process timer +`src/bin/bench.rs` that checks each arm against scalar before timing and prints the tier +(`avx512f=`, `avx2=`). + +Blocked: `cargo build` in that checkout was denied by the session's permission +classifier (three attempts), and so was `cargo install hyperfine`. + +Commands, one per tier (separate target dirs so the realizations don't overwrite each other): + +```sh +cd mandelbrot-simd +RUSTFLAGS="-C target-cpu=x86-64-v4" CARGO_TARGET_DIR=target/v4 cargo run --release --bin bench -- 3200 3200 15 scalar,autovect,wide,wide8,pulp,ndarray +RUSTFLAGS="-C target-cpu=x86-64-v3" CARGO_TARGET_DIR=target/v3 cargo run --release --bin bench -- 3200 3200 15 scalar,autovect,wide,wide8,pulp,ndarray +RAYON_NUM_THREADS=1 ... # same, single thread +``` + +Pre-registered predictions (from the code read in § 3; to be graded, not assumed): + +- P1 v4: `ndarray` within 15% of `pulp` and `wide8`. All three lower to + `vcmppd k` + masked blend on zmm. +- P2 v4: `wide` (f64x4) slower than `wide8` (256-bit vs 512-bit). +- P3 v3: `ndarray` slower than `wide` and `pulp`, by at least 2×: its compare and select + are scalar loops (TD 1). `pulp` on v3 picks V3 at run time (`f64s = f64x4`). +- P4 v3: under the pinned v3 build pulp's `Arch::new()` still finds AVX-512 at run time + and dispatches V4, so `pulp` should be roughly the same as on the v4 build. A + run-time dispatcher ignores the compile-time pin. +- Kill condition for TD 1's priority: if P3 fails (ndarray within 20% of wide on v3), + LLVM is re-vectorizing the scalar loop and item 1 drops in priority. + +Reverse-engineering step after the run: `objdump -d --no-show-raw-insn` the +`get_count` of each arm per tier, then count `vcmppd`, `vblendvpd`/`vblendmpd`, +`kortest`/`vmovmskpd`, and scalar `vucomisd` in the loop body. From d7e510542abd169a662278bcd41a669d30cf9952 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 13:24:15 +0000 Subject: [PATCH 02/13] docs(simd-comparison): add ISA/arch coverage and bf16 sections; correct runtime-dispatch row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ndarray has src/simd_runtime/ (feature runtime-dispatch, one LazyLock fn pointer per op); the first revision said it had none — no mapper unit covered that directory. Adds the target_arch / target_feature coverage matrix (wasm, AMX, SVE, relaxed-simd, NEON tiers) and the bf16 comparison: wide has no bf16, pulp only feature tokens, ndarray hardware conversions plus AMX tiles but scalar-only simd_half and NEON stubs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- ...d-crate-comparison-wide-pulp-2026-11-17.md | 66 ++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md b/.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md index 3183cd47..b13de712 100644 --- a/.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md +++ b/.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md @@ -19,6 +19,68 @@ orchestrator. Benchmark: `AdaWorldAPI/mandelbrot-simd` branch | Backends | SSE2/SSE4.1/AVX/AVX2/AVX-512 (f, vl, bw, ifma), NEON, wasm simd128, scalar | x86 V1–V4 (+ `V4_256`), aarch64 NEON (+ fcma), wasm (+ relaxed), aarch32, scalar | AVX-512, AVX2, AVX (no AVX2), v2→scalar, NEON (+dotprod, bf16, A72), wasm, wasm-scalar, AMX, nightly `core::simd` | | Size | 357 distinct fn names (`wide/src/simd*.rs` + backends) | `Simd` trait 414 items incl. macro-generated (mapper, `lib.rs:608`), 1,042 backend pub fns | 175 type-method names + 248 in the op modules (160 in `simd_masking_ops.rs`) | +## 1b. ISA and architecture coverage + +Counted from every `target_arch = "…"`, `target_feature = "…"` / `enable = "…"`, +`is_*_feature_detected!("…")` and pulp `f!("…")` string in each crate's `src/`. + +| | `wide` | `pulp` | `ndarray` | +|---|---|---|---| +| x86 SSE2…SSE4.2, AVX, AVX2, FMA | yes | yes (V1, V2, V3) | yes; AVX-without-AVX2 has its own arm (`simd_avx.rs`), v2 is routed to scalar | +| AVX-512 F/BW/DQ/VL | yes | yes (V4, plus `V4_256`) | yes (`simd_avx512.rs`) | +| AVX-512 extras | ifma, vbmi | cd, ifma, vbmi2, **fp16** | **bf16, vnni, vpopcntdq, vbmi, gfni**, avxvnni, avxvnniint8 | +| **AMX** (tile, int8, bf16) | none | none | **yes**: `simd_amx.rs`, `hpc/amx_matmul.rs`, `hpc/bf16_tile_gemm.rs` (30 files mention it) | +| AVX10 | none | none | detection only (`cpu_guard.rs:663`) | +| aarch64 NEON | yes | yes (`Neon`, `NeonFcma`) | yes, by tier: baseline, **dotprod**, **bf16** (stub, see § 2b), A72/A76 profiles, fp16 | +| aarch64 extras detected | none | fcma, dotprod, i8mm, rdm, frintts, aes/sha3/sm4 | dotprod, bf16, fp16, crc, sha2 | +| **SVE / SVE2** | none | detection tokens only (`core_arch/mod.rs:227-248`), no `Simd` backend | none | +| **wasm32 simd128** | yes | yes | yes (`simd_wasm.rs`, 3,043 lines) plus a **wasm-scalar** arm | +| wasm relaxed-simd | `swizzle_relaxed` only | **yes**, feature `relaxed-simd` (on by default) | not used; `simd_wasm.rs:37` documents why (`relaxed_madd` may or may not fuse) | +| aarch32 (ARMv7 NEON) | none | yes (`aarch32.rs`) | none | +| riscv / powerpc / mips / wasm64 | none | `core::arch` re-export under `nightly` only, no backend | scalar fallback (`simd.rs:249,480`) | +| LoongArch LSX/LASX | none | none | none | +| Portable `core::simd` | none | `nightly` feature for bytemuck | `nightly-simd` backend (validation only) | +| Runtime CPU detection | none | yes (`Arch::new`, raw-cpuid) | yes: `simd_caps.rs` (26 detects), `cpu_guard` refuses an unsupported binary | +| Checks that backends agree | — | — | `masking-parity` arms: native, nightly, wasm, wasm-scalar, neon-qemu, avx-qemu, v2-qemu | + +Coverage verdict: ndarray is widest on x86 server features (AMX, BF16, VNNI, +VPOPCNTDQ, GFNI) and is the only one that checks its backends agree. pulp is +widest across architectures (aarch32, relaxed-simd, fp16, i8mm detection). Nobody has SVE, +RISC-V V or LoongArch backends. Two pulp ideas are cheap to take here: AVX-512 FP16 (the host +this was written on has `avx512_fp16`) and wasm relaxed-simd behind a feature for the +non-FMA ops (min/max, dot, swizzle) where the result is deterministic. + +## 2b. bf16 comparison + +| | `wide` | `pulp` | `ndarray` | +|---|---|---|---| +| bf16 vector type | none | none (`Avx512bf16` and aarch64 `Bf16` are feature tokens, `core_arch/x86/mod.rs:363`) | `BF16x16(__m256bh)`, `BF16x8(__m128bh)` (`simd_avx512.rs:4135,4169`) | +| bf16 → f32 | do it yourself (widen u16, shift 16) | do it yourself | hardware `vcvtneebf162ps` + batch helpers; scalar shift on other tiers | +| f32 → bf16, round-to-nearest-even | none | none | `_mm512_cvtneps_pbh` + a software RNE certified byte-identical (`simd_avx512.rs:4378-4481`, `examples/bf16_rne_exhaustive.rs`) | +| bf16 dot / GEMM | none | none | `VDPBF16PS` and AMX `TDPBF16PS` paths (`hpc/bf16_tile_gemm.rs`, `hpc/amx_matmul.rs`, `simd_runtime/matmul.rs`) | +| aarch64 BFDOT | none | token only | **stub**: `BF16x8Stub` / `BF16x16Stub` (`simd_neon_bf16.rs:149,156`) | + +bf16 tech debt (ndarray): +- **Two different `BF16x16` types.** `simd_avx512::BF16x16(__m256bh)` is hardware; + `simd_half::BF16x16([u16; 16])` is scalar ("SIMD-accelerated fast paths … are a + follow-up", `simd_half.rs:1-7`). The same name means two costs depending on the import. +- **`simd_half` arithmetic is upcast-per-op scalar.** `add_bf16_inplace` and + `mul_bf16_inplace` (`simd_half.rs:245,266`) don't use the hardware conversions that + exist one file over. +- **NEON bf16 is stubs** while `simd_caps` already detects `bf16`. + +bf16 has no competitor: neither crate has a bf16 type, so "ndarray vs wide/pulp" here +means "ndarray vs what a wide/pulp user hand-writes" (widen u16→u32, shift, reinterpret). + +AVX-512 bf16 benchmark, prepared and not run (cargo was denied in this session, in this +checkout too). Arms per 1 Mi elements: (a) hand-written widen+shift on `U16x32`/`U32x16` (the +wide/pulp equivalent), (b) `BF16x16::to_f32x16` (`vcvtneebf162ps`), (c) `f32_to_bf16_batch_rne` +software vs `_mm512_cvtneps_pbh`, (d) bf16 dot: decode-then-FMA vs `VDPBF16PS` vs AMX +`TDPBF16PS` 16×16 tile. Pinned `--config .cargo/config-avx512.toml` (Sapphire Rapids; the +host has `avx512_bf16 amx_bf16 amx_tile`). +Predictions: P5 (b) ≥ 1.5× (a); P6 (c) software RNE within 2× of hardware; P7 (d) +VDPBF16PS ≥ 2× decode-then-FMA; AMX ≥ 4× VDPBF16PS at K ≥ 512. + ## 2. Feature inventory (concept → who has it) Counts are distinct fn names matching the concept in each crate's name set @@ -43,7 +105,7 @@ Counts are distinct fn names matching the concept in each crate's name set | Int8 dot (VNNI `dot4_u8_i8`), fixed-point (`saturating_rounding_doubling_high_mul`) | none | none | yes | ndarray-only | | AMX tile GEMM | none | none | yes (`simd_amx.rs`) | ndarray-only | | Masking-op library (`*_to_mask`, `mask_*`, `masked_*`) | none | none | 160 fns | ndarray-only | -| Runtime dispatch, one binary for v3+v4 | no | **yes** | no (by design) | | +| Runtime dispatch, one binary for v3+v4 | no | **yes**, whole kernel: `Arch::dispatch(WithSimd)` enters a `#[target_feature]` trampoline once | **yes, opt-in**: `src/simd_runtime/` (feature `runtime-dispatch`), one `LazyLock` fn pointer **per op** (`vnni_dot`, `matmul`, `casts`, `add_mul`) | ⊘ corrected: the first revision said "no"; no mapper unit covered `simd_runtime/` | | serde / bytemuck Pod | both | bytemuck | none found in `simd.rs` | | ## 3. ndarray tech debt found by this comparison @@ -94,7 +156,7 @@ Ordered by how directly it costs speed in real loops. | **Integer semantics breadth**: `overflowing_*`, `saturating_*` incl. mixed-sign, `widening_mul`, `mul_keep_high` | wide | The W1a contract already requires documented overflow semantics; wide's names are a ready vocabulary | Medium | | **f32/f64 transcendentals on the types** (sin_cos, exp, ln, pow, atan2) | wide | `hpc/vml.rs` and activations could call type methods instead of duplicating polynomial code | Medium | | **Complex `c32s`/`c64s` ops** (mul_add, conj_mul, abs2, swap_re_im) | pulp | `hpc/fft.rs` (Cooley-Tukey) is the consumer | Medium | -| **Opt-in runtime dispatch layer** (`Arch` + `WithSimd` + `#[target_feature]` trampoline) | pulp | One distribution binary that uses v4 when present. Conflicts with the compile-time doctrine, so only as an opt-in wrapper *above* the facade, never inside it | Large; decision for the operator | +| **Kernel-granular dispatch** (`WithSimd` + `#[target_feature]` trampoline) | pulp | ndarray already ships opt-in runtime dispatch (`simd_runtime/`), but per op: each call pays an indirect call (~2-3 ns, its own doc) and the op cannot inline into the caller's loop. pulp dispatches once per kernel and the whole loop body is compiled for the tier. Worth taking for the `runtime-dispatch` layer: a `with_tier(\|t\| ...)` entry that monomorphizes the closure per tier | Medium | What **not** to copy: wide's general `shuffle`/`swizzle`/`swizzle_half` family and pulp's `interleave_shfl`/`deinterleave_shfl` used as compute. Both are banned in this tree by the From 870f8c17e380bbbc3318876609fb72f5330e53ed Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 13:30:34 +0000 Subject: [PATCH 03/13] =?UTF-8?q?plan:=20simd-fruits-integration-v1=20?= =?UTF-8?q?=E2=80=94=20wide/pulp=20ideas=20ranked=20by=20signal,=20AVX-512?= =?UTF-8?q?=20first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P0: finish F32x8/F64x4 (compares, min/max, mul_add) on every stable arm, masks that match each tier (vector on AVX2, kmask on AVX-512), and mask any/all/none. P1: masked tail load/store, float classification and reciprocals, widening/high-half multiply, AVX2 integer burn-down ordered by measured cost. P2: bf16 debt, AVX-512 FP16 probe, run-time dispatch per kernel. Each item names its witness; three operator questions are open. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 9 ++ .claude/plans/simd-fruits-integration-v1.md | 168 ++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 .claude/plans/simd-fruits-integration-v1.md diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 194ca47f..b53c2daf 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,12 @@ +## 2026-11-17 — PLAN: `.claude/plans/simd-fruits-integration-v1.md` (wide/pulp ideas ranked by signal, AVX-512 first) + +P0 = F1 finish `F32x8`/`F64x4` (compares, min/max, mul_add) on every stable arm (also +removes the AVX2 `F64x8` scalar compares) · F2 masks that match each tier (vector on +AVX2, kmask on AVX-512) · F3 mask any/all/none. P1 = F4 masked tails, F5 float classification +and recip, F6 widening/high multiply, F7 AVX2 integer burn-down (U8x64 first, D-ADD-PM +witness). P2 = F8 bf16 debt, F9 FP16 probe, F10 kernel-level `with_tier` dispatch. Open: +the F2 mask field becoming private, the F10 scope, and cargo permission for the witnesses. + ## 2026-11-17 — INVENTORY: ndarray::simd vs wide 1.7.1 vs pulp 0.23 (doc: `.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md`) Feature matrix, 10 tech-debt items, an ideas-to-steal table. Headline debt, read diff --git a/.claude/plans/simd-fruits-integration-v1.md b/.claude/plans/simd-fruits-integration-v1.md new file mode 100644 index 00000000..6cb18258 --- /dev/null +++ b/.claude/plans/simd-fruits-integration-v1.md @@ -0,0 +1,168 @@ +# Plan: integrating the wide / pulp "fruits" — AVX-512 first, AVX2 second (v1) + +Source: `.claude/knowledge/simd-crate-comparison-wide-pulp-2026-11-17.md` (inventory, +tech debt TD1–TD10, bf16 § 2b). This plan orders the ideas taken from wide and pulp by +**signal**: how directly each one removes a measured or source-read cost on AVX-512 +(primary) and AVX2 (secondary), divided by its cost. Nothing here has been run yet; every item +names the witness that will grade it. + +## Ground rules (apply to every item) + +- **W1a consumer contract** (`.claude/knowledge/vertical-simd-consumer-contract.md`): + methods on the typed wrappers. AVX-512, AVX2 (+ the AVX arm where it gates), NEON and + scalar all ship together. Each item has a parity test and documents its + saturating/NaN semantics with a `///` example. +- **FACET STRIDE**: no shuffle-as-compute (R1). No scalar lane loop outside + `simd_scalar.rs` (R2); on AVX2 a 512-bit op is two 256-bit halves. No `as i8` on + unsigned data (R3). `facet-stride-guard.sh --diff origin/master` must stay clean. +- **Tier is read, not assumed**: every number carries `avx512f=`/`avx2=` from the + arm's own header. AVX-512 is pinned with `--config .cargo/config-v4.toml`, AVX2 with + `--config .cargo/config-v3.toml`, both under `env -u RUSTFLAGS`. +- **One chunk = one file, one deliverable** (`agent-output-durability.md`). Workers + edit; the orchestrator compiles, runs `masking-parity.sh` (native, v3-pinned, + avx-qemu, neon-qemu, wasm) and `codegen-witness.sh avx512` / `avx2`. + +## Priority list + +### P0 — the 256-bit halves and the mask (unlocks everything else) + +**F1. Finish `F32x8` and `F64x4` on every stable arm.** *(TD3; from wide)* +Today `F32x8` has `splat/from_slice/from_array/to_array/copy_to_slice/reduce_sum/abs` +(`simd_avx512.rs:2940`) and `F64x4` adds only `mul_add` (`:3078`). Add the `F64x8` set: +`simd_eq/ne/lt/le/gt/ge`, `simd_min/max/clamp`, `mul_add`, `sqrt`, `round`, `floor`, +`reduce_min/max`, `to_bits/from_bits`. +- AVX-512 (VL): `_mm256_cmp_p[sd]_mask` gives a `__mmask8` directly. +- AVX2: `_mm256_cmp_p[sd]` returns a vector mask (see F2). +- Why first: AVX2 `F64x8`/`F32x16` are built from these halves (`simd_avx2.rs:9`). Once + the halves have vector compares, TD1's scalar loops (`simd_avx2.rs:644, 982, 1153`) + become two compare instructions with nothing scalar in between. +- Closes the gap with the nightly backend: `F64Mask4` stops being nightly-only + (`simd_nightly/masks.rs:148`). +- Witness: `F64x4`/`F32x8` parity tests and masking-parity on all arms; `codegen-witness avx2` + shows `vcmppd`/`vcmpps` and no `vucomisd` in `F64x8::simd_le`. + +**F2. Mask types that match each tier.** *(TD1–TD2; from pulp `m64s`, wide)* +- AVX-512: masks stay `__mmask8`/`__mmask16` (unchanged). +- AVX2: `F64Mask4(__m256d)`, `F32Mask8(__m256)`, `F64Mask8([__m256d; 2])` and + `F32Mask16([__m256; 2])`, all full-width lane masks. +- Public API stays the same: `select(t, f)` becomes `vblendvpd` on AVX2 and + `vblendmpd k` on AVX-512. `to_bitmask()` stays as the shared way out (`vmovmskpd` on + AVX2, a plain move from the mask register on AVX-512). New: `from_bitmask(u8)`, which + existing callers that build a mask from bits need. +- Decision for the operator: whether changing the AVX2 mask's representation is + acceptable. The field is `pub` today (`F64Mask8(pub u8)`, `simd_avx2.rs:1145`). + Recommendation: make it private and route every reader through `to_bitmask()`. +- Witness: the Mandelbrot `ndarray` arm (AdaWorldAPI/mandelbrot-simd) on the pinned v3 + build. Prediction P3 says ndarray is ≥2× slower than `wide`/`pulp` **before** F1+F2; + after them it should be within 15% of `pulp` V3. A miss says the mask conversions + still cost something. + +**F3. `any()` / `all()` / `none()` plus `&`, `|`, `!` on every mask.** *(TD5; from wide)* +- AVX-512: `kortestw`/`kortestb`, and `kand`/`kor`/`knot`. +- AVX2: `vmovmskp[sd]` + `test`, or `vptest`, and `vandp[sd]`/`vorp[sd]`/`vxorp[sd]`. +- Trivial, but every early-exit loop (Mandelbrot, cascade, masking ops) currently writes + `to_bitmask() == 0`. +- Witness: parity test; codegen shows `kortest` (v4) and `vmovmskpd`/`vtestpd` (v3). + +### P1 — tails, classification, integer width + +**F4. Masked tail loads and stores on every type.** *(TD8; from pulp `partial_load/store`, `MemMask`)* +`load_partial(&[T]) -> Self` (zero-fills the missing lanes) and +`store_partial(self, &mut [T])`. +- AVX-512: `_mm512_maskz_loadu_*` / `_mm512_mask_storeu_*`, with the mask built by + `bzhi(!0, len)`. Byte and word types use AVX-512BW. +- AVX2, 32/64-bit lanes: `vmaskmovp[sd]` / `vpmaskmov[dq]`. +- AVX2, 8/16-bit lanes: there is no masked byte move. Copy the tail into a zeroed + aligned buffer with `ptr::copy_nonoverlapping` (memcpy), then do one full load. That + is a single copy, not a lane loop, so R2 is not violated. +- Fault safety: AVX-512 masked loads do not fault on masked-off lanes, so a tail can sit + at the end of a page. Test this with a `mmap` guard page. +- Witness: the guard-page test plus a sweep of tail lengths 0..LANES on every type. + +**F5. Float classification and reciprocals.** *(TD6; from wide)* +`is_nan` (`_CMP_UNORD_Q`), `is_finite`, `is_infinite`, `copysign`, `flip_signs`, +`recip`, `recip_sqrt`. +- AVX-512: `copysign` is one `vpternlogq` bit-select with a sign-bit mask operand. The immediate depends on operand order; derive it from the truth table and pin it in a test. +- AVX2: `copysign` is an `and`/`andnot`/`or` sequence. +- Semantics: there are two flavours of reciprocal. `recip_approx` uses `vrcp14p[sd]` + (AVX-512, 2⁻¹⁴ relative error) or `vrcpps` (AVX2, f32 only, 1.5·2⁻¹²). `recip` is exact + (`1/x`, i.e. `vdivp[sd]`). Both precisions are written in the doc comments. +- Witness: parity tests over NaN, ±Inf, ±0 and denormals; scalar is the reference. + +**F6. Widening multiply and high-half multiply.** *(TD9; from wide `widening_mul`, `mul_keep_high`)* +- `U32x16::mul_widen_even` → `U64x8`: `vpmuludq` on AVX-512, two `vpmuludq` on AVX2. +- `U16x32::mul_high`: `vpmulhuw` on both tiers. +- `I16x32::mul_high`: `vpmulhw` on both tiers. +- `U64x8::mul52_lo`/`mul52_hi`: `vpmadd52luq`/`vpmadd52huq`. AVX-512IFMA only, which + the host has. Gated by `cfg(target_feature = "avx512ifma")`; AVX2 and scalar use an + exact u128 path. +- Witness: parity test against u128 scalar products; codegen-witness. + +**F7. AVX2 integer burn-down, done in order of measured cost.** *(TD4; already on the burn-down list)* +- The 13 types in `avx2_int_type!` (`simd_avx2.rs:1332-1705, 2439-2441`) are `[T; N]` + arrays. They become pairs of `__m256i`, starting with the methods a measurement + already blames. +- First targets: `U8x64::mask_blend`, `saturating_add` and `saturating_sub`. They are + why D-ADD-PM costs 285 ns on AVX2 against 134 ns for the decode path (blackboard + 2026-11-15). +- Witness: re-run D-ADD-PM on v3. Target: below the 134 ns decode path. + +### P2 — the AVX-512-only features this host has and no competitor uses well + +**F8. Fix the bf16 debt.** *(§ 2b)* +- Remove the name collision: `simd_half::BF16x16([u16;16])` becomes either an alias or a + re-export of the hardware type on `avx512bf16`, and a scalar type elsewhere. One name + should mean one cost. +- `add_bf16_inplace`/`mul_bf16_inplace`/`cast_*` (`simd_half.rs:245-384`) should use + `vcvtneebf162ps` + `_mm512_cvtneps_pbh` on AVX-512. +- On AVX2: widen with `vpmovzxwd`, shift with `vpslld 16`, and narrow back with the + certified software round-to-nearest-even. No shuffles (R1). +- Witness: the bf16 bench arms (a)–(d) and predictions P5–P7 in the knowledge doc, plus + `examples/bf16_rne_exhaustive.rs` byte-equality. + +**F9. AVX-512 FP16 (`F16x32` arithmetic).** *(from pulp coverage; host has `avx512_fp16`)* +Measure before building. Stable Rust lacks the `f16` type, so the `_mm512_*_ph` +intrinsics may not be callable. If they aren't, the same raw-byte-encoding approach that +`simd_amx.rs` uses applies. Gate: a one-file probe that adds two `F16x32` vectors and +checks bit-equality against `F16x16` through f32. +- Kill: if neither the intrinsics nor the byte-encoded route is clean, defer. f32-upcast + through the existing `F16x16` stays the path. + +**F10. Run-time dispatch per kernel, not per op.** *(from pulp `WithSimd` + `vectorize`)* +- Add `simd_runtime::with_tier(|t| ...)`: it monomorphizes the closure per tier and + enters it through a `#[target_feature(enable = "avx512f,...")]` trampoline. The whole + loop then inlines for the tier and pays the indirect call once, not once per op. +- Additive and opt-in (feature `runtime-dispatch`). The compile-time facade is untouched. +- Witness: Mandelbrot `ndarray` arm through `with_tier` on a v3-pinned binary running + on this v4 host. It should match the v4-pinned build within 10%. That is what pulp's + prediction P4 tests. + +### P3 — breadth (valuable, lower signal for AVX-512) + +- **F11.** f64 `exp`/`ln` (and f32 `exp2`/`log2`) as methods on `F32x16`/`F64x8`, + moving the polynomials out of `hpc/vml.rs`. Witness: ULP sweep against libm. +- **F12.** `fast_min/fast_max` vs IEEE-correct `min/max`. First document what + `simd_min` actually does (`vminpd`: returns the second operand on NaN), then add the + correct variant (`vrangepd` on AVX-512DQ; cmp+blend on AVX2). +- **F13.** Complex `C32x8`/`C64x4` (`mul_add`, `conj_mul`, `abs2`) for `hpc/fft.rs`. + Witness: FFT round-trip error and timing. +- **F14.** wasm relaxed-simd behind a feature, only for deterministic ops. Not AVX; listed + for completeness. + +## Sequencing + +| wave | chunks (one file each) | gate before the next wave | +|---|---|---| +| 1 | F1 `simd_avx512.rs` (F32x8/F64x4 AVX-512 methods); F1 `simd_avx2.rs` (AVX2 bodies); F1 `simd_scalar.rs`; F1 `simd_neon.rs` | masking-parity green on all arms; guard clean | +| 2 | F2 mask types per arm (same four files) and F3 predicates | Mandelbrot v3 and v4 measured; P1/P3 graded | +| 3 | F4, F5, F6 (one per chunk) | parity, guard-page test, codegen-witness | +| 4 | F7 (U8x64 first), F8 | D-ADD-PM and bf16 bench re-run | +| 5 | F9 probe, F10 | probe verdict; dispatch witness | + +## Open questions for the operator + +1. F2: is the AVX2 mask representation allowed to change, i.e. make the public + `F64Mask8(pub u8)` field private? +2. F10: may `simd_runtime` grow a kernel-level entry, or must it stay per-op? +3. Benchmarks: cargo was denied in the session that wrote this, in both checkouts. The + waves need the orchestrator to be able to build and run. From dc49dbc4c6e9a3ed75e14c62b02a0190e89547c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 14:23:06 +0000 Subject: [PATCH 04/13] =?UTF-8?q?simd:=20F1+F2+F3=20=E2=80=94=20full=20F32?= =?UTF-8?q?x8/F64x4,=20tier-native=20AVX2=20masks,=20shared=20mask=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 (plan simd-fruits-integration-v1): F32x8/F64x4 gain compares, min/max/ clamp, sqrt, round (ties-even), floor, reduce_min/max and to_bits/from_bits on every stable arm, all AVX1 so one body serves AVX-512, AVX2 and AVX. New F32Mask8(__m256)/F64Mask4(__m256d) lane masks; F64Mask4 is no longer nightly-only. Scalar F32Mask8Scalar/F64Mask4Scalar renamed and exported. F2: AVX2 F32Mask16/F64Mask8 are pairs of those lane masks (fields private), so F32x16/F64x8 compare, select, min/max, sqrt, round, floor and reduce through two 256-bit ops instead of per-lane loops (removes the TD1 scalar compares at simd_avx2.rs:644/982/1153). AVX2 simd_min/max now use vminps/ vminpd, matching AVX-512 on NaN (second operand wins). F3: from_bitmask/any/all/none and & | ^ ! on every mask on every arm (simd_mask_api.rs macro for bitmask masks; by hand for vector masks; nightly gains none + operators). raster.rs mask_and uses a & b. Verified: lib tests native (AVX-512) 2743 and v3 2692 passed; new tests/simd_lane_masks.rs 7/7 on native, v3, avx, v2 and aarch64 (qemu); disable run (AVX2 to_bitmask <<4 -> <<3) turns 2 of them red; masking- parity native, wasm, wasm-scalar, neon-qemu, avx-qemu, v2-qemu PASS; clippy -D warnings, fmt and facet-stride guard clean; doctests 59 passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- src/hpc/splat3d/raster.rs | 2 +- src/lib.rs | 5 + src/simd.rs | 19 +- src/simd_avx2.rs | 337 ++++++++----------- src/simd_avx512.rs | 680 ++++++++++++++++++++++++++++++++++++++ src/simd_mask_api.rs | 120 +++++++ src/simd_neon.rs | 2 + src/simd_nightly/masks.rs | 46 +++ src/simd_scalar.rs | 88 ++++- src/simd_wasm.rs | 2 + tests/simd_lane_masks.rs | 280 ++++++++++++++++ 11 files changed, 1373 insertions(+), 208 deletions(-) create mode 100644 src/simd_mask_api.rs create mode 100644 tests/simd_lane_masks.rs diff --git a/src/hpc/splat3d/raster.rs b/src/hpc/splat3d/raster.rs index 2d3bc421..b76ced3a 100644 --- a/src/hpc/splat3d/raster.rs +++ b/src/hpc/splat3d/raster.rs @@ -48,7 +48,7 @@ pub const T_SATURATION_EPS: f32 = 1e-4; /// Combine two F32Mask16 with bitwise AND (both conditions must be true). #[inline(always)] fn mask_and(a: F32Mask16, b: F32Mask16) -> F32Mask16 { - F32Mask16(a.0 & b.0) + a & b } // ════════════════════════════════════════════════════════════════════════════ diff --git a/src/lib.rs b/src/lib.rs index 40f5e6e8..d715cf5b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -234,6 +234,11 @@ mod zip; mod dimension; +/// Shared mask API macro (`from_bitmask`, `any`, `all`, `none`, `& | ^ !`) +/// for the bitmask-backed SIMD masks; must precede every backend module. +#[macro_use] +mod simd_mask_api; + /// Portable SIMD types — `crate::simd::f32x16` today, `std::simd::f32x16` tomorrow. #[cfg(feature = "std")] #[allow(clippy::all, missing_docs, dead_code, unused_variables, unused_imports)] diff --git a/src/simd.rs b/src/simd.rs index b9ae6677..f0f75dac 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -295,10 +295,13 @@ pub use crate::simd_avx512::{ u8x64, u8x8, F32Mask16, + // 256-bit lane masks (vector masks on every x86 arm) + F32Mask8, // 512-bit (native AVX-512, __m512/__m512d/__m512i) F32x16, // 256-bit (AVX2, __m256/__m256d/__m256i) F32x8, + F64Mask4, F64Mask8, F64x4, F64x8, @@ -371,7 +374,7 @@ pub use crate::simd_avx512::{BF16x16, BF16x8}; ))] pub use crate::simd_avx512::{ batch_packed_i4_16, f32x8, f64x4, i16x16, i8x16, i8x32, palette_lookup_u8x8, prefetch_read_t0, prefetch_read_t1, - prefetch_read_t2, u16x8, u8x16, u8x8, F32x8, F64x4, I16x16, I8x16, I8x32, U16x8, U8x16, U8x8, + prefetch_read_t2, u16x8, u8x16, u8x8, F32Mask8, F32x8, F64Mask4, F64x4, I16x16, I8x16, I8x32, U16x8, U8x16, U8x8, }; #[cfg(all( @@ -452,8 +455,8 @@ pub use crate::simd_neon::{u32x16, U32x16}; pub use crate::simd_neon::{i32x16, u64x8, I32x16, U64x8}; #[cfg(all(target_arch = "aarch64", not(feature = "nightly-simd")))] pub use scalar::{ - f32x8, f64x4, i32x8, i64x4, i64x8, u16x16, u32x8, u64x4, u8x64, F32x8, F64x4, I32x8, I64x4, I64x8, U16x16, U16x32, - U32x8, U64x4, U8x64, + f32x8, f64x4, i32x8, i64x4, i64x8, u16x16, u32x8, u64x4, u8x64, F32Mask8, F32x8, F64Mask4, F64x4, I32x8, I64x4, + I64x8, U16x16, U16x32, U32x8, U64x4, U8x64, }; // wasm32 + simd128: the native v128 float hot path (F32x16 / F64x8 + masks) @@ -473,8 +476,9 @@ pub use crate::simd_wasm::wasm32_simd::{ #[cfg(all(target_arch = "wasm32", target_feature = "simd128", not(feature = "nightly-simd")))] pub use scalar::{ batch_packed_i4_16, f32x8, f64x4, i16x16, i16x32, i32x8, i64x4, i64x8, i8x32, i8x64, palette_lookup_u8x8, - prefetch_read_t0, prefetch_read_t1, prefetch_read_t2, u16x16, u16x8, u32x8, u64x4, u8x64, u8x8, F32x8, F64x4, - I16x16, I16x32, I32x8, I64x4, I64x8, I8x32, I8x64, U16x16, U16x32, U16x8, U32x8, U64x4, U8x64, U8x8, + prefetch_read_t0, prefetch_read_t1, prefetch_read_t2, u16x16, u16x8, u32x8, u64x4, u8x64, u8x8, F32Mask8, F32x8, + F64Mask4, F64x4, I16x16, I16x32, I32x8, I64x4, I64x8, I8x32, I8x64, U16x16, U16x32, U16x8, U32x8, U64x4, U8x64, + U8x8, }; // Other non-x86 targets — wasm32 without simd128, riscv, etc.: full scalar @@ -496,8 +500,9 @@ pub use scalar::{ pub use scalar::{ batch_packed_i4_16, f32x16, f32x8, f64x4, f64x8, i16x16, i16x32, i32x16, i32x8, i64x4, i64x8, i8x16, i8x32, i8x64, palette_lookup_u8x8, prefetch_read_t0, prefetch_read_t1, prefetch_read_t2, u16x16, u16x8, u32x16, u32x8, u64x4, - u64x8, u8x16, u8x64, u8x8, F32Mask16, F32x16, F32x8, F64Mask8, F64x4, F64x8, I16x16, I16x32, I32x16, I32x8, I64x4, - I64x8, I8x16, I8x32, I8x64, U16x16, U16x32, U16x8, U32x16, U32x8, U64x4, U64x8, U8x16, U8x64, U8x8, + u64x8, u8x16, u8x64, u8x8, F32Mask16, F32Mask8, F32x16, F32x8, F64Mask4, F64Mask8, F64x4, F64x8, I16x16, I16x32, + I32x16, I32x8, I64x4, I64x8, I8x16, I8x32, I8x64, U16x16, U16x32, U16x8, U32x16, U32x8, U64x4, U64x8, U8x16, U8x64, + U8x8, }; // Scalar BF16 conversion — always available on all platforms diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 3492407f..15f26a42 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -6,7 +6,7 @@ //! Targets: Intel Meteor Lake (U9 185H), Alder Lake, AMD Zen 2+, etc. //! These CPUs have AVX2 + AVX-VNNI (256-bit) but no AVX-512. -use crate::simd_avx512::{f32x8, f64x4}; +use crate::simd_avx512::{f32x8, f64x4, F32Mask8, F64Mask4}; // AVX2-native I8x32 / I16x16 live in simd_avx512.rs (256-bit __m256i types). // Re-export so consumers see a unified `crate::simd_avx2::I8x32` symbol. @@ -557,13 +557,11 @@ impl F32x16 { } #[inline(always)] pub fn reduce_min(self) -> f32 { - let a = self.to_array(); - a.iter().copied().fold(f32::INFINITY, f32::min) + self.0.reduce_min().min(self.1.reduce_min()) } #[inline(always)] pub fn reduce_max(self) -> f32 { - let a = self.to_array(); - a.iter().copied().fold(f32::NEG_INFINITY, f32::max) + self.0.reduce_max().max(self.1.reduce_max()) } #[inline(always)] pub fn abs(self) -> Self { @@ -571,32 +569,15 @@ impl F32x16 { } #[inline(always)] pub fn sqrt(self) -> Self { - let a = self.to_array(); - let mut o = [0.0f32; 16]; - for i in 0..16 { - o[i] = a[i].sqrt(); - } - Self::from_array(o) + Self(self.0.sqrt(), self.1.sqrt()) } #[inline(always)] pub fn round(self) -> Self { - let a = self.to_array(); - let mut o = [0.0f32; 16]; - for i in 0..16 { - // Ties to even, like AVX-512 roundscale and NEON/wasm nearest; - // f32::round / f64::round round ties away from zero. - o[i] = a[i].round_ties_even(); - } - Self::from_array(o) + Self(self.0.round(), self.1.round()) } #[inline(always)] pub fn floor(self) -> Self { - let a = self.to_array(); - let mut o = [0.0f32; 16]; - for i in 0..16 { - o[i] = a[i].floor(); - } - Self::from_array(o) + Self(self.0.floor(), self.1.floor()) } #[inline(always)] /// Fused multiply-add, single rounding: `_mm256_fmadd_ps` on each half @@ -606,23 +587,11 @@ impl F32x16 { } #[inline(always)] pub fn simd_min(self, other: Self) -> Self { - let a = self.to_array(); - let b = other.to_array(); - let mut o = [0.0f32; 16]; - for i in 0..16 { - o[i] = a[i].min(b[i]); - } - Self::from_array(o) + Self(self.0.simd_min(other.0), self.1.simd_min(other.1)) } #[inline(always)] pub fn simd_max(self, other: Self) -> Self { - let a = self.to_array(); - let b = other.to_array(); - let mut o = [0.0f32; 16]; - for i in 0..16 { - o[i] = a[i].max(b[i]); - } - Self::from_array(o) + Self(self.0.simd_max(other.0), self.1.simd_max(other.1)) } #[inline(always)] pub fn simd_clamp(self, lo: Self, hi: Self) -> Self { @@ -630,27 +599,11 @@ impl F32x16 { } #[inline(always)] pub fn simd_lt(self, other: Self) -> F32Mask16 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u16 = 0; - for i in 0..16 { - if a[i] < b[i] { - bits |= 1 << i; - } - } - F32Mask16(bits) + F32Mask16(self.0.simd_lt(other.0), self.1.simd_lt(other.1)) } #[inline(always)] pub fn simd_le(self, other: Self) -> F32Mask16 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u16 = 0; - for i in 0..16 { - if a[i] <= b[i] { - bits |= 1 << i; - } - } - F32Mask16(bits) + F32Mask16(self.0.simd_le(other.0), self.1.simd_le(other.1)) } #[inline(always)] pub fn simd_gt(self, other: Self) -> F32Mask16 { @@ -662,27 +615,11 @@ impl F32x16 { } #[inline(always)] pub fn simd_eq(self, other: Self) -> F32Mask16 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u16 = 0; - for i in 0..16 { - if a[i] == b[i] { - bits |= 1 << i; - } - } - F32Mask16(bits) + F32Mask16(self.0.simd_eq(other.0), self.1.simd_eq(other.1)) } #[inline(always)] pub fn simd_ne(self, other: Self) -> F32Mask16 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u16 = 0; - for i in 0..16 { - if a[i] != b[i] { - bits |= 1 << i; - } - } - F32Mask16(bits) + F32Mask16(self.0.simd_ne(other.0), self.1.simd_ne(other.1)) } /// Gather 16 f32 values at `base_ptr.offset(indices[i])` — the same /// signature and contract as the AVX-512 backend's `_mm512_i32gather_ps` @@ -815,20 +752,26 @@ impl Default for F32x16 { } } +/// 16-lane `F32x16` compare result on AVX2: two `F32Mask8` lane masks (one +/// per 256-bit half), so `select` is two `vblendvps` and `to_bitmask` two +/// `vmovmskps` — no per-lane loop. (Until plan simd-fruits-integration-v1 F2 +/// this was a `u16` bitmask, the AVX-512 shape, which forced every compare +/// and select through a scalar loop on this arm.) Read and combine it through +/// `to_bitmask` / `from_bitmask` / `& | ^ !`; the fields are private. #[derive(Copy, Clone, Debug)] -pub struct F32Mask16(pub u16); +pub struct F32Mask16(pub(crate) F32Mask8, pub(crate) F32Mask8); impl F32Mask16 { /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). /// The one representation-independent reading of a compare result: every - /// backend stores its mask differently (`__mmask16`, `u16`, - /// `core::simd::Mask`), so callers combine and inspect masks through this - /// rather than the tuple field (the `aabb` broadphase read `.0` directly - /// and did not compile on the portable backend — fixed 2026-09-14). + /// backend stores its mask differently (`__mmask16`, two lane-mask + /// vectors, `u16`, `core::simd::Mask`), so callers combine and inspect + /// masks through this rather than the tuple fields. /// /// # Examples /// Bit `i` is lane `i`: with lanes 0 and 15 below the threshold the /// `simd_lt` mask reads `0b1000_0000_0000_0001`. - /// ```rust,ignore + /// ``` + /// use ndarray::simd::F32x16; /// let mut a = [10.0f32; 16]; /// a[0] = -1.0; /// a[15] = -1.0; @@ -837,17 +780,38 @@ impl F32Mask16 { /// ``` #[inline(always)] pub fn to_bitmask(self) -> u16 { - self.0 + u16::from(self.0.to_bitmask()) | (u16::from(self.1.to_bitmask()) << 8) + } + /// Lane `i` is set iff bit `i` of `bits` is set. + /// + /// ``` + /// use ndarray::simd::F32Mask16; + /// let m = F32Mask16::from_bitmask(0x8001); + /// assert_eq!(m.to_bitmask(), 0x8001); + /// ``` + #[inline(always)] + pub fn from_bitmask(bits: u16) -> Self { + Self(F32Mask8::from_bitmask(bits as u8), F32Mask8::from_bitmask((bits >> 8) as u8)) } + /// Lane-wise `if mask { true_val } else { false_val }` (two `vblendvps`). #[inline(always)] pub fn select(self, true_val: F32x16, false_val: F32x16) -> F32x16 { - let t = true_val.to_array(); - let f = false_val.to_array(); - let mut o = [0.0f32; 16]; - for i in 0..16 { - o[i] = if (self.0 >> i) & 1 == 1 { t[i] } else { f[i] }; - } - F32x16::from_array(o) + F32x16(self.0.select(true_val.0, false_val.0), self.1.select(true_val.1, false_val.1)) + } + /// `true` if at least one lane is set. + #[inline(always)] + pub fn any(self) -> bool { + self.to_bitmask() != 0 + } + /// `true` if every lane is set. + #[inline(always)] + pub fn all(self) -> bool { + self.to_bitmask() == u16::MAX + } + /// `true` if no lane is set. + #[inline(always)] + pub fn none(self) -> bool { + self.to_bitmask() == 0 } } @@ -890,51 +854,27 @@ impl F64x8 { } #[inline(always)] pub fn reduce_min(self) -> f64 { - let a = self.to_array(); - a.iter().copied().fold(f64::INFINITY, f64::min) + self.0.reduce_min().min(self.1.reduce_min()) } #[inline(always)] pub fn reduce_max(self) -> f64 { - let a = self.to_array(); - a.iter().copied().fold(f64::NEG_INFINITY, f64::max) + self.0.reduce_max().max(self.1.reduce_max()) } #[inline(always)] pub fn abs(self) -> Self { - let a = self.to_array(); - let mut o = [0.0f64; 8]; - for i in 0..8 { - o[i] = a[i].abs(); - } - Self::from_array(o) + Self(self.0.abs(), self.1.abs()) } #[inline(always)] pub fn sqrt(self) -> Self { - let a = self.to_array(); - let mut o = [0.0f64; 8]; - for i in 0..8 { - o[i] = a[i].sqrt(); - } - Self::from_array(o) + Self(self.0.sqrt(), self.1.sqrt()) } #[inline(always)] pub fn round(self) -> Self { - let a = self.to_array(); - let mut o = [0.0f64; 8]; - for i in 0..8 { - // Ties to even, like AVX-512 roundscale and NEON/wasm nearest; - // f32::round / f64::round round ties away from zero. - o[i] = a[i].round_ties_even(); - } - Self::from_array(o) + Self(self.0.round(), self.1.round()) } #[inline(always)] pub fn floor(self) -> Self { - let a = self.to_array(); - let mut o = [0.0f64; 8]; - for i in 0..8 { - o[i] = a[i].floor(); - } - Self::from_array(o) + Self(self.0.floor(), self.1.floor()) } #[inline(always)] /// Fused multiply-add, single rounding: `_mm256_fmadd_pd` on each half @@ -944,23 +884,11 @@ impl F64x8 { } #[inline(always)] pub fn simd_min(self, other: Self) -> Self { - let a = self.to_array(); - let b = other.to_array(); - let mut o = [0.0f64; 8]; - for i in 0..8 { - o[i] = a[i].min(b[i]); - } - Self::from_array(o) + Self(self.0.simd_min(other.0), self.1.simd_min(other.1)) } #[inline(always)] pub fn simd_max(self, other: Self) -> Self { - let a = self.to_array(); - let b = other.to_array(); - let mut o = [0.0f64; 8]; - for i in 0..8 { - o[i] = a[i].max(b[i]); - } - Self::from_array(o) + Self(self.0.simd_max(other.0), self.1.simd_max(other.1)) } #[inline(always)] pub fn simd_clamp(self, lo: Self, hi: Self) -> Self { @@ -968,79 +896,31 @@ impl F64x8 { } #[inline(always)] pub fn simd_ge(self, other: Self) -> F64Mask8 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u8 = 0; - for i in 0..8 { - if a[i] >= b[i] { - bits |= 1 << i; - } - } - F64Mask8(bits) + F64Mask8(self.0.simd_ge(other.0), self.1.simd_ge(other.1)) } #[inline(always)] pub fn simd_le(self, other: Self) -> F64Mask8 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u8 = 0; - for i in 0..8 { - if a[i] <= b[i] { - bits |= 1 << i; - } - } - F64Mask8(bits) + F64Mask8(self.0.simd_le(other.0), self.1.simd_le(other.1)) } /// Lane-wise `==` — ordered: false if either lane is NaN; `+0.0 == -0.0` (AVX-512 `_CMP_EQ_OQ`). #[inline(always)] pub fn simd_eq(self, other: Self) -> F64Mask8 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u8 = 0; - for i in 0..8 { - if a[i] == b[i] { - bits |= 1 << i; - } - } - F64Mask8(bits) + F64Mask8(self.0.simd_eq(other.0), self.1.simd_eq(other.1)) } /// Lane-wise `!=` — unordered: true if either lane is NaN (AVX-512 `_CMP_NEQ_UQ`). #[inline(always)] pub fn simd_ne(self, other: Self) -> F64Mask8 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u8 = 0; - for i in 0..8 { - if a[i] != b[i] { - bits |= 1 << i; - } - } - F64Mask8(bits) + F64Mask8(self.0.simd_ne(other.0), self.1.simd_ne(other.1)) } /// Lane-wise `<` — ordered: false if either lane is NaN (AVX-512 `_CMP_LT_OS`). #[inline(always)] pub fn simd_lt(self, other: Self) -> F64Mask8 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u8 = 0; - for i in 0..8 { - if a[i] < b[i] { - bits |= 1 << i; - } - } - F64Mask8(bits) + F64Mask8(self.0.simd_lt(other.0), self.1.simd_lt(other.1)) } /// Lane-wise `>` — ordered: false if either lane is NaN (AVX-512 `other.simd_lt(self)`). #[inline(always)] pub fn simd_gt(self, other: Self) -> F64Mask8 { - let a = self.to_array(); - let b = other.to_array(); - let mut bits: u8 = 0; - for i in 0..8 { - if a[i] > b[i] { - bits |= 1 << i; - } - } - F64Mask8(bits) + F64Mask8(self.0.simd_gt(other.0), self.1.simd_gt(other.1)) } #[inline(always)] pub fn to_bits(self) -> U64x8 { @@ -1141,23 +1021,52 @@ impl Default for F64x8 { } } +/// 8-lane `F64x8` compare result on AVX2: two `F64Mask4` lane masks (one per +/// 256-bit half). Same design and API as `F32Mask16` above; the fields are +/// private. #[derive(Copy, Clone, Debug)] -pub struct F64Mask8(pub u8); +pub struct F64Mask8(pub(crate) F64Mask4, pub(crate) F64Mask4); impl F64Mask8 { /// Lane `i` of the comparison is bit `i` of the result. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let m = F64x8::from_array([0.0, 2.0, 0.0, 2.0, 0.0, 2.0, 0.0, 2.0]).simd_lt(F64x8::splat(1.0)); + /// assert_eq!(m.to_bitmask(), 0b0101_0101); + /// ``` #[inline(always)] pub fn to_bitmask(self) -> u8 { - self.0 + self.0.to_bitmask() | (self.1.to_bitmask() << 4) + } + /// Lane `i` is set iff bit `i` of `bits` is set. + /// + /// ``` + /// use ndarray::simd::F64Mask8; + /// assert_eq!(F64Mask8::from_bitmask(0x81).to_bitmask(), 0x81); + /// ``` + #[inline(always)] + pub fn from_bitmask(bits: u8) -> Self { + Self(F64Mask4::from_bitmask(bits & 0xF), F64Mask4::from_bitmask(bits >> 4)) } + /// Lane-wise `if mask { true_val } else { false_val }` (two `vblendvpd`). #[inline(always)] pub fn select(self, true_val: F64x8, false_val: F64x8) -> F64x8 { - let t = true_val.to_array(); - let f = false_val.to_array(); - let mut o = [0.0f64; 8]; - for i in 0..8 { - o[i] = if (self.0 >> i) & 1 == 1 { t[i] } else { f[i] }; - } - F64x8::from_array(o) + F64x8(self.0.select(true_val.0, false_val.0), self.1.select(true_val.1, false_val.1)) + } + /// `true` if at least one lane is set. + #[inline(always)] + pub fn any(self) -> bool { + self.to_bitmask() != 0 + } + /// `true` if every lane is set. + #[inline(always)] + pub fn all(self) -> bool { + self.to_bitmask() == u8::MAX + } + /// `true` if no lane is set. + #[inline(always)] + pub fn none(self) -> bool { + self.to_bitmask() == 0 } } @@ -5070,3 +4979,39 @@ fn ternlog_two_input_u32x16(t: u8, a: U32x16, b: U32x16) -> U32x16 { } } // GEN-TERNLOG-END + +// Lane-wise logic on the AVX2 512-bit mask pairs: one op per half. +macro_rules! impl_mask_pair_logic { + ($mask:ident) => { + impl BitAnd for $mask { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: Self) -> Self { + Self(self.0 & rhs.0, self.1 & rhs.1) + } + } + impl BitOr for $mask { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: Self) -> Self { + Self(self.0 | rhs.0, self.1 | rhs.1) + } + } + impl BitXor for $mask { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + Self(self.0 ^ rhs.0, self.1 ^ rhs.1) + } + } + impl Not for $mask { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + Self(!self.0, !self.1) + } + } + }; +} +impl_mask_pair_logic!(F32Mask16); +impl_mask_pair_logic!(F64Mask8); diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 074d383a..421552de 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -286,6 +286,8 @@ impl PartialEq for F32x16 { #[repr(transparent)] pub struct F32Mask16(pub __mmask16); +impl_bitmask_mask_api!(F32Mask16, u16, 16, F32x16); + impl F32Mask16 { /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). /// The one representation-independent reading of a compare result: every @@ -506,6 +508,8 @@ impl PartialEq for F64x8 { #[repr(transparent)] pub struct F64Mask8(pub __mmask8); +impl_bitmask_mask_api!(F64Mask8, u8, 8, F64x8); + impl F64Mask8 { /// Lane `i` of the comparison is bit `i` of the result. #[inline(always)] @@ -2995,6 +2999,217 @@ impl F32x8 { Self(_mm256_and_ps(self.0, mask)) } } + + /// Smallest lane, folded with `vminps`: on a NaN lane the result follows + /// `vminps` (the second operand wins), as on `F32x16::reduce_min`. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let v = F32x8::from_array([3.0, -1.0, 7.0, 2.0, 9.0, 0.5, 4.0, 8.0]); + /// assert_eq!(v.reduce_min(), -1.0); + /// ``` + #[inline(always)] + pub fn reduce_min(self) -> f32 { + // SAFETY: AVX/SSE3 register-only ops; this type is only compiled into + // x86 arms that have AVX (`simd.rs` routes v2 to the scalar backend). + unsafe { + let m4 = _mm_min_ps(_mm256_castps256_ps128(self.0), _mm256_extractf128_ps(self.0, 1)); + let m2 = _mm_min_ps(m4, _mm_movehl_ps(m4, m4)); + _mm_cvtss_f32(_mm_min_ss(m2, _mm_movehdup_ps(m2))) + } + } + + /// Largest lane, folded with `vmaxps` (NaN: second operand wins). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let v = F32x8::from_array([3.0, -1.0, 7.0, 2.0, 9.0, 0.5, 4.0, 8.0]); + /// assert_eq!(v.reduce_max(), 9.0); + /// ``` + #[inline(always)] + pub fn reduce_max(self) -> f32 { + // SAFETY: as `reduce_min`. + unsafe { + let m4 = _mm_max_ps(_mm256_castps256_ps128(self.0), _mm256_extractf128_ps(self.0, 1)); + let m2 = _mm_max_ps(m4, _mm_movehl_ps(m4, m4)); + _mm_cvtss_f32(_mm_max_ss(m2, _mm_movehdup_ps(m2))) + } + } + + /// Lane-wise minimum, `vminps`: if either lane is NaN the lane of `other` + /// is returned (the same rule as `F32x16::simd_min` on AVX-512). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let m = F32x8::splat(1.0).simd_min(F32x8::splat(2.0)); + /// assert_eq!(m.to_array(), [1.0; 8]); + /// ``` + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + // SAFETY: AVX register-only op (see `reduce_min`). + Self(unsafe { _mm256_min_ps(self.0, other.0) }) + } + + /// Lane-wise maximum, `vmaxps` (NaN: the lane of `other` is returned). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(1.0).simd_max(F32x8::splat(2.0)).to_array(), [2.0; 8]); + /// ``` + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_max_ps(self.0, other.0) }) + } + + /// `self.simd_max(lo).simd_min(hi)`. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let c = F32x8::splat(5.0).simd_clamp(F32x8::splat(0.0), F32x8::splat(1.0)); + /// assert_eq!(c.to_array(), [1.0; 8]); + /// ``` + #[inline(always)] + pub fn simd_clamp(self, lo: Self, hi: Self) -> Self { + self.simd_max(lo).simd_min(hi) + } + + /// Lane-wise square root, `vsqrtps` (correctly rounded). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(9.0).sqrt().to_array(), [3.0; 8]); + /// ``` + #[inline(always)] + pub fn sqrt(self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_sqrt_ps(self.0) }) + } + + /// Round to nearest, ties to even (`vroundps` imm 0x08, like the AVX-512 + /// `roundscale` and the NEON/wasm nearest). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let r = F32x8::from_array([0.5, 1.5, 2.5, -0.5, -1.5, 2.4, 2.6, -2.6]).round(); + /// assert_eq!(r.to_array(), [0.0, 2.0, 2.0, -0.0, -2.0, 2.0, 3.0, -3.0]); + /// ``` + #[inline(always)] + pub fn round(self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_round_ps::<0x08>(self.0) }) + } + + /// Round toward negative infinity (`vroundps` imm 0x09). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(-1.5).floor().to_array(), [-2.0; 8]); + /// ``` + #[inline(always)] + pub fn floor(self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_round_ps::<0x09>(self.0) }) + } + + /// The raw IEEE-754 bits of each lane. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(1.0).to_bits().to_array(), [0x3F80_0000; 8]); + /// ``` + #[inline(always)] + pub fn to_bits(self) -> U32x8 { + let mut out = [0u32; 8]; + // SAFETY: `out` is 32 writable bytes; `storeu` has no alignment need. + unsafe { _mm256_storeu_si256(out.as_mut_ptr().cast::<__m256i>(), _mm256_castps_si256(self.0)) }; + U32x8::from_array(out) + } + + /// Reinterprets raw IEEE-754 bits as lanes (inverse of `to_bits`). + /// + /// ``` + /// use ndarray::simd::{F32x8, U32x8}; + /// assert_eq!(F32x8::from_bits(U32x8::splat(0x4000_0000)).to_array(), [2.0; 8]); + /// ``` + #[inline(always)] + pub fn from_bits(bits: U32x8) -> Self { + let a = bits.to_array(); + // SAFETY: `a` is 32 readable bytes; `loadu` has no alignment need. + Self(unsafe { _mm256_castsi256_ps(_mm256_loadu_si256(a.as_ptr().cast::<__m256i>())) }) + } + + /// Lane-wise `==` (ordered: NaN compares unequal). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert!(F32x8::splat(1.0).simd_eq(F32x8::splat(1.0)).all()); + /// assert!(F32x8::splat(f32::NAN).simd_eq(F32x8::splat(f32::NAN)).none()); + /// ``` + #[inline(always)] + pub fn simd_eq(self, other: Self) -> F32Mask8 { + // SAFETY: AVX register-only op. + F32Mask8(unsafe { _mm256_cmp_ps::<_CMP_EQ_OQ>(self.0, other.0) }) + } + + /// Lane-wise `!=` (unordered: a NaN lane compares unequal). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert!(F32x8::splat(f32::NAN).simd_ne(F32x8::splat(f32::NAN)).all()); + /// ``` + #[inline(always)] + pub fn simd_ne(self, other: Self) -> F32Mask8 { + // SAFETY: AVX register-only op. + F32Mask8(unsafe { _mm256_cmp_ps::<_CMP_NEQ_UQ>(self.0, other.0) }) + } + + /// Lane-wise `<` (ordered: false on NaN). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let m = F32x8::from_array([0.0, 2.0, 0.0, 2.0, 0.0, 2.0, 0.0, 2.0]).simd_lt(F32x8::splat(1.0)); + /// assert_eq!(m.to_bitmask(), 0b0101_0101); + /// ``` + #[inline(always)] + pub fn simd_lt(self, other: Self) -> F32Mask8 { + // SAFETY: AVX register-only op. + F32Mask8(unsafe { _mm256_cmp_ps::<_CMP_LT_OS>(self.0, other.0) }) + } + + /// Lane-wise `<=` (ordered: false on NaN). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert!(F32x8::splat(1.0).simd_le(F32x8::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn simd_le(self, other: Self) -> F32Mask8 { + // SAFETY: AVX register-only op. + F32Mask8(unsafe { _mm256_cmp_ps::<_CMP_LE_OS>(self.0, other.0) }) + } + + /// Lane-wise `>` (ordered: false on NaN). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert!(F32x8::splat(2.0).simd_gt(F32x8::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn simd_gt(self, other: Self) -> F32Mask8 { + other.simd_lt(self) + } + + /// Lane-wise `>=` (ordered: false on NaN). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert!(F32x8::splat(1.0).simd_ge(F32x8::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn simd_ge(self, other: Self) -> F32Mask8 { + other.simd_le(self) + } } impl Add for F32x8 { @@ -3144,6 +3359,207 @@ impl F64x4 { Self::from_array(core::array::from_fn(|i| x[i].exact_fma(y[i], z[i]))) } } + + /// Smallest lane, folded with `vminpd` (NaN: second operand wins). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::from_array([3.0, -1.0, 7.0, 2.0]).reduce_min(), -1.0); + /// ``` + #[inline(always)] + pub fn reduce_min(self) -> f64 { + // SAFETY: AVX/SSE2 register-only ops (x86 arms with AVX only). + unsafe { + let m2 = _mm_min_pd(_mm256_castpd256_pd128(self.0), _mm256_extractf128_pd(self.0, 1)); + _mm_cvtsd_f64(_mm_min_sd(m2, _mm_unpackhi_pd(m2, m2))) + } + } + + /// Largest lane, folded with `vmaxpd` (NaN: second operand wins). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::from_array([3.0, -1.0, 7.0, 2.0]).reduce_max(), 7.0); + /// ``` + #[inline(always)] + pub fn reduce_max(self) -> f64 { + // SAFETY: as `reduce_min`. + unsafe { + let m2 = _mm_max_pd(_mm256_castpd256_pd128(self.0), _mm256_extractf128_pd(self.0, 1)); + _mm_cvtsd_f64(_mm_max_sd(m2, _mm_unpackhi_pd(m2, m2))) + } + } + + /// Lane-wise minimum, `vminpd`: if either lane is NaN the lane of `other` + /// is returned (the same rule as `F64x8::simd_min` on AVX-512). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(1.0).simd_min(F64x4::splat(2.0)).to_array(), [1.0; 4]); + /// ``` + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_min_pd(self.0, other.0) }) + } + + /// Lane-wise maximum, `vmaxpd` (NaN: the lane of `other` is returned). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(1.0).simd_max(F64x4::splat(2.0)).to_array(), [2.0; 4]); + /// ``` + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_max_pd(self.0, other.0) }) + } + + /// `self.simd_max(lo).simd_min(hi)`. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// let c = F64x4::splat(-5.0).simd_clamp(F64x4::splat(0.0), F64x4::splat(1.0)); + /// assert_eq!(c.to_array(), [0.0; 4]); + /// ``` + #[inline(always)] + pub fn simd_clamp(self, lo: Self, hi: Self) -> Self { + self.simd_max(lo).simd_min(hi) + } + + /// Lane-wise square root, `vsqrtpd` (correctly rounded). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(16.0).sqrt().to_array(), [4.0; 4]); + /// ``` + #[inline(always)] + pub fn sqrt(self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_sqrt_pd(self.0) }) + } + + /// Round to nearest, ties to even (`vroundpd` imm 0x08). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::from_array([0.5, 1.5, 2.5, -2.5]).round().to_array(), [0.0, 2.0, 2.0, -2.0]); + /// ``` + #[inline(always)] + pub fn round(self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_round_pd::<0x08>(self.0) }) + } + + /// Round toward negative infinity (`vroundpd` imm 0x09). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(2.7).floor().to_array(), [2.0; 4]); + /// ``` + #[inline(always)] + pub fn floor(self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { _mm256_round_pd::<0x09>(self.0) }) + } + + /// The raw IEEE-754 bits of each lane. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(1.0).to_bits().to_array(), [0x3FF0_0000_0000_0000; 4]); + /// ``` + #[inline(always)] + pub fn to_bits(self) -> U64x4 { + let mut out = [0u64; 4]; + // SAFETY: `out` is 32 writable bytes; `storeu` has no alignment need. + unsafe { _mm256_storeu_si256(out.as_mut_ptr().cast::<__m256i>(), _mm256_castpd_si256(self.0)) }; + U64x4::from_array(out) + } + + /// Reinterprets raw IEEE-754 bits as lanes (inverse of `to_bits`). + /// + /// ``` + /// use ndarray::simd::{F64x4, U64x4}; + /// assert_eq!(F64x4::from_bits(U64x4::splat(0x4000_0000_0000_0000)).to_array(), [2.0; 4]); + /// ``` + #[inline(always)] + pub fn from_bits(bits: U64x4) -> Self { + let a = bits.to_array(); + // SAFETY: `a` is 32 readable bytes; `loadu` has no alignment need. + Self(unsafe { _mm256_castsi256_pd(_mm256_loadu_si256(a.as_ptr().cast::<__m256i>())) }) + } + + /// Lane-wise `==` (ordered: NaN compares unequal). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert!(F64x4::splat(1.0).simd_eq(F64x4::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn simd_eq(self, other: Self) -> F64Mask4 { + // SAFETY: AVX register-only op. + F64Mask4(unsafe { _mm256_cmp_pd::<_CMP_EQ_OQ>(self.0, other.0) }) + } + + /// Lane-wise `!=` (unordered: a NaN lane compares unequal). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert!(F64x4::splat(f64::NAN).simd_ne(F64x4::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn simd_ne(self, other: Self) -> F64Mask4 { + // SAFETY: AVX register-only op. + F64Mask4(unsafe { _mm256_cmp_pd::<_CMP_NEQ_UQ>(self.0, other.0) }) + } + + /// Lane-wise `<` (ordered: false on NaN). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// let m = F64x4::from_array([0.0, 2.0, 0.0, f64::NAN]).simd_lt(F64x4::splat(1.0)); + /// assert_eq!(m.to_bitmask(), 0b0101); + /// ``` + #[inline(always)] + pub fn simd_lt(self, other: Self) -> F64Mask4 { + // SAFETY: AVX register-only op. + F64Mask4(unsafe { _mm256_cmp_pd::<_CMP_LT_OS>(self.0, other.0) }) + } + + /// Lane-wise `<=` (ordered: false on NaN). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert!(F64x4::splat(1.0).simd_le(F64x4::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn simd_le(self, other: Self) -> F64Mask4 { + // SAFETY: AVX register-only op. + F64Mask4(unsafe { _mm256_cmp_pd::<_CMP_LE_OS>(self.0, other.0) }) + } + + /// Lane-wise `>` (ordered: false on NaN). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert!(F64x4::splat(2.0).simd_gt(F64x4::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn simd_gt(self, other: Self) -> F64Mask4 { + other.simd_lt(self) + } + + /// Lane-wise `>=` (ordered: false on NaN). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert!(F64x4::splat(1.0).simd_ge(F64x4::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn simd_ge(self, other: Self) -> F64Mask4 { + other.simd_le(self) + } } impl Add for F64x4 { @@ -3218,6 +3634,270 @@ impl PartialEq for F64x4 { } } +// ============================================================================ +// F32Mask8 / F64Mask4 — 256-bit compare results as full-width lane masks +// ============================================================================ +// +// Every lane is all-ones (true) or all-zeros (false), the natural output of +// `vcmpps`/`vcmppd`. `select` is then one `vblendv`, `to_bitmask` one +// `vmovmsk`, and the AND/OR/XOR are one float-domain logic op each. All of +// these are AVX1, so the same code serves the AVX-512, AVX2 and AVX arms +// (on AVX-512VL LLVM may keep the result in a k-register). The AVX2 +// `F32Mask16`/`F64Mask8` are pairs of these. + +/// Lane mask produced by `F32x8` comparisons (one all-ones/all-zeros lane per `f32`). +/// +/// ``` +/// use ndarray::simd::F32x8; +/// let x = F32x8::from_array([1.0, 5.0, 2.0, 6.0, 3.0, 7.0, 4.0, 8.0]); +/// let small = x.simd_lt(F32x8::splat(4.5)); +/// assert_eq!(small.to_bitmask(), 0b0101_0101); +/// assert_eq!(small.select(x, F32x8::splat(0.0)).reduce_sum(), 10.0); +/// ``` +#[derive(Copy, Clone, Debug)] +#[repr(transparent)] +pub struct F32Mask8(pub(crate) __m256); + +impl F32Mask8 { + /// Bit `i` is lane `i` (`vmovmskps`). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(1.0).simd_lt(F32x8::splat(2.0)).to_bitmask(), 0xFF); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u8 { + // SAFETY: AVX register-only op; the result has only the low 8 bits set. + unsafe { _mm256_movemask_ps(self.0) as u8 } + } + + /// Lane `i` is set iff bit `i` of `bits` is set. + /// + /// Each bit is ANDed into the exponent field of its lane (bit `i` → + /// `2^i << 23`), so a set bit leaves a normal, non-zero float and a clear + /// bit leaves `+0.0`; `vcmpps NEQ 0` turns that into the lane mask. No + /// shuffle, no table, and nothing denormal (so DAZ cannot flip a lane). + /// + /// ``` + /// use ndarray::simd::{F32Mask8, F32x8}; + /// let m = F32Mask8::from_bitmask(0b1000_0001); + /// assert_eq!(m.to_bitmask(), 0b1000_0001); + /// let v = m.select(F32x8::splat(1.0), F32x8::splat(0.0)).to_array(); + /// assert_eq!(v, [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]); + /// ``` + #[inline(always)] + pub fn from_bitmask(bits: u8) -> Self { + // SAFETY: AVX register-only ops. + unsafe { + let broadcast = _mm256_castsi256_ps(_mm256_set1_epi32(i32::from(bits) << 23)); + let lane_bit = _mm256_castsi256_ps(_mm256_setr_epi32( + 1 << 23, + 2 << 23, + 4 << 23, + 8 << 23, + 16 << 23, + 32 << 23, + 64 << 23, + 128 << 23, + )); + Self(_mm256_cmp_ps::<_CMP_NEQ_UQ>(_mm256_and_ps(broadcast, lane_bit), _mm256_setzero_ps())) + } + } + + /// Lane-wise `if mask { true_val } else { false_val }` (`vblendvps`). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let m = F32x8::splat(1.0).simd_lt(F32x8::splat(2.0)); + /// assert_eq!(m.select(F32x8::splat(3.0), F32x8::splat(4.0)).to_array(), [3.0; 8]); + /// ``` + #[inline(always)] + pub fn select(self, true_val: F32x8, false_val: F32x8) -> F32x8 { + // SAFETY: AVX register-only op. + F32x8(unsafe { _mm256_blendv_ps(false_val.0, true_val.0, self.0) }) + } + + /// `true` if at least one lane is set. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert!(F32x8::splat(1.0).simd_lt(F32x8::splat(2.0)).any()); + /// ``` + #[inline(always)] + pub fn any(self) -> bool { + self.to_bitmask() != 0 + } + + /// `true` if every lane is set. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert!(F32x8::splat(1.0).simd_le(F32x8::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn all(self) -> bool { + self.to_bitmask() == 0xFF + } + + /// `true` if no lane is set — the early-exit test of a masked loop. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert!(F32x8::splat(3.0).simd_lt(F32x8::splat(2.0)).none()); + /// ``` + #[inline(always)] + pub fn none(self) -> bool { + self.to_bitmask() == 0 + } +} + +/// Lane mask produced by `F64x4` comparisons (one all-ones/all-zeros lane per `f64`). +/// +/// ``` +/// use ndarray::simd::F64x4; +/// let x = F64x4::from_array([1.0, 5.0, 2.0, 6.0]); +/// let m = x.simd_lt(F64x4::splat(4.5)); +/// assert_eq!(m.to_bitmask(), 0b0101); +/// assert_eq!(m.select(x, F64x4::splat(0.0)).reduce_sum(), 3.0); +/// ``` +#[derive(Copy, Clone, Debug)] +#[repr(transparent)] +pub struct F64Mask4(pub(crate) __m256d); + +impl F64Mask4 { + /// Bit `i` is lane `i` (`vmovmskpd`). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(1.0).simd_lt(F64x4::splat(2.0)).to_bitmask(), 0xF); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u8 { + // SAFETY: AVX register-only op; the result has only the low 4 bits set. + unsafe { _mm256_movemask_pd(self.0) as u8 } + } + + /// Lane `i` is set iff bit `i` of `bits` is set; bits 4..8 are ignored. + /// Same construction as `F32Mask8::from_bitmask` (bit `i` → `2^i << 52`). + /// + /// ``` + /// use ndarray::simd::{F64Mask4, F64x4}; + /// let m = F64Mask4::from_bitmask(0b1001); + /// assert_eq!(m.to_bitmask(), 0b1001); + /// assert_eq!(m.select(F64x4::splat(1.0), F64x4::splat(0.0)).to_array(), [1.0, 0.0, 0.0, 1.0]); + /// ``` + #[inline(always)] + pub fn from_bitmask(bits: u8) -> Self { + // SAFETY: AVX register-only ops. + unsafe { + let broadcast = _mm256_castsi256_pd(_mm256_set1_epi64x(i64::from(bits) << 52)); + let lane_bit = _mm256_castsi256_pd(_mm256_setr_epi64x(1 << 52, 2 << 52, 4 << 52, 8 << 52)); + Self(_mm256_cmp_pd::<_CMP_NEQ_UQ>(_mm256_and_pd(broadcast, lane_bit), _mm256_setzero_pd())) + } + } + + /// Lane-wise `if mask { true_val } else { false_val }` (`vblendvpd`). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// let m = F64x4::splat(1.0).simd_lt(F64x4::splat(2.0)); + /// assert_eq!(m.select(F64x4::splat(3.0), F64x4::splat(4.0)).to_array(), [3.0; 4]); + /// ``` + #[inline(always)] + pub fn select(self, true_val: F64x4, false_val: F64x4) -> F64x4 { + // SAFETY: AVX register-only op. + F64x4(unsafe { _mm256_blendv_pd(false_val.0, true_val.0, self.0) }) + } + + /// `true` if at least one lane is set. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert!(F64x4::splat(1.0).simd_lt(F64x4::splat(2.0)).any()); + /// ``` + #[inline(always)] + pub fn any(self) -> bool { + self.to_bitmask() != 0 + } + + /// `true` if every lane is set. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert!(F64x4::splat(1.0).simd_le(F64x4::splat(1.0)).all()); + /// ``` + #[inline(always)] + pub fn all(self) -> bool { + self.to_bitmask() == 0xF + } + + /// `true` if no lane is set. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert!(F64x4::splat(3.0).simd_lt(F64x4::splat(2.0)).none()); + /// ``` + #[inline(always)] + pub fn none(self) -> bool { + self.to_bitmask() == 0 + } +} + +macro_rules! impl_vector_mask_logic { + ($mask:ident, $and:ident, $or:ident, $xor:ident, $ones:expr) => { + impl BitAnd for $mask { + type Output = Self; + /// Lane-wise AND. + #[inline(always)] + fn bitand(self, rhs: Self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { $and(self.0, rhs.0) }) + } + } + impl BitOr for $mask { + type Output = Self; + /// Lane-wise OR. + #[inline(always)] + fn bitor(self, rhs: Self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { $or(self.0, rhs.0) }) + } + } + impl BitXor for $mask { + type Output = Self; + /// Lane-wise XOR. + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + // SAFETY: AVX register-only op. + Self(unsafe { $xor(self.0, rhs.0) }) + } + } + impl Not for $mask { + type Output = Self; + /// Lane-wise NOT (XOR with all-ones). + #[inline(always)] + fn not(self) -> Self { + // SAFETY: AVX register-only ops. + Self(unsafe { $xor(self.0, $ones) }) + } + } + }; +} +impl_vector_mask_logic!( + F32Mask8, + _mm256_and_ps, + _mm256_or_ps, + _mm256_xor_ps, + _mm256_castsi256_ps(_mm256_set1_epi32(-1)) +); +impl_vector_mask_logic!( + F64Mask4, + _mm256_and_pd, + _mm256_or_pd, + _mm256_xor_pd, + _mm256_castsi256_pd(_mm256_set1_epi64x(-1)) +); + // ============================================================================ // Type aliases — lowercase names matching portable_simd convention // ============================================================================ diff --git a/src/simd_mask_api.rs b/src/simd_mask_api.rs new file mode 100644 index 00000000..f134b351 --- /dev/null +++ b/src/simd_mask_api.rs @@ -0,0 +1,120 @@ +//! Shared mask API for SIMD compare results stored as a packed bitmask +//! (bit `i` = lane `i`): AVX-512 `__mmask8` / `__mmask16`, and the `u8` / +//! `u16` masks of the scalar, NEON and wasm backends. +//! +//! Masks stored as full-width lane vectors (the x86 256-bit `F32Mask8` / +//! `F64Mask4`, and the AVX2 512-bit pairs built from them) implement the same +//! API by hand in their own backend file, so that consumer code reads +//! identically on every arm: +//! +//! | method | meaning | +//! |---|---| +//! | `from_bitmask(bits)` | lane `i` set iff bit `i` is set; bits at or above `LANES` are ignored | +//! | `any()` / `all()` / `none()` | some / every / no lane set | +//! | `a & b`, `a \| b`, `a ^ b`, `!a` | lane-wise logic; `!` only flips the `LANES` live bits | + +/// Implements `from_bitmask`, `any`, `all`, `none` and the bit operators for a +/// mask newtype whose field `.0` is a packed bitmask of type `$prim`. +/// +/// `$vec` is the vector type whose comparisons produce this mask; it is used +/// only for the doc examples. +macro_rules! impl_bitmask_mask_api { + ($mask:ident, $prim:ty, $lanes:expr, $vec:ident) => { + impl $mask { + /// The live bits of this mask: one per lane, LSB-first. + const LIVE_BITS: $prim = ((1u64 << $lanes) - 1) as $prim; + + #[doc = concat!( + "Builds a mask from a packed bitmask: lane `i` is set iff bit `i` is set.\n", + "Bits at or above `", stringify!($lanes), "` are ignored.\n\n", + "```\n", + "use ndarray::simd::{", stringify!($mask), ", ", stringify!($vec), "};\n", + "let m = ", stringify!($mask), "::from_bitmask(0b101);\n", + "assert_eq!(m.to_bitmask(), 0b101);\n", + "let v = m.select(", stringify!($vec), "::splat(1.0), ", stringify!($vec), "::splat(0.0)).to_array();\n", + "assert_eq!((v[0], v[1], v[2]), (1.0, 0.0, 1.0));\n", + "```" + )] + #[inline(always)] + pub fn from_bitmask(bits: $prim) -> Self { + Self(bits & Self::LIVE_BITS) + } + + #[doc = concat!( + "`true` if at least one lane is set.\n\n", + "```\n", + "use ndarray::simd::", stringify!($vec), ";\n", + "let x = ", stringify!($vec), "::splat(3.0);\n", + "assert!(x.simd_lt(", stringify!($vec), "::splat(4.0)).any());\n", + "assert!(!x.simd_lt(", stringify!($vec), "::splat(2.0)).any());\n", + "```" + )] + #[inline(always)] + pub fn any(self) -> bool { + (self.0 & Self::LIVE_BITS) != 0 + } + + #[doc = concat!( + "`true` if every lane is set.\n\n", + "```\n", + "use ndarray::simd::", stringify!($vec), ";\n", + "let x = ", stringify!($vec), "::splat(3.0);\n", + "assert!(x.simd_le(", stringify!($vec), "::splat(3.0)).all());\n", + "```" + )] + #[inline(always)] + pub fn all(self) -> bool { + (self.0 & Self::LIVE_BITS) == Self::LIVE_BITS + } + + #[doc = concat!( + "`true` if no lane is set — the early-exit test of a masked loop.\n\n", + "```\n", + "use ndarray::simd::", stringify!($vec), ";\n", + "let x = ", stringify!($vec), "::splat(5.0);\n", + "assert!(x.simd_lt(", stringify!($vec), "::splat(1.0)).none());\n", + "```" + )] + #[inline(always)] + pub fn none(self) -> bool { + !self.any() + } + } + + impl core::ops::BitAnd for $mask { + type Output = Self; + /// Lane-wise AND: set where both masks are set. + #[inline(always)] + fn bitand(self, rhs: Self) -> Self { + Self(self.0 & rhs.0) + } + } + + impl core::ops::BitOr for $mask { + type Output = Self; + /// Lane-wise OR: set where either mask is set. + #[inline(always)] + fn bitor(self, rhs: Self) -> Self { + Self(self.0 | rhs.0) + } + } + + impl core::ops::BitXor for $mask { + type Output = Self; + /// Lane-wise XOR: set where exactly one mask is set. + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + Self(self.0 ^ rhs.0) + } + } + + impl core::ops::Not for $mask { + type Output = Self; + /// Lane-wise NOT over the live lanes only. + #[inline(always)] + fn not(self) -> Self { + Self(!self.0 & Self::LIVE_BITS) + } + } + }; +} diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 3517ff09..0bc6e8a0 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -822,6 +822,7 @@ pub mod aarch64_simd { #[derive(Copy, Clone, Debug)] pub struct F32Mask16(pub u16); + impl_bitmask_mask_api!(F32Mask16, u16, 16, F32x16); impl F32Mask16 { /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). /// The one representation-independent reading of a compare result: every @@ -1199,6 +1200,7 @@ pub mod aarch64_simd { #[derive(Copy, Clone, Debug)] pub struct F64Mask8(pub u8); + impl_bitmask_mask_api!(F64Mask8, u8, 8, F64x8); impl F64Mask8 { /// Lane `i` of the comparison is bit `i` of the result. #[inline(always)] diff --git a/src/simd_nightly/masks.rs b/src/simd_nightly/masks.rs index ab9b7f7d..1575bd88 100644 --- a/src/simd_nightly/masks.rs +++ b/src/simd_nightly/masks.rs @@ -179,3 +179,49 @@ impl F64Mask4 { self.0.any() } } + +// ============================================================================ +// Shared mask API completion: `none` and lane-wise `& | ^ !`, matching the +// stable arms (`simd_mask_api.rs`, and the x86 vector masks). +// ============================================================================ + +macro_rules! nightly_mask_logic { + ($($mask:ident),*) => {$( + impl $mask { + /// Returns `true` if no lane is set. + #[inline(always)] + pub fn none(self) -> bool { + !self.0.any() + } + } + impl core::ops::BitAnd for $mask { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: Self) -> Self { + Self(self.0 & rhs.0) + } + } + impl core::ops::BitOr for $mask { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: Self) -> Self { + Self(self.0 | rhs.0) + } + } + impl core::ops::BitXor for $mask { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + Self(self.0 ^ rhs.0) + } + } + impl core::ops::Not for $mask { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + Self(!self.0) + } + } + )*}; +} +nightly_mask_logic!(F32Mask16, F32Mask8, F64Mask8, F64Mask4); diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 4b128b3e..10f1dcef 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -513,12 +513,45 @@ impl_float_type!(F64x8, f64, 8, F64Mask8, u8); // 256-bit AVX2 float types // The macro `impl_float_type!` already emits `pub struct $mask(pub $mask_prim);`, -// so calling it with `F32Mask8Scalar` / `F64Mask4Scalar` defines those mask -// structs. The previous explicit re-declaration was a duplicate that +// so calling it with `F32Mask8` / `F64Mask4` defines those mask structs +// (named `F32Mask8Scalar` / `F64Mask4Scalar` until the 256-bit masks were +// exported on every arm, plan simd-fruits-integration-v1 F1). The previous explicit re-declaration was a duplicate that // tripped E0428 + 6× E0119 on i686-unknown-linux-gnu (where this scalar // module compiles — `#[cfg(not(target_arch = "x86_64"))]`). -impl_float_type!(F32x8, f32, 8, F32Mask8Scalar, u8); -impl_float_type!(F64x4, f64, 4, F64Mask4Scalar, u8); +impl_float_type!(F32x8, f32, 8, F32Mask8, u8); +impl_float_type!(F64x4, f64, 4, F64Mask4, u8); + +// Shared mask API (from_bitmask / any / all / none / & | ^ !) on every mask. +impl_bitmask_mask_api!(F32Mask16, u16, 16, F32x16); +impl_bitmask_mask_api!(F64Mask8, u8, 8, F64x8); +impl_bitmask_mask_api!(F32Mask8, u8, 8, F32x8); +impl_bitmask_mask_api!(F64Mask4, u8, 4, F64x4); + +impl F32Mask8 { + /// Lane `i` of the comparison is bit `i` of the result. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(1.0).simd_lt(F32x8::splat(2.0)).to_bitmask(), 0xFF); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u8 { + self.0 + } +} + +impl F64Mask4 { + /// Lane `i` of the comparison is bit `i` of the result. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(1.0).simd_lt(F64x4::splat(2.0)).to_bitmask(), 0xF); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u8 { + self.0 + } +} // 512-bit integer types impl_int_type!(U8x64, u8, 64, 0u8); @@ -1524,6 +1557,53 @@ impl F64x8 { } } +// Extra for the 256-bit float types: to_bits/from_bits (same API as the x86 arms). +impl F32x8 { + /// The raw IEEE-754 bits of each lane. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(1.0).to_bits().to_array(), [0x3F80_0000; 8]); + /// ``` + #[inline(always)] + pub fn to_bits(self) -> U32x8 { + U32x8(self.0.map(f32::to_bits)) + } + /// Reinterprets raw IEEE-754 bits as lanes (inverse of `to_bits`). + /// + /// ``` + /// use ndarray::simd::{F32x8, U32x8}; + /// assert_eq!(F32x8::from_bits(U32x8::splat(0x4000_0000)).to_array(), [2.0; 8]); + /// ``` + #[inline(always)] + pub fn from_bits(bits: U32x8) -> Self { + Self(bits.0.map(f32::from_bits)) + } +} + +impl F64x4 { + /// The raw IEEE-754 bits of each lane. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(1.0).to_bits().to_array(), [0x3FF0_0000_0000_0000; 4]); + /// ``` + #[inline(always)] + pub fn to_bits(self) -> U64x4 { + U64x4(self.0.map(f64::to_bits)) + } + /// Reinterprets raw IEEE-754 bits as lanes (inverse of `to_bits`). + /// + /// ``` + /// use ndarray::simd::{F64x4, U64x4}; + /// assert_eq!(F64x4::from_bits(U64x4::splat(0x4000_0000_0000_0000)).to_array(), [2.0; 4]); + /// ``` + #[inline(always)] + pub fn from_bits(bits: U64x4) -> Self { + Self(bits.0.map(f64::from_bits)) + } +} + // Extra for I64x8 impl I64x8 { #[inline(always)] diff --git a/src/simd_wasm.rs b/src/simd_wasm.rs index 53a97f31..13d7c6e5 100644 --- a/src/simd_wasm.rs +++ b/src/simd_wasm.rs @@ -422,6 +422,7 @@ pub mod wasm32_simd { /// 16-lane comparison mask for `F32x16` (one bit per lane). #[derive(Copy, Clone, Debug)] pub struct F32Mask16(pub u16); + impl_bitmask_mask_api!(F32Mask16, u16, 16, F32x16); impl F32Mask16 { /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). /// The one representation-independent reading of a compare result: every @@ -789,6 +790,7 @@ pub mod wasm32_simd { /// 8-lane comparison mask for `F64x8` (one bit per lane). #[derive(Copy, Clone, Debug)] pub struct F64Mask8(pub u8); + impl_bitmask_mask_api!(F64Mask8, u8, 8, F64x8); impl F64Mask8 { /// Lane `i` of the comparison is bit `i` of the result. #[inline(always)] diff --git a/tests/simd_lane_masks.rs b/tests/simd_lane_masks.rs new file mode 100644 index 00000000..07117424 --- /dev/null +++ b/tests/simd_lane_masks.rs @@ -0,0 +1,280 @@ +//! Parity for plan `simd-fruits-integration-v1` F1 + F2 + F3. +//! +//! F1: the 256-bit `F32x8` / `F64x4` carry the full float method set +//! (compares, min/max/clamp, sqrt, round, floor, reductions, bit casts) on +//! every stable arm. F2: the AVX2 `F32Mask16` / `F64Mask8` are pairs of +//! 256-bit lane masks, and `F32x16` / `F64x8` compare, select, min/max, +//! sqrt, round and floor through those halves instead of per-lane loops. +//! F3: every mask has `from_bitmask`, `any`, `all`, `none` and `& | ^ !`. +//! +//! Every expectation is computed here, lane by lane, with plain `f32`/`f64` +//! operations, so the same file grades whichever arm the build compiled +//! (AVX-512, AVX2, AVX, x86-64-v2 scalar, NEON, wasm). Read the arm from the +//! printed header of `simd_lane_masks_report_arm`. +//! +//! NaN in min/max is deliberately not graded: x86 `vminps` returns the second +//! operand while the scalar arm's `f32::min` returns the non-NaN one; that +//! difference predates this plan and is documented on the methods. + +use ndarray::simd::{F32Mask16, F32Mask8, F32x16, F32x8, F64Mask4, F64Mask8, F64x4, F64x8}; + +/// Deterministic test values: ordinary numbers, ±0, ties for rounding, +/// infinities and NaN, so ordered/unordered compares are both exercised. +const F32_POOL: [f32; 24] = [ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + 1.5, + 2.5, + -2.5, + 3.25, + -7.75, + 1e-3, + -1e9, + f32::INFINITY, + f32::NEG_INFINITY, + f32::NAN, + 2.0, + 4.0, + -0.5, + 100.0, + 0.1, + -3.5, + 8.0, + 1e30, + -1e-30, +]; + +fn f32_lanes(seed: usize) -> [f32; N] { + core::array::from_fn(|i| F32_POOL[(seed * 7 + i * 5) % F32_POOL.len()]) +} + +fn f64_lanes(seed: usize) -> [f64; N] { + core::array::from_fn(|i| f64::from(F32_POOL[(seed * 11 + i * 3) % F32_POOL.len()])) +} + +fn bits_of(pred: impl Fn(usize) -> bool, lanes: usize) -> u32 { + (0..lanes).filter(|&i| pred(i)).fold(0, |m, i| m | (1 << i)) +} + +/// Bit-exact equality that treats every NaN as equal to every NaN. +fn same_f32(a: f32, b: f32) -> bool { + (a.is_nan() && b.is_nan()) || a.to_bits() == b.to_bits() +} +fn same_f64(a: f64, b: f64) -> bool { + (a.is_nan() && b.is_nan()) || a.to_bits() == b.to_bits() +} + +macro_rules! compare_parity { + ($vec:ident, $elem:ty, $lanes:expr, $gen:ident, $seeds:expr) => { + for seed in 0..$seeds { + let a: [$elem; $lanes] = $gen(seed); + let b: [$elem; $lanes] = $gen(seed + 3); + let (va, vb) = ($vec::from_array(a), $vec::from_array(b)); + let cases: [(&str, u32, u32); 6] = [ + ("eq", va.simd_eq(vb).to_bitmask().into(), bits_of(|i| a[i] == b[i], $lanes)), + ("ne", va.simd_ne(vb).to_bitmask().into(), bits_of(|i| a[i] != b[i], $lanes)), + ("lt", va.simd_lt(vb).to_bitmask().into(), bits_of(|i| a[i] < b[i], $lanes)), + ("le", va.simd_le(vb).to_bitmask().into(), bits_of(|i| a[i] <= b[i], $lanes)), + ("gt", va.simd_gt(vb).to_bitmask().into(), bits_of(|i| a[i] > b[i], $lanes)), + ("ge", va.simd_ge(vb).to_bitmask().into(), bits_of(|i| a[i] >= b[i], $lanes)), + ]; + for (name, got, want) in cases { + assert_eq!(got, want, "{}::simd_{} seed {}: a={:?} b={:?}", stringify!($vec), name, seed, a, b); + } + } + }; +} + +#[test] +fn simd_lane_masks_report_arm() { + // The arm this file graded. A green run says nothing about the arms + // that were not compiled; run it again under each pinned config. + println!( + "simd_lane_masks: avx512f={} avx2={} avx={} neon={} simd128={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx2"), + cfg!(target_feature = "avx"), + cfg!(target_feature = "neon"), + cfg!(target_feature = "simd128"), + ); +} + +#[test] +fn compares_match_scalar_on_every_width() { + compare_parity!(F32x8, f32, 8, f32_lanes, 40); + compare_parity!(F64x4, f64, 4, f64_lanes, 40); + compare_parity!(F32x16, f32, 16, f32_lanes, 40); + compare_parity!(F64x8, f64, 8, f64_lanes, 40); +} + +macro_rules! unary_parity { + ($vec:ident, $elem:ty, $lanes:expr, $gen:ident, $same:ident) => { + for seed in 0..40 { + let a: [$elem; $lanes] = $gen(seed); + let v = $vec::from_array(a); + let checks: [(&str, [$elem; $lanes], [$elem; $lanes]); 3] = [ + ("sqrt", v.sqrt().to_array(), a.map(<$elem>::sqrt)), + ("round", v.round().to_array(), a.map(<$elem>::round_ties_even)), + ("floor", v.floor().to_array(), a.map(<$elem>::floor)), + ]; + for (name, got, want) in checks { + for i in 0..$lanes { + assert!( + $same(got[i], want[i]), + "{}::{} lane {} of {:?}: got {:?} want {:?}", + stringify!($vec), + name, + i, + a, + got[i], + want[i] + ); + } + } + } + }; +} + +#[test] +fn sqrt_round_floor_match_scalar() { + unary_parity!(F32x8, f32, 8, f32_lanes, same_f32); + unary_parity!(F64x4, f64, 4, f64_lanes, same_f64); + unary_parity!(F32x16, f32, 16, f32_lanes, same_f32); + unary_parity!(F64x8, f64, 8, f64_lanes, same_f64); +} + +/// Pool without NaN: min/max/clamp/reductions are graded on ordered inputs. +fn no_nan_f32(seed: usize) -> [f32; N] { + f32_lanes::(seed).map(|x| if x.is_nan() { 6.0 } else { x }) +} +fn no_nan_f64(seed: usize) -> [f64; N] { + f64_lanes::(seed).map(|x| if x.is_nan() { 6.0 } else { x }) +} + +macro_rules! minmax_parity { + ($vec:ident, $elem:ty, $lanes:expr, $gen:ident) => { + for seed in 0..40 { + let a: [$elem; $lanes] = $gen(seed); + let b: [$elem; $lanes] = $gen(seed + 5); + let (va, vb) = ($vec::from_array(a), $vec::from_array(b)); + let lo = $vec::splat(-2.0); + let hi = $vec::splat(3.0); + let min: [$elem; $lanes] = core::array::from_fn(|i| a[i].min(b[i])); + let max: [$elem; $lanes] = core::array::from_fn(|i| a[i].max(b[i])); + let clamp: [$elem; $lanes] = a.map(|x| x.max(-2.0).min(3.0)); + // ±0 ties are implementation-defined in min/max: compare by value. + assert_eq!(va.simd_min(vb).to_array(), min, "{} simd_min {:?} {:?}", stringify!($vec), a, b); + assert_eq!(va.simd_max(vb).to_array(), max, "{} simd_max {:?} {:?}", stringify!($vec), a, b); + assert_eq!(va.simd_clamp(lo, hi).to_array(), clamp, "{} simd_clamp {:?}", stringify!($vec), a); + let rmin = a.iter().copied().fold(<$elem>::INFINITY, <$elem>::min); + let rmax = a.iter().copied().fold(<$elem>::NEG_INFINITY, <$elem>::max); + assert_eq!(va.reduce_min(), rmin, "{} reduce_min {:?}", stringify!($vec), a); + assert_eq!(va.reduce_max(), rmax, "{} reduce_max {:?}", stringify!($vec), a); + } + }; +} + +#[test] +fn min_max_clamp_reduce_match_scalar() { + minmax_parity!(F32x8, f32, 8, no_nan_f32); + minmax_parity!(F64x4, f64, 4, no_nan_f64); + minmax_parity!(F32x16, f32, 16, no_nan_f32); + minmax_parity!(F64x8, f64, 8, no_nan_f64); +} + +#[test] +fn bit_casts_round_trip_256() { + let a: [f32; 8] = f32_lanes(1); + let bits = F32x8::from_array(a).to_bits().to_array(); + assert_eq!(bits, a.map(f32::to_bits)); + let back = F32x8::from_bits(F32x8::from_array(a).to_bits()).to_array(); + assert!(back.iter().zip(&a).all(|(x, y)| same_f32(*x, *y))); + + let d: [f64; 4] = f64_lanes(2); + assert_eq!(F64x4::from_array(d).to_bits().to_array(), d.map(f64::to_bits)); + let back = F64x4::from_bits(F64x4::from_array(d).to_bits()).to_array(); + assert!(back.iter().zip(&d).all(|(x, y)| same_f64(*x, *y))); +} + +macro_rules! mask_api_parity { + ($mask:ident, $vec:ident, $prim:ty, $lanes:expr) => {{ + let live: u32 = if $lanes == 32 { u32::MAX } else { (1u32 << $lanes) - 1 }; + // Every bit pattern for the narrow masks, a spread for the 16-lane one. + let patterns: Vec = if $lanes <= 8 { + (0..(1u32 << $lanes)).collect() + } else { + (0..=u32::from(u16::MAX)) + .step_by(97) + .chain([0, 1, 0x8000, 0xFFFF]) + .collect() + }; + for &p in &patterns { + let m = $mask::from_bitmask(p as $prim); + assert_eq!(u32::from(m.to_bitmask()), p & live, "{} from/to_bitmask {:#x}", stringify!($mask), p); + assert_eq!(m.any(), p & live != 0, "{} any {:#x}", stringify!($mask), p); + assert_eq!(m.all(), p & live == live, "{} all {:#x}", stringify!($mask), p); + assert_eq!(m.none(), p & live == 0, "{} none {:#x}", stringify!($mask), p); + assert_eq!(u32::from((!m).to_bitmask()), !p & live, "{} not {:#x}", stringify!($mask), p); + let q = (p.wrapping_mul(0x9E37) ^ 0x5A5A) & live; + let n = $mask::from_bitmask(q as $prim); + assert_eq!(u32::from((m & n).to_bitmask()), p & q & live, "{} and", stringify!($mask)); + assert_eq!(u32::from((m | n).to_bitmask()), (p | q) & live, "{} or", stringify!($mask)); + assert_eq!(u32::from((m ^ n).to_bitmask()), (p ^ q) & live, "{} xor", stringify!($mask)); + let t: [_; $lanes] = core::array::from_fn(|i| (i + 1) as _); + let f: [_; $lanes] = core::array::from_fn(|i| -((i + 1) as i32) as _); + let sel = m + .select($vec::from_array(t), $vec::from_array(f)) + .to_array(); + for i in 0..$lanes { + let want = if (p >> i) & 1 == 1 { t[i] } else { f[i] }; + assert_eq!(sel[i], want, "{} select lane {} pattern {:#x}", stringify!($mask), i, p); + } + } + }}; +} + +#[test] +fn mask_api_matches_bit_arithmetic() { + mask_api_parity!(F32Mask8, F32x8, u8, 8); + mask_api_parity!(F64Mask4, F64x4, u8, 4); + mask_api_parity!(F64Mask8, F64x8, u8, 8); + mask_api_parity!(F32Mask16, F32x16, u16, 16); +} + +#[test] +fn compare_select_early_exit_loop() { + // The shape of the Mandelbrot inner loop that motivated F1/F2: compare, + // early-exit on `none()`, and count with `select`. + let c_re = F64x8::from_array([-2.0, -1.0, -0.5, 0.0, 0.25, 0.3, 0.5, 1.0]); + let (mut re, mut im) = (c_re, F64x8::splat(0.0)); + let mut count = F64x8::splat(0.0); + for _ in 0..50 { + let (rr, ii) = (re * re, im * im); + let alive = (rr + ii).simd_le(F64x8::splat(4.0)); + if alive.none() { + break; + } + count += alive.select(F64x8::splat(1.0), F64x8::splat(0.0)); + let ri = re * im; + re = c_re + (rr - ii); + im = ri + ri; + } + // Scalar reference with the same iteration rule. + let want: [f64; 8] = c_re.to_array().map(|c| { + let (mut r, mut i, mut n) = (c, 0.0f64, 0.0f64); + for _ in 0..50 { + if r * r + i * i > 4.0 { + break; + } + n += 1.0; + let ri = r * i; + r = c + (r * r - i * i); + i = ri + ri; + } + n + }); + assert_eq!(count.to_array(), want); +} From 36e007d000724b1509b0755a9097af8ec3b60b71 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 14:38:16 +0000 Subject: [PATCH 05/13] =?UTF-8?q?simd:=20F5=20=E2=80=94=20is=5Fnan/is=5Ffi?= =?UTF-8?q?nite/is=5Finfinite/copysign/recip/recip=5Fapprox=20on=20all=20a?= =?UTF-8?q?rms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan simd-fruits-integration-v1 F5, drafted by one Sonnet worker per backend file (avx512, avx2, scalar, neon, wasm), reviewed and verified here. - classification composes the existing compares (x != x; |x| < inf; |x| == inf) - copysign: AVX-512 one vpternlog (imm 0xD8 = c ? b : a); AVX/AVX2 and/andnot/or; NEON vbslq; wasm v128_bitselect; scalar f32::copysign - recip: exact 1/x; recip_approx: vrcp14 (2^-14) on AVX-512 incl. VL 256-bit, vrcpps (1.5*2^-12) on AVX f32, exact elsewhere (documented per arm) - nightly backend gains the same six methods and the missing F64x4::from_bits Verified: tests/simd_float_classify.rs 3/3 on native(v4), v3, avx, v2, aarch64 (qemu) and nightly-simd; measured recip_approx worst rel. error v4 3.96e-5, v3 2.99e-4; disable run (copysign imm 0xE4) red; lib 2743 (v4) / 2692 (v3) passed; masking-parity wasm, wasm-scalar, neon-qemu PASS; F5 doctests 36/36 on v4 and v3; clippy -D warnings, fmt, facet-stride guard clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 19 ++ src/simd_avx2.rs | 143 ++++++++++++++ src/simd_avx512.rs | 357 ++++++++++++++++++++++++++++++++++ src/simd_neon.rs | 190 ++++++++++++++++++ src/simd_nightly/f32_types.rs | 68 +++++++ src/simd_nightly/f64_types.rs | 74 +++++++ src/simd_scalar.rs | 54 +++++ src/simd_wasm.rs | 160 +++++++++++++++ tests/simd_float_classify.rs | 157 +++++++++++++++ 9 files changed, 1222 insertions(+) create mode 100644 tests/simd_float_classify.rs diff --git a/.claude/blackboard.md b/.claude/blackboard.md index b53c2daf..cfdabe7e 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,22 @@ +## 2026-11-17 — LANDED: simd-fruits F1+F2+F3 and F5 (branch claude/practical-newton-u8lir8, PR #435) + +- F1+F2+F3 (orchestrator-written): full F32x8/F64x4 API with F32Mask8/F64Mask4 lane masks on every + stable arm; AVX2 F32Mask16/F64Mask8 are now pairs of lane masks, so the F32x16/F64x8 scalar + compare/select/min/max/sqrt/round/floor loops are gone; every mask gets from_bitmask/any/all/none + and & | ^ !. AVX2 simd_min/max NaN rule now matches AVX-512 (vminps). +- F5 (Sonnet-drafted, one worker per backend file): is_nan/is_finite/is_infinite/copysign/recip/ + recip_approx on F32x16/F64x8/F32x8/F64x4. Measured recip_approx worst relative error (f32 sweep): + AVX-512 3.96e-5 (vrcp14), AVX2/AVX 2.99e-4 (vrcpps), exact arms 4.3e-8. +- Evidence: tests/simd_lane_masks.rs 7/7 and tests/simd_float_classify.rs 3/3 on native(v4), v3, avx, + v2, aarch64-qemu and nightly-simd; disable runs red (AVX2 to_bitmask shift; copysign imm 0xE4); + lib 2743 (v4) / 2692 (v3) passed; masking-parity all 6 arms PASS; clippy/fmt/guard clean. + +Sonnet card log (F5 wave): 5 runs (avx512, avx2, scalar, neon, wasm), all code compiled first try on +every arm and passed parity; items wrong: 0 code, 1 report claim (scalar worker said F32Mask16/ +F64Mask8 have no to_bitmask — they do, simd_scalar.rs:1433), doctests in scalar+neon read mask `.0` +(representation-dependent) — orchestrator switched 9 to to_bitmask(). Nightly backend F5 + the +missing F64x4::from_bits added by the orchestrator. + ## 2026-11-17 — PLAN: `.claude/plans/simd-fruits-integration-v1.md` (wide/pulp ideas ranked by signal, AVX-512 first) P0 = F1 finish `F32x8`/`F64x4` (compares, min/max, mul_add) on every stable arm (also diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 15f26a42..d210792a 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -672,6 +672,81 @@ impl F32x16 { } } +impl F32x16 { + /// Lane set iff the lane is NaN. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [0.0f32; 16]; + /// a[1] = f32::NAN; + /// a[15] = f32::NAN; + /// assert_eq!(F32x16::from_array(a).is_nan().to_bitmask(), 0b1000_0000_0000_0010); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F32Mask16 { + F32Mask16(self.0.is_nan(), self.1.is_nan()) + } + /// Lane set iff the lane is neither NaN nor ±Inf. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [0.0f32; 16]; + /// a[0] = f32::NAN; + /// a[9] = f32::INFINITY; + /// assert_eq!(F32x16::from_array(a).is_finite().to_bitmask(), 0b1111_1101_1111_1110); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F32Mask16 { + F32Mask16(self.0.is_finite(), self.1.is_finite()) + } + /// Lane set iff the lane is +Inf or −Inf. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [0.0f32; 16]; + /// a[0] = f32::NAN; + /// a[9] = f32::NEG_INFINITY; + /// a[3] = f32::INFINITY; + /// assert_eq!(F32x16::from_array(a).is_infinite().to_bitmask(), 0b0000_0010_0000_1000); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F32Mask16 { + F32Mask16(self.0.is_infinite(), self.1.is_infinite()) + } + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and ±0). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(3.0).copysign(F32x16::splat(-0.0)).to_array(), [-3.0; 16]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + Self(self.0.copysign(sign.0), self.1.copysign(sign.1)) + } + /// Exactly `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(4.0).recip().to_array(), [0.25; 16]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + Self(self.0.recip(), self.1.recip()) + } + /// Approximation of `1/self`. On AVX2 this is the 256-bit half's bound: + /// ≤ 1.5·2^-12 relative error for f32 (`vrcpps`), exact for f64. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let r = F32x16::splat(4.0).recip_approx().to_array(); + /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 2f32.powi(-11))); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + Self(self.0.recip_approx(), self.1.recip_approx()) + } +} + impl Add for F32x16 { type Output = Self; #[inline(always)] @@ -941,6 +1016,74 @@ impl F64x8 { } } +impl F64x8 { + /// Lane set iff the lane is NaN. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let m = F64x8::from_array([0.0, f64::NAN, 1.0, 2.0, 3.0, 4.0, 5.0, f64::NAN]).is_nan(); + /// assert_eq!(m.to_bitmask(), 0b1000_0010); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F64Mask8 { + F64Mask8(self.0.is_nan(), self.1.is_nan()) + } + /// Lane set iff the lane is neither NaN nor ±Inf. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let m = F64x8::from_array([0.0, f64::NAN, f64::INFINITY, 2.0, 3.0, f64::NEG_INFINITY, 5.0, 6.0]).is_finite(); + /// assert_eq!(m.to_bitmask(), 0b1101_1001); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F64Mask8 { + F64Mask8(self.0.is_finite(), self.1.is_finite()) + } + /// Lane set iff the lane is +Inf or −Inf. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let m = F64x8::from_array([0.0, f64::NAN, f64::INFINITY, 2.0, 3.0, f64::NEG_INFINITY, 5.0, 6.0]).is_infinite(); + /// assert_eq!(m.to_bitmask(), 0b0010_0100); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F64Mask8 { + F64Mask8(self.0.is_infinite(), self.1.is_infinite()) + } + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and ±0). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(3.0).copysign(F64x8::splat(-0.0)).to_array(), [-3.0; 8]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + Self(self.0.copysign(sign.0), self.1.copysign(sign.1)) + } + /// Exactly `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(4.0).recip().to_array(), [0.25; 8]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + Self(self.0.recip(), self.1.recip()) + } + /// Approximation of `1/self`. On AVX2 this is the 256-bit half's bound: + /// ≤ 1.5·2^-12 relative error for f32 (`vrcpps`), exact for f64. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let r = F64x8::splat(4.0).recip_approx().to_array(); + /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 2f64.powi(-11))); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + Self(self.0.recip_approx(), self.1.recip_approx()) + } +} + impl Add for F64x8 { type Output = Self; #[inline(always)] diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 421552de..84af5372 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -246,6 +246,92 @@ impl F32x16 { } } +impl F32x16 { + /// Lane-wise NaN test: a lane is set iff it is NaN (`x != x`, unordered). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [1.0; 16]; + /// a[1] = f32::NAN; + /// assert_eq!(F32x16::from_array(a).is_nan().to_bitmask(), 0b10); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F32Mask16 { + self.simd_ne(self) + } + + /// Lane-wise finite test: set iff the lane is neither NaN nor +-Inf + /// (`|x| < INFINITY`, ordered, so false for NaN). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [1.0; 16]; + /// a[0] = f32::INFINITY; + /// a[1] = f32::NAN; + /// assert_eq!(F32x16::from_array(a).is_finite().to_bitmask(), 0b1111_1111_1111_1100); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F32Mask16 { + self.abs().simd_lt(F32x16::splat(f32::INFINITY)) + } + + /// Lane-wise infinity test: set iff the lane is +Inf or -Inf. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [1.0; 16]; + /// a[0] = f32::INFINITY; + /// a[1] = f32::NEG_INFINITY; + /// a[2] = f32::NAN; + /// assert_eq!(F32x16::from_array(a).is_infinite().to_bitmask(), 0b011); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F32Mask16 { + self.abs().simd_eq(F32x16::splat(f32::INFINITY)) + } + + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and +-0). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(3.0).copysign(F32x16::splat(-0.0)).to_array(), [-3.0; 16]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + // SAFETY: AVX-512F register-only ops. Imm 0xD8 = `c ? b : a` per bit. + unsafe { + let a = _mm512_castps_si512(self.0); + let b = _mm512_castps_si512(sign.0); + let c = _mm512_set1_epi32(i32::MIN); + Self(_mm512_castsi512_ps(_mm512_ternarylogic_epi32::<0xD8>(a, b, c))) + } + } + + /// Exact reciprocal: `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(4.0).recip().to_array(), [0.25; 16]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + F32x16::splat(1.0) / self + } + + /// Approximate reciprocal `1 / self`. `_mm512_rcp14_ps`: relative error <= 2^-14. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let r = F32x16::splat(4.0).recip_approx().to_array(); + /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 0.001)); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + // SAFETY: AVX-512F register-only op (vrcp14ps). + Self(unsafe { _mm512_rcp14_ps(self.0) }) + } +} + impl_bin_op!(F32x16, Add, add, _mm512_add_ps); impl_bin_op!(F32x16, Sub, sub, _mm512_sub_ps); impl_bin_op!(F32x16, Mul, mul, _mm512_mul_ps); @@ -468,6 +554,92 @@ impl F64x8 { } } +impl F64x8 { + /// Lane-wise NaN test: a lane is set iff it is NaN (`x != x`, unordered). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let mut a = [1.0; 8]; + /// a[1] = f64::NAN; + /// assert_eq!(F64x8::from_array(a).is_nan().to_bitmask(), 0b10); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F64Mask8 { + self.simd_ne(self) + } + + /// Lane-wise finite test: set iff the lane is neither NaN nor +-Inf + /// (`|x| < INFINITY`, ordered, so false for NaN). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let mut a = [1.0; 8]; + /// a[0] = f64::INFINITY; + /// a[1] = f64::NAN; + /// assert_eq!(F64x8::from_array(a).is_finite().to_bitmask(), 0b1111_1100); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F64Mask8 { + self.abs().simd_lt(F64x8::splat(f64::INFINITY)) + } + + /// Lane-wise infinity test: set iff the lane is +Inf or -Inf. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let mut a = [1.0; 8]; + /// a[0] = f64::INFINITY; + /// a[1] = f64::NEG_INFINITY; + /// a[2] = f64::NAN; + /// assert_eq!(F64x8::from_array(a).is_infinite().to_bitmask(), 0b011); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F64Mask8 { + self.abs().simd_eq(F64x8::splat(f64::INFINITY)) + } + + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and +-0). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(3.0).copysign(F64x8::splat(-0.0)).to_array(), [-3.0; 8]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + // SAFETY: AVX-512F register-only ops. Imm 0xD8 = `c ? b : a` per bit. + unsafe { + let a = _mm512_castpd_si512(self.0); + let b = _mm512_castpd_si512(sign.0); + let c = _mm512_set1_epi64(i64::MIN); + Self(_mm512_castsi512_pd(_mm512_ternarylogic_epi64::<0xD8>(a, b, c))) + } + } + + /// Exact reciprocal: `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(4.0).recip().to_array(), [0.25; 8]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + F64x8::splat(1.0) / self + } + + /// Approximate reciprocal `1 / self`. `_mm512_rcp14_pd`: relative error <= 2^-14. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let r = F64x8::splat(4.0).recip_approx().to_array(); + /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 0.001)); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + // SAFETY: AVX-512F register-only op (vrcp14pd). + Self(unsafe { _mm512_rcp14_pd(self.0) }) + } +} + impl_bin_op!(F64x8, Add, add, _mm512_add_pd); impl_bin_op!(F64x8, Sub, sub, _mm512_sub_pd); impl_bin_op!(F64x8, Mul, mul, _mm512_mul_pd); @@ -3212,6 +3384,99 @@ impl F32x8 { } } +impl F32x8 { + /// Lane-wise NaN test: a lane is set iff it is NaN (`x != x`, unordered). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let mut a = [1.0; 8]; + /// a[1] = f32::NAN; + /// assert_eq!(F32x8::from_array(a).is_nan().to_bitmask(), 0b10); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F32Mask8 { + self.simd_ne(self) + } + + /// Lane-wise finite test: set iff the lane is neither NaN nor +-Inf + /// (`|x| < INFINITY`, ordered, so false for NaN). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let mut a = [1.0; 8]; + /// a[0] = f32::INFINITY; + /// a[1] = f32::NAN; + /// assert_eq!(F32x8::from_array(a).is_finite().to_bitmask(), 0b1111_1100); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F32Mask8 { + self.abs().simd_lt(F32x8::splat(f32::INFINITY)) + } + + /// Lane-wise infinity test: set iff the lane is +Inf or -Inf. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let mut a = [1.0; 8]; + /// a[0] = f32::INFINITY; + /// a[1] = f32::NEG_INFINITY; + /// a[2] = f32::NAN; + /// assert_eq!(F32x8::from_array(a).is_infinite().to_bitmask(), 0b011); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F32Mask8 { + self.abs().simd_eq(F32x8::splat(f32::INFINITY)) + } + + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and +-0). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(3.0).copysign(F32x8::splat(-0.0)).to_array(), [-3.0; 8]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + // SAFETY: AVX register-only ops. + unsafe { + let m = _mm256_set1_ps(-0.0); + Self(_mm256_or_ps(_mm256_andnot_ps(m, self.0), _mm256_and_ps(m, sign.0))) + } + } + + /// Exact reciprocal: `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F32x8; + /// assert_eq!(F32x8::splat(4.0).recip().to_array(), [0.25; 8]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + F32x8::splat(1.0) / self + } + + /// Approximate reciprocal `1 / self`. With `avx512vl` enabled: `_mm256_rcp14_ps`, relative error <= 2^-14. + /// Otherwise: `_mm256_rcp_ps`, relative error <= 1.5 * 2^-12. + /// + /// ``` + /// use ndarray::simd::F32x8; + /// let r = F32x8::splat(4.0).recip_approx().to_array(); + /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 0.001)); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + #[cfg(target_feature = "avx512vl")] + { + // SAFETY: avx512vl is enabled at compile time (vrcp14ps ymm). + Self(unsafe { _mm256_rcp14_ps(self.0) }) + } + #[cfg(not(target_feature = "avx512vl"))] + { + // SAFETY: AVX register-only op (vrcpps). + Self(unsafe { _mm256_rcp_ps(self.0) }) + } + } +} + impl Add for F32x8 { type Output = Self; #[inline(always)] @@ -3562,6 +3827,98 @@ impl F64x4 { } } +impl F64x4 { + /// Lane-wise NaN test: a lane is set iff it is NaN (`x != x`, unordered). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// let mut a = [1.0; 4]; + /// a[1] = f64::NAN; + /// assert_eq!(F64x4::from_array(a).is_nan().to_bitmask(), 0b10); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F64Mask4 { + self.simd_ne(self) + } + + /// Lane-wise finite test: set iff the lane is neither NaN nor +-Inf + /// (`|x| < INFINITY`, ordered, so false for NaN). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// let mut a = [1.0; 4]; + /// a[0] = f64::INFINITY; + /// a[1] = f64::NAN; + /// assert_eq!(F64x4::from_array(a).is_finite().to_bitmask(), 0b1100); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F64Mask4 { + self.abs().simd_lt(F64x4::splat(f64::INFINITY)) + } + + /// Lane-wise infinity test: set iff the lane is +Inf or -Inf. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// let mut a = [1.0; 4]; + /// a[0] = f64::INFINITY; + /// a[1] = f64::NEG_INFINITY; + /// a[2] = f64::NAN; + /// assert_eq!(F64x4::from_array(a).is_infinite().to_bitmask(), 0b011); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F64Mask4 { + self.abs().simd_eq(F64x4::splat(f64::INFINITY)) + } + + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and +-0). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(3.0).copysign(F64x4::splat(-0.0)).to_array(), [-3.0; 4]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + // SAFETY: AVX register-only ops. + unsafe { + let m = _mm256_set1_pd(-0.0); + Self(_mm256_or_pd(_mm256_andnot_pd(m, self.0), _mm256_and_pd(m, sign.0))) + } + } + + /// Exact reciprocal: `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F64x4; + /// assert_eq!(F64x4::splat(4.0).recip().to_array(), [0.25; 4]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + F64x4::splat(1.0) / self + } + + /// Approximate reciprocal `1 / self`. With `avx512vl` enabled: `_mm256_rcp14_pd`, relative error <= 2^-14. + /// Otherwise AVX has no f64 reciprocal estimate, so this is the exact `recip()`. + /// + /// ``` + /// use ndarray::simd::F64x4; + /// let r = F64x4::splat(4.0).recip_approx().to_array(); + /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 0.001)); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + #[cfg(target_feature = "avx512vl")] + { + // SAFETY: avx512vl is enabled at compile time (vrcp14pd ymm). + Self(unsafe { _mm256_rcp14_pd(self.0) }) + } + #[cfg(not(target_feature = "avx512vl"))] + { + self.recip() + } + } +} + impl Add for F64x4 { type Output = Self; #[inline(always)] diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 0bc6e8a0..796da3f2 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -717,6 +717,101 @@ pub mod aarch64_simd { } } + impl F32x16 { + /// Lane-wise NaN test: bit `i` of the mask is set iff lane `i` is NaN. + /// + /// Realized as `self.simd_ne(self)` (unordered `!=`, true only for NaN). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [1.0f32; 16]; + /// a[1] = f32::NAN; + /// assert_eq!(F32x16::from_array(a).is_nan().to_bitmask(), 0b10); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F32Mask16 { + self.simd_ne(self) + } + + /// Lane-wise finite test: bit `i` is set iff lane `i` is neither NaN nor +/-Inf. + /// + /// Realized as `self.abs().simd_lt(splat(INFINITY))` (ordered `<`). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [1.0f32; 16]; + /// a[0] = f32::INFINITY; + /// a[1] = f32::NAN; + /// assert_eq!(F32x16::from_array(a).is_finite().to_bitmask(), 0xFFFFu16 & !0b11); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F32Mask16 { + self.abs().simd_lt(Self::splat(f32::INFINITY)) + } + + /// Lane-wise infinity test: bit `i` is set iff lane `i` is +Inf or -Inf. + /// + /// Realized as `self.abs().simd_eq(splat(INFINITY))`. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [1.0f32; 16]; + /// a[0] = f32::INFINITY; + /// a[2] = f32::NEG_INFINITY; + /// a[3] = f32::NAN; + /// assert_eq!(F32x16::from_array(a).is_infinite().to_bitmask(), 0b101); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F32Mask16 { + self.abs().simd_eq(Self::splat(f32::INFINITY)) + } + + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and +/-0). + /// + /// One `vbslq_f32` bit-select per 128-bit register with the sign-bit mask. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let r = F32x16::splat(3.0).copysign(F32x16::splat(-0.0)); + /// assert_eq!(r.to_array(), [-3.0f32; 16]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + // SAFETY: NEON is baseline on aarch64; register-only bit-select, no memory access. + unsafe { + let m = vdupq_n_u32(0x8000_0000); + Self([ + vbslq_f32(m, sign.0[0], self.0[0]), + vbslq_f32(m, sign.0[1], self.0[1]), + vbslq_f32(m, sign.0[2], self.0[2]), + vbslq_f32(m, sign.0[3], self.0[3]), + ]) + } + } + + /// Reciprocal: exactly `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(4.0).recip().to_array(), [0.25f32; 16]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + Self::splat(1.0) / self + } + + /// Approximate reciprocal; exact on NEON (no estimate is used; `vrecpe` alone is ~8 bits). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(4.0).recip_approx().to_array(), [0.25f32; 16]); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } + } + impl Add for F32x16 { type Output = Self; #[inline(always)] @@ -1095,6 +1190,101 @@ pub mod aarch64_simd { } } + impl F64x8 { + /// Lane-wise NaN test: bit `i` of the mask is set iff lane `i` is NaN. + /// + /// Realized as `self.simd_ne(self)` (unordered `!=`, true only for NaN). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let mut a = [1.0f64; 8]; + /// a[1] = f64::NAN; + /// assert_eq!(F64x8::from_array(a).is_nan().to_bitmask(), 0b10); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F64Mask8 { + self.simd_ne(self) + } + + /// Lane-wise finite test: bit `i` is set iff lane `i` is neither NaN nor +/-Inf. + /// + /// Realized as `self.abs().simd_lt(splat(INFINITY))` (ordered `<`). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let mut a = [1.0f64; 8]; + /// a[0] = f64::INFINITY; + /// a[1] = f64::NAN; + /// assert_eq!(F64x8::from_array(a).is_finite().to_bitmask(), 0xFFu8 & !0b11); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F64Mask8 { + self.abs().simd_lt(Self::splat(f64::INFINITY)) + } + + /// Lane-wise infinity test: bit `i` is set iff lane `i` is +Inf or -Inf. + /// + /// Realized as `self.abs().simd_eq(splat(INFINITY))`. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let mut a = [1.0f64; 8]; + /// a[0] = f64::INFINITY; + /// a[2] = f64::NEG_INFINITY; + /// a[3] = f64::NAN; + /// assert_eq!(F64x8::from_array(a).is_infinite().to_bitmask(), 0b101); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F64Mask8 { + self.abs().simd_eq(Self::splat(f64::INFINITY)) + } + + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and +/-0). + /// + /// One `vbslq_f64` bit-select per 128-bit register with the sign-bit mask. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let r = F64x8::splat(3.0).copysign(F64x8::splat(-0.0)); + /// assert_eq!(r.to_array(), [-3.0f64; 8]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + // SAFETY: NEON is baseline on aarch64; register-only bit-select, no memory access. + unsafe { + let m = vdupq_n_u64(0x8000_0000_0000_0000); + Self([ + vbslq_f64(m, sign.0[0], self.0[0]), + vbslq_f64(m, sign.0[1], self.0[1]), + vbslq_f64(m, sign.0[2], self.0[2]), + vbslq_f64(m, sign.0[3], self.0[3]), + ]) + } + } + + /// Reciprocal: exactly `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(4.0).recip().to_array(), [0.25f64; 8]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + Self::splat(1.0) / self + } + + /// Approximate reciprocal; exact on NEON (no estimate is used; `vrecpe` alone is ~8 bits). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(4.0).recip_approx().to_array(), [0.25f64; 8]); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } + } + impl Add for F64x8 { type Output = Self; #[inline(always)] diff --git a/src/simd_nightly/f32_types.rs b/src/simd_nightly/f32_types.rs index 53def4a1..4e2da6ef 100644 --- a/src/simd_nightly/f32_types.rs +++ b/src/simd_nightly/f32_types.rs @@ -446,3 +446,71 @@ impl Default for F32x8 { Self::splat(0.0) } } + +// F5 (plan simd-fruits-integration-v1): classification, copysign, reciprocal. +impl F32x16 { + /// Lane set iff the lane is NaN. + #[inline(always)] + pub fn is_nan(self) -> F32Mask16 { + F32Mask16(self.0.is_nan()) + } + /// Lane set iff the lane is neither NaN nor infinite. + #[inline(always)] + pub fn is_finite(self) -> F32Mask16 { + F32Mask16(self.0.is_finite()) + } + /// Lane set iff the lane is +Inf or -Inf. + #[inline(always)] + pub fn is_infinite(self) -> F32Mask16 { + F32Mask16(self.0.is_infinite()) + } + /// Magnitude of `self`, sign bit of `sign` (bit-exact). + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + Self(self.0.copysign(sign.0)) + } + /// Exactly `1.0 / self` per lane. + #[inline(always)] + pub fn recip(self) -> Self { + Self(self.0.recip()) + } + /// Approximate `1/self`; exact on the portable backend. + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } +} + +// F5 (plan simd-fruits-integration-v1): classification, copysign, reciprocal. +impl F32x8 { + /// Lane set iff the lane is NaN. + #[inline(always)] + pub fn is_nan(self) -> F32Mask8 { + F32Mask8(self.0.is_nan()) + } + /// Lane set iff the lane is neither NaN nor infinite. + #[inline(always)] + pub fn is_finite(self) -> F32Mask8 { + F32Mask8(self.0.is_finite()) + } + /// Lane set iff the lane is +Inf or -Inf. + #[inline(always)] + pub fn is_infinite(self) -> F32Mask8 { + F32Mask8(self.0.is_infinite()) + } + /// Magnitude of `self`, sign bit of `sign` (bit-exact). + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + Self(self.0.copysign(sign.0)) + } + /// Exactly `1.0 / self` per lane. + #[inline(always)] + pub fn recip(self) -> Self { + Self(self.0.recip()) + } + /// Approximate `1/self`; exact on the portable backend. + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } +} diff --git a/src/simd_nightly/f64_types.rs b/src/simd_nightly/f64_types.rs index 6d58a8e0..6052d676 100644 --- a/src/simd_nightly/f64_types.rs +++ b/src/simd_nightly/f64_types.rs @@ -308,6 +308,12 @@ impl F64x4 { super::u_word_types::U64x4(self.0.to_bits()) } + /// Reinterprets raw IEEE-754 bits as lanes (inverse of `to_bits`). + #[inline(always)] + pub fn from_bits(bits: super::u_word_types::U64x4) -> Self { + Self(core_f64x4::from_bits(bits.0)) + } + // ── Comparisons → mask ───────────────────────────────────────── /// Per-lane `==`. @@ -346,3 +352,71 @@ impl F64x4 { super::masks::F64Mask4(self.0.simd_ge(other.0)) } } + +// F5 (plan simd-fruits-integration-v1): classification, copysign, reciprocal. +impl F64x8 { + /// Lane set iff the lane is NaN. + #[inline(always)] + pub fn is_nan(self) -> super::masks::F64Mask8 { + super::masks::F64Mask8(self.0.is_nan()) + } + /// Lane set iff the lane is neither NaN nor infinite. + #[inline(always)] + pub fn is_finite(self) -> super::masks::F64Mask8 { + super::masks::F64Mask8(self.0.is_finite()) + } + /// Lane set iff the lane is +Inf or -Inf. + #[inline(always)] + pub fn is_infinite(self) -> super::masks::F64Mask8 { + super::masks::F64Mask8(self.0.is_infinite()) + } + /// Magnitude of `self`, sign bit of `sign` (bit-exact). + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + Self(self.0.copysign(sign.0)) + } + /// Exactly `1.0 / self` per lane. + #[inline(always)] + pub fn recip(self) -> Self { + Self(self.0.recip()) + } + /// Approximate `1/self`; exact on the portable backend. + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } +} + +// F5 (plan simd-fruits-integration-v1): classification, copysign, reciprocal. +impl F64x4 { + /// Lane set iff the lane is NaN. + #[inline(always)] + pub fn is_nan(self) -> super::masks::F64Mask4 { + super::masks::F64Mask4(self.0.is_nan()) + } + /// Lane set iff the lane is neither NaN nor infinite. + #[inline(always)] + pub fn is_finite(self) -> super::masks::F64Mask4 { + super::masks::F64Mask4(self.0.is_finite()) + } + /// Lane set iff the lane is +Inf or -Inf. + #[inline(always)] + pub fn is_infinite(self) -> super::masks::F64Mask4 { + super::masks::F64Mask4(self.0.is_infinite()) + } + /// Magnitude of `self`, sign bit of `sign` (bit-exact). + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + Self(self.0.copysign(sign.0)) + } + /// Exactly `1.0 / self` per lane. + #[inline(always)] + pub fn recip(self) -> Self { + Self(self.0.recip()) + } + /// Approximate `1/self`; exact on the portable backend. + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } +} diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 10f1dcef..e7b34b96 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -157,6 +157,60 @@ macro_rules! impl_float_type { Self(out) } + #[doc = concat!("Lane mask: bit `i` set iff lane `i` is NaN.\n\n```\nuse ndarray::simd::", stringify!($name), ";\nlet mut a = [1.0; ", stringify!($lanes), "];\na[1] = f64::NAN as _;\nlet m = ", stringify!($name), "::from_array(a).is_nan();\nassert_eq!(m.to_bitmask(), 0b10);\n```")] + #[inline(always)] + pub fn is_nan(self) -> $mask { + let mut bits: $mask_prim = 0; + for i in 0..$lanes { + if self.0[i].is_nan() { + bits |= 1 << i; + } + } + $mask(bits) + } + + #[doc = concat!("Lane mask: bit `i` set iff lane `i` is neither NaN nor infinite.\n\n```\nuse ndarray::simd::", stringify!($name), ";\nlet mut a = [1.0; ", stringify!($lanes), "];\na[0] = f64::INFINITY as _;\nlet m = ", stringify!($name), "::from_array(a).is_finite();\nassert_eq!(m.to_bitmask() & 1, 0);\nassert_eq!((m.to_bitmask() >> 1) & 1, 1);\n```")] + #[inline(always)] + pub fn is_finite(self) -> $mask { + let mut bits: $mask_prim = 0; + for i in 0..$lanes { + if self.0[i].is_finite() { + bits |= 1 << i; + } + } + $mask(bits) + } + + #[doc = concat!("Lane mask: bit `i` set iff lane `i` is +Inf or -Inf.\n\n```\nuse ndarray::simd::", stringify!($name), ";\nlet mut a = [1.0; ", stringify!($lanes), "];\na[0] = f64::NEG_INFINITY as _;\nlet m = ", stringify!($name), "::from_array(a).is_infinite();\nassert_eq!(m.to_bitmask(), 0b1);\n```")] + #[inline(always)] + pub fn is_infinite(self) -> $mask { + let mut bits: $mask_prim = 0; + for i in 0..$lanes { + if self.0[i].is_infinite() { + bits |= 1 << i; + } + } + $mask(bits) + } + + #[doc = concat!("Magnitude of `self`, sign bit of `sign` (bit-exact, also for NaN and zeros).\n\n```\nuse ndarray::simd::", stringify!($name), ";\nlet r = ", stringify!($name), "::splat(3.0).copysign(", stringify!($name), "::splat(-0.0));\nassert_eq!(r.to_array(), [-3.0; ", stringify!($lanes), "]);\n```")] + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + Self(core::array::from_fn(|i| self.0[i].copysign(sign.0[i]))) + } + + #[doc = concat!("Exactly `1.0 / self` per lane (IEEE division).\n\n```\nuse ndarray::simd::", stringify!($name), ";\nlet r = ", stringify!($name), "::splat(4.0).recip();\nassert_eq!(r.to_array(), [0.25; ", stringify!($lanes), "]);\n```")] + #[inline(always)] + pub fn recip(self) -> Self { + Self(core::array::from_fn(|i| 1.0 / self.0[i])) + } + + #[doc = concat!("Approximate `1/self`. Exact on the scalar arm (relative error 0, the same as `recip`).\n\n```\nuse ndarray::simd::", stringify!($name), ";\nlet r = ", stringify!($name), "::splat(4.0).recip_approx();\nassert_eq!(r.to_array(), [0.25; ", stringify!($lanes), "]);\n```")] + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } + #[inline(always)] pub fn simd_lt(self, other: Self) -> $mask { let mut bits: $mask_prim = 0; diff --git a/src/simd_wasm.rs b/src/simd_wasm.rs index 13d7c6e5..b0491e29 100644 --- a/src/simd_wasm.rs +++ b/src/simd_wasm.rs @@ -324,6 +324,86 @@ pub mod wasm32_simd { } } + impl F32x16 { + /// Lane mask: set iff the lane is NaN (`self != self`, unordered). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let m = F32x16::from_array([0.0, f32::NAN, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0]).is_nan(); + /// assert_eq!(m.to_bitmask(), 0b10); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F32Mask16 { + self.simd_ne(self) + } + + /// Lane mask: set iff the lane is neither NaN nor infinite. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let m = F32x16::from_array([0.0, f32::NAN, f32::INFINITY, f32::NEG_INFINITY, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0]).is_finite(); + /// assert_eq!(m.to_bitmask(), 0b1111_1111_1111_0001); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F32Mask16 { + self.abs().simd_lt(Self::splat(f32::INFINITY)) + } + + /// Lane mask: set iff the lane is +Inf or -Inf. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let m = F32x16::from_array([0.0, f32::NAN, f32::INFINITY, f32::NEG_INFINITY, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0]).is_infinite(); + /// assert_eq!(m.to_bitmask(), 0b1100); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F32Mask16 { + self.abs().simd_eq(Self::splat(f32::INFINITY)) + } + + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and ±0). + /// + /// One `v128_bitselect` per register. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(3.0).copysign(F32x16::splat(-0.0)).to_array(), [-3.0; 16]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + let m = u32x4_splat(0x8000_0000); + Self([ + v128_bitselect(sign.0[0], self.0[0], m), + v128_bitselect(sign.0[1], self.0[1], m), + v128_bitselect(sign.0[2], self.0[2], m), + v128_bitselect(sign.0[3], self.0[3], m), + ]) + } + + /// Exactly `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(4.0).recip().to_array(), [0.25; 16]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + Self::splat(1.0) / self + } + + /// Reciprocal "approximation": exact on wasm (simd128 has no reciprocal + /// estimate), i.e. identical to [`Self::recip`]. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// assert_eq!(F32x16::splat(4.0).recip_approx().to_array(), [0.25; 16]); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } + } + impl Add for F32x16 { type Output = Self; #[inline(always)] @@ -692,6 +772,86 @@ pub mod wasm32_simd { } } + impl F64x8 { + /// Lane mask: set iff the lane is NaN (`self != self`, unordered). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let m = F64x8::from_array([0.0, f64::NAN, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).is_nan(); + /// assert_eq!(m.to_bitmask(), 0b10); + /// ``` + #[inline(always)] + pub fn is_nan(self) -> F64Mask8 { + self.simd_ne(self) + } + + /// Lane mask: set iff the lane is neither NaN nor infinite. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let m = F64x8::from_array([0.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY, 1.0, 2.0, 3.0, 4.0]).is_finite(); + /// assert_eq!(m.to_bitmask(), 0b1111_0001); + /// ``` + #[inline(always)] + pub fn is_finite(self) -> F64Mask8 { + self.abs().simd_lt(Self::splat(f64::INFINITY)) + } + + /// Lane mask: set iff the lane is +Inf or -Inf. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// let m = F64x8::from_array([0.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY, 1.0, 2.0, 3.0, 4.0]).is_infinite(); + /// assert_eq!(m.to_bitmask(), 0b1100); + /// ``` + #[inline(always)] + pub fn is_infinite(self) -> F64Mask8 { + self.abs().simd_eq(Self::splat(f64::INFINITY)) + } + + /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and ±0). + /// + /// One `v128_bitselect` per register. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(3.0).copysign(F64x8::splat(-0.0)).to_array(), [-3.0; 8]); + /// ``` + #[inline(always)] + pub fn copysign(self, sign: Self) -> Self { + let m = u64x2_splat(0x8000_0000_0000_0000); + Self([ + v128_bitselect(sign.0[0], self.0[0], m), + v128_bitselect(sign.0[1], self.0[1], m), + v128_bitselect(sign.0[2], self.0[2], m), + v128_bitselect(sign.0[3], self.0[3], m), + ]) + } + + /// Exactly `1.0 / self` (IEEE division, correctly rounded). + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(4.0).recip().to_array(), [0.25; 8]); + /// ``` + #[inline(always)] + pub fn recip(self) -> Self { + Self::splat(1.0) / self + } + + /// Reciprocal "approximation": exact on wasm (simd128 has no reciprocal + /// estimate), i.e. identical to [`Self::recip`]. + /// + /// ``` + /// use ndarray::simd::F64x8; + /// assert_eq!(F64x8::splat(4.0).recip_approx().to_array(), [0.25; 8]); + /// ``` + #[inline(always)] + pub fn recip_approx(self) -> Self { + self.recip() + } + } + impl Add for F64x8 { type Output = Self; #[inline(always)] diff --git a/tests/simd_float_classify.rs b/tests/simd_float_classify.rs new file mode 100644 index 00000000..b3e60db2 --- /dev/null +++ b/tests/simd_float_classify.rs @@ -0,0 +1,157 @@ +//! Parity for plan `simd-fruits-integration-v1` F5: `is_nan`, `is_finite`, +//! `is_infinite`, `copysign`, `recip` and `recip_approx` on `F32x16`, +//! `F64x8`, `F32x8` and `F64x4`. +//! +//! Expectations are computed lane by lane with the `f32`/`f64` methods of +//! the same name, so this file grades whichever arm the build compiled. +//! `copysign` and `recip` must be bit-exact; `recip_approx` must be within +//! a relative error of 2^-11 on normal inputs whose reciprocal is normal +//! (the documented contract; AVX-512 `vrcp14` is 2^-14, AVX `vrcpps` +//! 1.5·2^-12, the other arms are exact). + +use ndarray::simd::{F32x16, F32x8, F64x4, F64x8}; + +const POOL: [f64; 26] = [ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + -2.5, + 3.0, + 1e-3, + -7.75, + 1e9, + -1e-9, + 123.456, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + -f64::NAN, + 2.0, + -4.0, + 0.1, + 65504.0, + -1e30, + 1e-30, + 1e300, + -1e-300, + f64::MIN_POSITIVE, + 7.0, +]; + +fn lanes_f64(seed: usize) -> [f64; N] { + core::array::from_fn(|i| POOL[(seed * 7 + i * 3) % POOL.len()]) +} +fn lanes_f32(seed: usize) -> [f32; N] { + lanes_f64::(seed).map(|x| x as f32) +} + +fn mask_of(pred: impl Fn(usize) -> bool, lanes: usize) -> u32 { + (0..lanes).filter(|&i| pred(i)).fold(0, |m, i| m | (1 << i)) +} + +#[test] +fn simd_float_classify_report_arm() { + println!( + "simd_float_classify: avx512f={} avx512vl={} avx2={} avx={} neon={} simd128={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx512vl"), + cfg!(target_feature = "avx2"), + cfg!(target_feature = "avx"), + cfg!(target_feature = "neon"), + cfg!(target_feature = "simd128"), + ); +} + +macro_rules! classify_parity { + ($vec:ident, $elem:ty, $lanes:expr, $gen:ident) => { + for seed in 0..60 { + let a: [$elem; $lanes] = $gen(seed); + let s: [$elem; $lanes] = $gen(seed + 4); + let (va, vs) = ($vec::from_array(a), $vec::from_array(s)); + let t = stringify!($vec); + assert_eq!(u32::from(va.is_nan().to_bitmask()), mask_of(|i| a[i].is_nan(), $lanes), "{t}::is_nan {a:?}"); + assert_eq!( + u32::from(va.is_finite().to_bitmask()), + mask_of(|i| a[i].is_finite(), $lanes), + "{t}::is_finite {a:?}" + ); + assert_eq!( + u32::from(va.is_infinite().to_bitmask()), + mask_of(|i| a[i].is_infinite(), $lanes), + "{t}::is_infinite {a:?}" + ); + let cs = va.copysign(vs).to_array(); + let rc = va.recip().to_array(); + let ra = va.recip_approx().to_array(); + for i in 0..$lanes { + assert_eq!( + cs[i].to_bits(), + a[i].copysign(s[i]).to_bits(), + "{t}::copysign lane {i}: {:?} sign {:?}", + a[i], + s[i] + ); + let want = 1.0 / a[i]; + assert!( + (rc[i].is_nan() && want.is_nan()) || rc[i].to_bits() == want.to_bits(), + "{t}::recip lane {i}: 1/{:?} got {:?} want {:?}", + a[i], + rc[i], + want + ); + if a[i].is_normal() && want.is_normal() { + let rel = ((ra[i] - want) / want).abs(); + assert!( + rel <= 1.0 / 2048.0, + "{t}::recip_approx lane {i}: 1/{:?} got {:?} want {:?} rel {:e}", + a[i], + ra[i], + want, + rel + ); + } + } + } + }; +} + +#[test] +fn classification_copysign_recip_match_scalar() { + classify_parity!(F32x16, f32, 16, lanes_f32); + classify_parity!(F64x8, f64, 8, lanes_f64); + classify_parity!(F32x8, f32, 8, lanes_f32); + classify_parity!(F64x4, f64, 4, lanes_f64); +} + +#[test] +fn recip_approx_dense_sweep_f32() { + // 4,096 normal f32 values spread over many binades and both signs. + let mut worst = 0.0f64; + for k in 0..256u32 { + let base: [f32; 16] = core::array::from_fn(|i| { + let e = (k % 60) as i32 - 30; + let m = 1.0 + (i as f32 + 16.0 * (k / 60) as f32) / 128.0; + let x = m * 2f32.powi(e); + if (k + i as u32).is_multiple_of(2) { + x + } else { + -x + } + }); + let got = F32x16::from_array(base).recip_approx().to_array(); + let got8 = F32x8::from_array(base[..8].try_into().unwrap()) + .recip_approx() + .to_array(); + for i in 0..16 { + let want = 1.0 / f64::from(base[i]); + worst = worst.max(((f64::from(got[i]) - want) / want).abs()); + if i < 8 { + worst = worst.max(((f64::from(got8[i]) - want) / want).abs()); + } + } + } + assert!(worst <= 1.0 / 2048.0, "recip_approx worst relative error {worst:e} > 2^-11"); + println!("recip_approx f32 worst relative error: {worst:e}"); +} From 3527e902dc22981f1a992f25ae3c441ed5c22685 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 14:50:53 +0000 Subject: [PATCH 06/13] =?UTF-8?q?simd:=20F6=20=E2=80=94=20U32x16::mul=5Fwi?= =?UTF-8?q?den=5Feven,=20U16x32/I16x32::mul=5Fhigh=20on=20all=20arms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan simd-fruits-integration-v1 F6 (drafted per backend file by Sonnet, reviewed and verified here; NEON I16x32 by the orchestrator). - mul_widen_even: exact u32*u32->u64 of the even lanes. AVX-512 vpmuludq; AVX2 two 256-bit vpmuludq, AVX-only arm four 128-bit SSE2; NEON vmovn_u64 + vmull_u32; wasm and-mask + i64x2_mul; nightly reinterpret+mask. - mul_high: vpmulhuw / vpmulhw (AVX-512BW, AVX2 256-bit, SSE2 128-bit); NEON I16x32 vmull_s16 + vshrn_n_s32::<16>; scalar and nightly widen-shift. - IFMA mul52 deferred (plan note): no non-IFMA realization fits R2 + W1a. Verified: tests/simd_widen_mul.rs 3/3 on native(v4), v3, avx, v2, aarch64 (qemu) and nightly-simd; disable run (AVX2 mulhi epu16 -> epi16) red; lib 2743 (v4) / 2692 (v3); doctests 6/6; masking-parity wasm, wasm-scalar, neon-qemu PASS; clippy -D warnings, fmt, facet-stride guard clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 9 ++ .claude/plans/simd-fruits-integration-v1.md | 4 + src/simd_avx2.rs | 170 ++++++++++++++++++++ src/simd_avx512.rs | 52 ++++++ src/simd_neon.rs | 67 ++++++++ src/simd_nightly/i_word_types.rs | 10 ++ src/simd_nightly/u_word_types.rs | 25 +++ src/simd_scalar.rs | 48 ++++++ src/simd_wasm.rs | 28 ++++ tests/simd_widen_mul.rs | 103 ++++++++++++ 10 files changed, 516 insertions(+) create mode 100644 tests/simd_widen_mul.rs diff --git a/.claude/blackboard.md b/.claude/blackboard.md index cfdabe7e..c2757244 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,12 @@ +## 2026-11-17 — LANDED: simd-fruits F6 (widening / high-half multiply) + +`U32x16::mul_widen_even -> U64x8` (vpmuludq; AVX2 2×256 / AVX 4×128 SSE2; NEON vmovn+vmull; +wasm and-mask + i64x2_mul; nightly transmute+mask), `U16x32::mul_high` / `I16x32::mul_high` +(vpmulhuw/vpmulhw; NEON I16x32 vmull+vshrn written by orchestrator). tests/simd_widen_mul.rs +3/3 on v4, v3, avx, v2, aarch64-qemu, nightly; disable (AVX2 epu16→epi16) red. IFMA mul52 deferred +(plan note). Sonnet card: 5 runs, 0 code defects; 1 brief defect (mine): NEON I16x32 is a NEON +polyfill, not scalar — orchestrator added it. + ## 2026-11-17 — LANDED: simd-fruits F1+F2+F3 and F5 (branch claude/practical-newton-u8lir8, PR #435) - F1+F2+F3 (orchestrator-written): full F32x8/F64x4 API with F32Mask8/F64Mask4 lane masks on every diff --git a/.claude/plans/simd-fruits-integration-v1.md b/.claude/plans/simd-fruits-integration-v1.md index 6cb18258..e86c210f 100644 --- a/.claude/plans/simd-fruits-integration-v1.md +++ b/.claude/plans/simd-fruits-integration-v1.md @@ -96,6 +96,10 @@ Today `F32x8` has `splat/from_slice/from_array/to_array/copy_to_slice/reduce_sum - `U64x8::mul52_lo`/`mul52_hi`: `vpmadd52luq`/`vpmadd52huq`. AVX-512IFMA only, which the host has. Gated by `cfg(target_feature = "avx512ifma")`; AVX2 and scalar use an exact u128 path. + ⊘ **Deferred 2026-11-17 (landing of F6):** `mul52_*` is not built. A non-IFMA + realization is a per-lane u128 loop, which R2 forbids outside `simd_scalar.rs`, + and an IFMA-only method would break the W1a "same API on every arm" rule. Revisit + with a consumer that needs 52-bit limbs. - Witness: parity test against u128 scalar products; codegen-witness. **F7. AVX2 integer burn-down, done in order of measured cost.** *(TD4; already on the burn-down list)* diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index d210792a..532aba97 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -1385,6 +1385,62 @@ avx2_int_type!(U8x64, u8, 64, 0u8); avx2_int_type!(I8x64, i8, 64, 0i8); avx2_int_type!(I16x32, i16, 32, 0i16); +impl I16x32 { + /// High half of the lane-wise widened product: lane `i` = + /// `(self[i] as i32 * rhs[i] as i32) >> 16`, as `i16`. + /// + /// Realized with `pmulh*` (`_mm256_mulhi_epi16` per 256-bit half on AVX2, + /// `_mm_mulhi_epi16` per 128-bit quarter on the AVX-without-AVX2 arm). + /// + /// # Examples + /// + /// ``` + /// use ndarray::simd::I16x32; + /// let r = I16x32::splat(-32768).mul_high(I16x32::splat(2)).to_array(); + /// assert_eq!(r, [-1; 32]); + /// ``` + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + let mut o = [0 as i16; 32]; + #[cfg(target_feature = "avx2")] + { + // SAFETY: AVX2 is enabled at compile time on this arm; `self.0`, `rhs.0` and `o` + // are 32-element i16 arrays (64 bytes), so the two 32-byte unaligned + // loads/stores at register offsets 0 and 1 stay in bounds; loadu/storeu need no alignment. + unsafe { + let a = self.0.as_ptr().cast::(); + let b = rhs.0.as_ptr().cast::(); + let p = o.as_mut_ptr().cast::(); + for k in 0..2 { + let r = core::arch::x86_64::_mm256_mulhi_epi16( + core::arch::x86_64::_mm256_loadu_si256(a.add(k)), + core::arch::x86_64::_mm256_loadu_si256(b.add(k)), + ); + core::arch::x86_64::_mm256_storeu_si256(p.add(k), r); + } + } + } + #[cfg(not(target_feature = "avx2"))] + { + // SAFETY: SSE2 is baseline on x86_64; the arrays are 64 bytes, so the four 16-byte + // unaligned loads/stores at register offsets 0..4 stay in bounds. + unsafe { + let a = self.0.as_ptr().cast::(); + let b = rhs.0.as_ptr().cast::(); + let p = o.as_mut_ptr().cast::(); + for k in 0..4 { + let r = core::arch::x86_64::_mm_mulhi_epi16( + core::arch::x86_64::_mm_loadu_si128(a.add(k)), + core::arch::x86_64::_mm_loadu_si128(b.add(k)), + ); + core::arch::x86_64::_mm_storeu_si128(p.add(k), r); + } + } + } + Self(o) + } +} + // I8x64 / I16x32: AVX2 scalar polyfill — methods matching the AVX-512BW API impl I8x64 { #[inline(always)] @@ -1756,6 +1812,120 @@ avx2_int_type!(U16x32, u16, 32, 0u16); avx2_int_type!(U32x16, u32, 16, 0u32); avx2_int_type!(U64x8, u64, 8, 0u64); +impl U16x32 { + /// High half of the lane-wise widened product: lane `i` = + /// `(self[i] as u32 * rhs[i] as u32) >> 16`, as `u16`. + /// + /// Realized with `pmulh*` (`_mm256_mulhi_epu16` per 256-bit half on AVX2, + /// `_mm_mulhi_epu16` per 128-bit quarter on the AVX-without-AVX2 arm). + /// + /// # Examples + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let r = U16x32::splat(0x8000).mul_high(U16x32::splat(4)).to_array(); + /// assert_eq!(r, [2; 32]); + /// ``` + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + let mut o = [0 as u16; 32]; + #[cfg(target_feature = "avx2")] + { + // SAFETY: AVX2 is enabled at compile time on this arm; `self.0`, `rhs.0` and `o` + // are 32-element u16 arrays (64 bytes), so the two 32-byte unaligned + // loads/stores at register offsets 0 and 1 stay in bounds; loadu/storeu need no alignment. + unsafe { + let a = self.0.as_ptr().cast::(); + let b = rhs.0.as_ptr().cast::(); + let p = o.as_mut_ptr().cast::(); + for k in 0..2 { + let r = core::arch::x86_64::_mm256_mulhi_epu16( + core::arch::x86_64::_mm256_loadu_si256(a.add(k)), + core::arch::x86_64::_mm256_loadu_si256(b.add(k)), + ); + core::arch::x86_64::_mm256_storeu_si256(p.add(k), r); + } + } + } + #[cfg(not(target_feature = "avx2"))] + { + // SAFETY: SSE2 is baseline on x86_64; the arrays are 64 bytes, so the four 16-byte + // unaligned loads/stores at register offsets 0..4 stay in bounds. + unsafe { + let a = self.0.as_ptr().cast::(); + let b = rhs.0.as_ptr().cast::(); + let p = o.as_mut_ptr().cast::(); + for k in 0..4 { + let r = core::arch::x86_64::_mm_mulhi_epu16( + core::arch::x86_64::_mm_loadu_si128(a.add(k)), + core::arch::x86_64::_mm_loadu_si128(b.add(k)), + ); + core::arch::x86_64::_mm_storeu_si128(p.add(k), r); + } + } + } + Self(o) + } +} + +impl U32x16 { + /// Full 64-bit product of the even `u32` lanes: lane `k` (0..8) = + /// `(self[2k] as u64) * (rhs[2k] as u64)`. Odd lanes are ignored. + /// + /// Realized with `pmuludq` (`_mm256_mul_epu32` per 256-bit half on AVX2, + /// `_mm_mul_epu32` per 128-bit quarter on the AVX-without-AVX2 arm). + /// + /// # Examples + /// + /// ``` + /// use ndarray::simd::U32x16; + /// let a = U32x16::splat(u32::MAX); + /// let w = a.mul_widen_even(a).to_array(); + /// assert_eq!(w, [u64::from(u32::MAX) * u64::from(u32::MAX); 8]); + /// ``` + #[inline(always)] + pub fn mul_widen_even(self, rhs: Self) -> U64x8 { + let mut o = [0u64; 8]; + #[cfg(target_feature = "avx2")] + { + // SAFETY: AVX2 is enabled at compile time on this arm; inputs are 64-byte arrays and + // `o` is 64 bytes, so the two 32-byte unaligned loads/stores at register offsets 0 + // and 1 stay in bounds. Register k yields u64 lanes 4k..4k+4. + unsafe { + let a = self.0.as_ptr().cast::(); + let b = rhs.0.as_ptr().cast::(); + let p = o.as_mut_ptr().cast::(); + for k in 0..2 { + let r = core::arch::x86_64::_mm256_mul_epu32( + core::arch::x86_64::_mm256_loadu_si256(a.add(k)), + core::arch::x86_64::_mm256_loadu_si256(b.add(k)), + ); + core::arch::x86_64::_mm256_storeu_si256(p.add(k), r); + } + } + } + #[cfg(not(target_feature = "avx2"))] + { + // SAFETY: SSE2 is baseline on x86_64; inputs and `o` are 64 bytes, so the four + // 16-byte unaligned loads/stores at register offsets 0..4 stay in bounds. + // Register k yields u64 lanes 2k..2k+2. + unsafe { + let a = self.0.as_ptr().cast::(); + let b = rhs.0.as_ptr().cast::(); + let p = o.as_mut_ptr().cast::(); + for k in 0..4 { + let r = core::arch::x86_64::_mm_mul_epu32( + core::arch::x86_64::_mm_loadu_si128(a.add(k)), + core::arch::x86_64::_mm_loadu_si128(b.add(k)), + ); + core::arch::x86_64::_mm_storeu_si128(p.add(k), r); + } + } + } + U64x8(o) + } +} + /// u64 ARX rotate — the BLAKE2b / argon2 lane, and the mask family's /// word rotate. /// diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 84af5372..14ddc49d 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -1928,6 +1928,23 @@ impl U16x32 { } } +impl U16x32 { + /// Per-lane high half of the unsigned 16x16 product: `((a * b) >> 16) as u16` (`vpmulhuw`). + /// + /// # Examples + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let r = U16x32::splat(0x8000).mul_high(U16x32::splat(4)).to_array(); + /// assert_eq!(r, [2u16; 32]); + /// ``` + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + // SAFETY: avx512bw/avx512f is compiled in on this arm; register-only. + Self(unsafe { _mm512_mulhi_epu16(self.0, rhs.0) }) + } +} + impl Default for U16x32 { #[inline(always)] fn default() -> Self { @@ -2230,6 +2247,24 @@ impl U32x16 { } } +impl U32x16 { + /// Full 64-bit product of the even u32 lanes (odd lanes ignored): lane k = `a[2k] * b[2k]` (`vpmuludq`). + /// + /// # Examples + /// + /// ``` + /// use ndarray::simd::U32x16; + /// let a = U32x16::splat(u32::MAX); + /// let w = a.mul_widen_even(a).to_array(); + /// assert_eq!(w, [u64::from(u32::MAX) * u64::from(u32::MAX); 8]); + /// ``` + #[inline(always)] + pub fn mul_widen_even(self, rhs: Self) -> U64x8 { + // SAFETY: avx512bw/avx512f is compiled in on this arm; register-only. + U64x8(unsafe { _mm512_mul_epu32(self.0, rhs.0) }) + } +} + impl U32x16 { pub const LANES: usize = 16; @@ -2959,6 +2994,23 @@ impl I16x32 { } } +impl I16x32 { + /// Per-lane high half of the signed 16x16 product: `((a * b) >> 16) as i16` (arithmetic shift, `vpmulhw`). + /// + /// # Examples + /// + /// ``` + /// use ndarray::simd::I16x32; + /// let r = I16x32::splat(-32768).mul_high(I16x32::splat(2)).to_array(); + /// assert_eq!(r, [-1i16; 32]); + /// ``` + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + // SAFETY: avx512bw/avx512f is compiled in on this arm; register-only. + Self(unsafe { _mm512_mulhi_epi16(self.0, rhs.0) }) + } +} + impl_bin_op!(I16x32, Add, add, _mm512_add_epi16); impl_bin_op!(I16x32, Sub, sub, _mm512_sub_epi16); impl_assign_op!(I16x32, AddAssign, add_assign, _mm512_add_epi16); diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 796da3f2..6153cb1f 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -2149,6 +2149,41 @@ impl U32x16 { } } +#[cfg(target_arch = "aarch64")] +impl U32x16 { + /// Widening multiply of the even `u32` lanes: output lane `k` (0..8) is + /// `(self[2k] as u64) * (rhs[2k] as u64)`, the exact 64-bit product. Odd + /// input lanes are ignored. + /// + /// Per 128-bit register: reinterpret as `u64x2`, narrow with `vmovn_u64` + /// (keeps the low `u32` of each `u64`, i.e. the even lanes), then + /// `vmull_u32` widens the multiply. + /// + /// # Examples + /// + /// ``` + /// use ndarray::simd::U32x16; + /// let a = U32x16::splat(u32::MAX); + /// let w = a.mul_widen_even(a).to_array(); + /// assert_eq!(w, [u64::from(u32::MAX) * u64::from(u32::MAX); 8]); + /// ``` + #[inline(always)] + pub fn mul_widen_even(self, rhs: Self) -> U64x8 { + #[inline(always)] + fn quad(a: U32x4, b: U32x4) -> U64x2 { + // SAFETY: NEON is baseline on aarch64; register-only narrowing + // and widening multiply, no memory access. + U64x2(unsafe { vmull_u32(vmovn_u64(vreinterpretq_u64_u32(a.0)), vmovn_u64(vreinterpretq_u64_u32(b.0))) }) + } + U64x8([ + quad(self.0[0], rhs.0[0]), + quad(self.0[1], rhs.0[1]), + quad(self.0[2], rhs.0[2]), + quad(self.0[3], rhs.0[3]), + ]) + } +} + #[cfg(target_arch = "aarch64")] impl core::ops::Add for U32x16 { type Output = Self; @@ -2600,6 +2635,38 @@ neon_int_polyfill!(I16x16, i16, 16, 0i16, u16); #[cfg(target_arch = "aarch64")] neon_int_polyfill!(I16x32, i16, 32, 0i16, u32); +#[cfg(target_arch = "aarch64")] +impl I16x32 { + /// High 16 bits of the 32-bit signed product, per lane: + /// `((self[i] as i32 * rhs[i] as i32) >> 16) as i16`. + /// + /// Per 128-bit register: `vmull_s16` / `vmull_high_s16` widen to i32 + /// products, `vshrn_n_s32::<16>` / `vshrn_high_n_s32::<16>` narrow the + /// high halves back (narrowing moves, no shuffle). + /// + /// ``` + /// use ndarray::simd::I16x32; + /// assert_eq!(I16x32::splat(-32768).mul_high(I16x32::splat(2)).to_array(), [-1; 32]); + /// ``` + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + let mut o = [0i16; 32]; + // SAFETY: NEON is baseline on aarch64. Each register k reads/writes + // lanes 8k..8k+8 of 32-lane arrays (k < 4), so every access is in + // bounds; vld1q/vst1q need no alignment. + unsafe { + for k in 0..4 { + let a = vld1q_s16(self.0.as_ptr().add(8 * k)); + let b = vld1q_s16(rhs.0.as_ptr().add(8 * k)); + let lo = vmull_s16(vget_low_s16(a), vget_low_s16(b)); + let hi = vmull_high_s16(a, b); + vst1q_s16(o.as_mut_ptr().add(8 * k), vshrn_high_n_s32::<16>(vshrn_n_s32::<16>(lo), hi)); + } + } + Self(o) + } +} + #[cfg(target_arch = "aarch64")] impl I16x32 { /// Narrow two 32×i16 vectors to one 64×i8 vector with signed saturation: lanes 0..32 diff --git a/src/simd_nightly/i_word_types.rs b/src/simd_nightly/i_word_types.rs index 25f33903..c7b102f1 100644 --- a/src/simd_nightly/i_word_types.rs +++ b/src/simd_nightly/i_word_types.rs @@ -985,3 +985,13 @@ impl PartialEq for I64x4 { self.to_array() == other.to_array() } } + +// F6 (plan simd-fruits-integration-v1): high-half multiply. +impl I16x32 { + /// High 16 bits of the 32-bit signed product, per lane (arithmetic shift). + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + let p = self.0.cast::() * rhs.0.cast::(); + Self((p >> core::simd::Simd::splat(16)).cast::()) + } +} diff --git a/src/simd_nightly/u_word_types.rs b/src/simd_nightly/u_word_types.rs index 33992a3b..46e16091 100644 --- a/src/simd_nightly/u_word_types.rs +++ b/src/simd_nightly/u_word_types.rs @@ -1345,3 +1345,28 @@ impl U32x16 { Self(r) } } + +// F6 (plan simd-fruits-integration-v1): widening and high-half multiply. +impl U32x16 { + /// Exact `u32 × u32 → u64` of the even lanes: output lane k is + /// `self[2k] * rhs[2k]`. The 64 bytes are reinterpreted as `u64x8` + /// (each u64 holds lanes 2k, 2k+1 little-endian), masked to the even + /// lane, and multiplied; the product of two values < 2^32 fits in u64. + #[inline(always)] + pub fn mul_widen_even(self, rhs: Self) -> U64x8 { + // SAFETY: `Simd` and `Simd` are both 64 bytes of + // plain integers with no invalid bit patterns. + let (a, b): (u64x8, u64x8) = unsafe { (core::mem::transmute(self.0), core::mem::transmute(rhs.0)) }; + let lo = u64x8::splat(0xFFFF_FFFF); + U64x8((a & lo) * (b & lo)) + } +} + +impl U16x32 { + /// High 16 bits of the 32-bit product, per lane. + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + let p = self.0.cast::() * rhs.0.cast::(); + Self((p >> core::simd::Simd::splat(16)).cast::()) + } +} diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index e7b34b96..6c6d6280 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -615,6 +615,38 @@ impl_int_type!(U16x32, u16, 32, 0u16); impl_int_type!(U32x16, u32, 16, 0u32); impl_int_type!(U64x8, u64, 8, 0u64); +impl U32x16 { + /// Full 64-bit product of the even `u32` lanes: lane `k` (0..8) is + /// `self[2k] * rhs[2k]` widened to `u64`. Odd lanes are ignored. Exact. + /// Mirrors `pmuludq` (`_mm*_mul_epu32`). + /// + /// ``` + /// use ndarray::simd::U32x16; + /// let a = U32x16::splat(u32::MAX); + /// let w = a.mul_widen_even(a).to_array(); + /// assert_eq!(w, [u64::from(u32::MAX) * u64::from(u32::MAX); 8]); + /// ``` + #[inline(always)] + pub fn mul_widen_even(self, rhs: Self) -> U64x8 { + U64x8(core::array::from_fn(|k| u64::from(self.0[2 * k]) * u64::from(rhs.0[2 * k]))) + } +} + +impl U16x32 { + /// High 16 bits of the unsigned 32-bit product, per lane: + /// `((a * b) >> 16) as u16`. Mirrors `pmulhuw` (`_mm*_mulhi_epu16`). + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let r = U16x32::splat(0x8000).mul_high(U16x32::splat(4)).to_array(); + /// assert_eq!(r, [2u16; 32]); + /// ``` + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + Self(core::array::from_fn(|i| ((u32::from(self.0[i]) * u32::from(rhs.0[i])) >> 16) as u16)) + } +} + impl U64x8 { /// Lane `j` gains the sum of the bytes `a[i]`, `i` in `8j..8j+8`, whose bit /// in `keep` is set: `vpsadbw` against zero on the kept bytes, accumulated @@ -730,6 +762,22 @@ impl U64x8 { impl_int_type!(I8x64, i8, 64, 0i8); impl_int_type!(I8x32, i8, 32, 0i8); impl_int_type!(I16x32, i16, 32, 0i16); + +impl I16x32 { + /// High 16 bits of the signed 32-bit product, per lane: + /// `((a * b) >> 16) as i16` (arithmetic shift). Mirrors `pmulhw` + /// (`_mm*_mulhi_epi16`). + /// + /// ``` + /// use ndarray::simd::I16x32; + /// let r = I16x32::splat(-32768).mul_high(I16x32::splat(2)).to_array(); + /// assert_eq!(r, [-1i16; 32]); + /// ``` + #[inline(always)] + pub fn mul_high(self, rhs: Self) -> Self { + Self(core::array::from_fn(|i| ((i32::from(self.0[i]) * i32::from(rhs.0[i])) >> 16) as i16)) + } +} impl_int_type!(I16x16, i16, 16, 0i16); // 256-bit int lanes — scalar polyfills filling the gap surfaced by the diff --git a/src/simd_wasm.rs b/src/simd_wasm.rs index b0491e29..feb3a575 100644 --- a/src/simd_wasm.rs +++ b/src/simd_wasm.rs @@ -1679,6 +1679,34 @@ pub mod wasm32_simd { } } + impl U32x16 { + /// Full 64-bit product of the EVEN u32 lanes: lane `k` (0..8) of the + /// result is `(self[2k] as u64) * (rhs[2k] as u64)`; odd lanes are + /// ignored. Exact, never wraps (`(2^32 - 1)^2 < 2^64`). + /// + /// Per `v128`: the high 32 bits of each u64 are cleared (leaving the even + /// u32 lane zero-extended), then one `i64x2.mul` — both factors are + /// below 2^32, so the wrapping 64-bit product is the exact u64 product. + /// + /// # Examples + /// + /// ``` + /// use ndarray::simd::U32x16; + /// let a = U32x16::splat(u32::MAX); + /// let w = a.mul_widen_even(a).to_array(); + /// assert_eq!(w, [u64::from(u32::MAX) * u64::from(u32::MAX); 8]); + /// ``` + #[inline(always)] + pub fn mul_widen_even(self, rhs: Self) -> U64x8 { + let low32 = u64x2_splat(0xFFFF_FFFF); + U64x8(core::array::from_fn(|k| { + let a = v128_and(self.0[k].0, low32); + let b = v128_and(rhs.0[k].0, low32); + U64x2(i64x2_mul(a, b)) + })) + } + } + impl Add for U32x16 { type Output = Self; #[inline(always)] diff --git a/tests/simd_widen_mul.rs b/tests/simd_widen_mul.rs new file mode 100644 index 00000000..84f07f77 --- /dev/null +++ b/tests/simd_widen_mul.rs @@ -0,0 +1,103 @@ +//! Parity for plan `simd-fruits-integration-v1` F6: `U32x16::mul_widen_even` +//! (exact u32×u32→u64 of the even lanes) and `U16x32::mul_high` / +//! `I16x32::mul_high` (high 16 bits of the 32-bit product), against `u64` / +//! `u32` / `i32` arithmetic computed here, so the file grades whichever arm +//! the build compiled. + +use ndarray::simd::{I16x32, U16x32, U32x16}; + +/// Small xorshift so the inputs cover the full range, including the extremes. +fn xs(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state +} + +#[test] +fn simd_widen_mul_report_arm() { + println!( + "simd_widen_mul: avx512f={} avx512bw={} avx2={} avx={} neon={} simd128={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx512bw"), + cfg!(target_feature = "avx2"), + cfg!(target_feature = "avx"), + cfg!(target_feature = "neon"), + cfg!(target_feature = "simd128"), + ); +} + +#[test] +fn mul_widen_even_matches_u64() { + let mut s = 0x9E37_79B9_7F4A_7C15u64; + let edge = [0u32, 1, 2, u32::MAX, u32::MAX - 1, 0x8000_0000, 0xFFFF, 0x1_0000]; + for round in 0..500 { + let a: [u32; 16] = core::array::from_fn(|i| { + if round < 8 { + edge[(i + round) % 8] + } else { + xs(&mut s) as u32 + } + }); + let b: [u32; 16] = core::array::from_fn(|i| { + if round < 8 { + edge[(i * 3 + round) % 8] + } else { + xs(&mut s) as u32 + } + }); + let got = U32x16::from_array(a) + .mul_widen_even(U32x16::from_array(b)) + .to_array(); + let want: [u64; 8] = core::array::from_fn(|k| u64::from(a[2 * k]) * u64::from(b[2 * k])); + assert_eq!(got, want, "mul_widen_even a={a:?} b={b:?}"); + } +} + +#[test] +fn mul_high_matches_u32_and_i32() { + let mut s = 0xD1B5_4A32_D192_ED03u64; + let ue = [0u16, 1, 2, u16::MAX, 0x8000, 0x7FFF, 0x0100, 0x00FF]; + let ie = [0i16, 1, -1, i16::MIN, i16::MAX, -2, 256, -256]; + for round in 0..500 { + let a: [u16; 32] = core::array::from_fn(|i| { + if round < 8 { + ue[(i + round) % 8] + } else { + xs(&mut s) as u16 + } + }); + let b: [u16; 32] = core::array::from_fn(|i| { + if round < 8 { + ue[(i * 5 + round) % 8] + } else { + xs(&mut s) as u16 + } + }); + let got = U16x32::from_array(a) + .mul_high(U16x32::from_array(b)) + .to_array(); + let want: [u16; 32] = core::array::from_fn(|i| ((u32::from(a[i]) * u32::from(b[i])) >> 16) as u16); + assert_eq!(got, want, "U16x32::mul_high a={a:?} b={b:?}"); + + let c: [i16; 32] = core::array::from_fn(|i| { + if round < 8 { + ie[(i + round) % 8] + } else { + xs(&mut s) as i16 + } + }); + let d: [i16; 32] = core::array::from_fn(|i| { + if round < 8 { + ie[(i * 3 + round) % 8] + } else { + xs(&mut s) as i16 + } + }); + let got = I16x32::from_array(c) + .mul_high(I16x32::from_array(d)) + .to_array(); + let want: [i16; 32] = core::array::from_fn(|i| ((i32::from(c[i]) * i32::from(d[i])) >> 16) as i16); + assert_eq!(got, want, "I16x32::mul_high c={c:?} d={d:?}"); + } +} From 775d95579514386396da685977e59c2e2c06dc6c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 15:04:44 +0000 Subject: [PATCH 07/13] simd: F7 AVX2 U8x64 burn-down + fix CI (no-std unused macro, ternlog drift) F7 (plan simd-fruits-integration-v1, drafted by Sonnet, verified here): AVX2 U8x64::saturating_add/sub compose two native U8x32 halves (vpaddusb/vpsubusb); mask_blend(u64) expands the bitmask to MSB byte masks with a shuffle-free SWAR multiply and blends with two vpblendvb. Measured D-ADD-PM on a pinned v3 build (best of 50): add_pm 285.6 -> 48.6 ns per 64 lanes, sub_pm 289.6 -> 52.1 ns; the decode path stays ~133 ns, so AVX2 add_pm now beats it 2.7x. CI fixes for failures since the F1 commit: - tests/*, nostd, blas-msrv: without the SIMD modules are not built and impl_bitmask_mask_api was unused under -D warnings; the module is now cfg(feature = "std") and the macro allows unused. - realization/avx2: the mask-pair logic macro sat after GEN-TERNLOG-END, which tools/gen_ternlog_bodies.py owns; moved above BEGIN. Verified: tests/simd_u8x64_blend.rs 3/3 and add_pm 11/11 on native(v4), v3, avx, v2; lib 2743 (v4) / 2692 (v3); masking-parity native, avx-qemu, v2-qemu PASS; cargo check --no-default-features (+approx,serde) clean with -D warnings; gen_ternlog_bodies.py --check current; clippy, fmt and facet-stride guard clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 14 +++++ src/lib.rs | 1 + src/simd_avx2.rs | 124 ++++++++++++++++++++++++-------------- src/simd_mask_api.rs | 1 + tests/simd_u8x64_blend.rs | 67 ++++++++++++++++++++ 5 files changed, 162 insertions(+), 45 deletions(-) create mode 100644 tests/simd_u8x64_blend.rs diff --git a/.claude/blackboard.md b/.claude/blackboard.md index c2757244..710dc54f 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,17 @@ +## 2026-11-17 — LANDED: simd-fruits F7 + CI fix (AVX2 U8x64 burn-down; D-ADD-PM loose end closed) + +AVX2 `U8x64::saturating_add/sub` are two `U8x32` vpaddusb/vpsubusb halves; `mask_blend(u64)` expands +the bits to MSB byte masks with a SWAR multiply (bits*0x0101.. & 0x8040201008040201, +0x7F | ., &0x80..) +— no shuffle, no table — then two vpblendvb. Measured D-ADD-PM, pinned v3, best of 50, same host: +add_pm 285.6 → 48.6 ns / 64 lanes (5.9×), sub_pm 289.6 → 52.1 ns; decode path 133–135 ns both runs, so +add_pm on AVX2 now beats decode 2.7× (was 0.47×). tests/simd_u8x64_blend.rs 3/3 + add_pm 11/11 on v4, +v3, avx, v2. Sonnet card: 1 run, 0 defects. +CI fix in the same commit (red since F1 on PR #435, caught by CI, not by my local gates): (1) no-std +builds left `impl_bitmask_mask_api` unused under -D warnings → module gated on `std`, lint allowed; +(2) appended code after `// GEN-TERNLOG-END` in simd_avx2.rs = generator drift → moved above BEGIN. +LESSON: add `cargo check --no-default-features` and `python3 tools/gen_ternlog_bodies.py --check` to +the local gate list for any simd_*.rs change. + ## 2026-11-17 — LANDED: simd-fruits F6 (widening / high-half multiply) `U32x16::mul_widen_even -> U64x8` (vpmuludq; AVX2 2×256 / AVX 4×128 SSE2; NEON vmovn+vmull; diff --git a/src/lib.rs b/src/lib.rs index d715cf5b..2617d805 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -236,6 +236,7 @@ mod dimension; /// Shared mask API macro (`from_bitmask`, `any`, `all`, `none`, `& | ^ !`) /// for the bitmask-backed SIMD masks; must precede every backend module. +#[cfg(feature = "std")] #[macro_use] mod simd_mask_api; diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 532aba97..0deb36e0 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -1636,9 +1636,12 @@ impl U8x64 { #[inline(always)] pub fn saturating_sub(self, other: Self) -> Self { let mut out = [0u8; 64]; - for i in 0..64 { - out[i] = self.0[i].saturating_sub(other.0[i]); - } + U8x32::from_slice(&self.0[..32]) + .saturating_sub(U8x32::from_slice(&other.0[..32])) + .copy_to_slice(&mut out[..32]); + U8x32::from_slice(&self.0[32..]) + .saturating_sub(U8x32::from_slice(&other.0[32..])) + .copy_to_slice(&mut out[32..]); Self(out) } @@ -1668,9 +1671,12 @@ impl U8x64 { #[inline(always)] pub fn mask_blend(mask: u64, a: Self, b: Self) -> Self { let mut out = [0u8; 64]; - for i in 0..64 { - out[i] = if mask & (1 << i) != 0 { b.0[i] } else { a.0[i] }; - } + let lo = U8x32::bits_to_msb_bytes(mask as u32); + let hi = U8x32::bits_to_msb_bytes((mask >> 32) as u32); + U8x32::mask_blend(lo, U8x32::from_slice(&b.0[..32]), U8x32::from_slice(&a.0[..32])) + .copy_to_slice(&mut out[..32]); + U8x32::mask_blend(hi, U8x32::from_slice(&b.0[32..]), U8x32::from_slice(&a.0[32..])) + .copy_to_slice(&mut out[32..]); Self(out) } #[inline(always)] @@ -1696,9 +1702,12 @@ impl U8x64 { #[inline(always)] pub fn saturating_add(self, other: Self) -> Self { let mut out = [0u8; 64]; - for i in 0..64 { - out[i] = self.0[i].saturating_add(other.0[i]); - } + U8x32::from_slice(&self.0[..32]) + .saturating_add(U8x32::from_slice(&other.0[..32])) + .copy_to_slice(&mut out[..32]); + U8x32::from_slice(&self.0[32..]) + .saturating_add(U8x32::from_slice(&other.0[32..])) + .copy_to_slice(&mut out[32..]); Self(out) } #[inline(always)] @@ -4148,6 +4157,31 @@ impl U8x32 { } } +#[cfg(target_arch = "x86_64")] +impl U8x32 { + /// Expand 32 mask bits into a 32-byte blend mask: byte `i` has its MSB + /// set iff bit `i` of `bits` is set (other bits of the byte are zero). + /// + /// SWAR in general-purpose registers, 8 lanes per `u64`, no shuffle and + /// no table: multiplying the byte by `0x0101..01` replicates it into all + /// eight bytes, `& 0x8040_2010_0804_0201` keeps bit `j` in byte `j`, so + /// each byte is 0 or a single power of two (<= 0x80). Adding `0x7F` to + /// each byte then carries into the MSB exactly when the byte is nonzero, + /// with no carry crossing a byte boundary; OR-ing the original back in + /// and masking with `0x80..80` leaves the MSB set iff the byte was nonzero. + #[inline(always)] + fn bits_to_msb_bytes(bits: u32) -> Self { + let mut arr = [0u8; 32]; + for q in 0..4usize { + let byte = u64::from((bits >> (8 * q)) & 0xFF); + let spread = byte.wrapping_mul(0x0101_0101_0101_0101) & 0x8040_2010_0804_0201; + let msb = (spread.wrapping_add(0x7F7F_7F7F_7F7F_7F7F) | spread) & 0x8080_8080_8080_8080; + arr[8 * q..8 * q + 8].copy_from_slice(&msb.to_le_bytes()); + } + Self::from_array(arr) + } +} + // Bitwise + arithmetic operator impls so consumers can use natural // `a + b`, `a & b`, etc. without method chaining. Match the U8x64 shape. @@ -5239,6 +5273,42 @@ impl U32x16 { } } +// Lane-wise logic on the AVX2 512-bit mask pairs: one op per half. +macro_rules! impl_mask_pair_logic { + ($mask:ident) => { + impl BitAnd for $mask { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: Self) -> Self { + Self(self.0 & rhs.0, self.1 & rhs.1) + } + } + impl BitOr for $mask { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: Self) -> Self { + Self(self.0 | rhs.0, self.1 | rhs.1) + } + } + impl BitXor for $mask { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + Self(self.0 ^ rhs.0, self.1 ^ rhs.1) + } + } + impl Not for $mask { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + Self(!self.0, !self.1) + } + } + }; +} +impl_mask_pair_logic!(F32Mask16); +impl_mask_pair_logic!(F64Mask8); + // GEN-TERNLOG-BEGIN (tools/gen_ternlog_bodies.py — regenerate, do not hand-edit) /// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function /// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most @@ -5292,39 +5362,3 @@ fn ternlog_two_input_u32x16(t: u8, a: U32x16, b: U32x16) -> U32x16 { } } // GEN-TERNLOG-END - -// Lane-wise logic on the AVX2 512-bit mask pairs: one op per half. -macro_rules! impl_mask_pair_logic { - ($mask:ident) => { - impl BitAnd for $mask { - type Output = Self; - #[inline(always)] - fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0, self.1 & rhs.1) - } - } - impl BitOr for $mask { - type Output = Self; - #[inline(always)] - fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0, self.1 | rhs.1) - } - } - impl BitXor for $mask { - type Output = Self; - #[inline(always)] - fn bitxor(self, rhs: Self) -> Self { - Self(self.0 ^ rhs.0, self.1 ^ rhs.1) - } - } - impl Not for $mask { - type Output = Self; - #[inline(always)] - fn not(self) -> Self { - Self(!self.0, !self.1) - } - } - }; -} -impl_mask_pair_logic!(F32Mask16); -impl_mask_pair_logic!(F64Mask8); diff --git a/src/simd_mask_api.rs b/src/simd_mask_api.rs index f134b351..a8c540db 100644 --- a/src/simd_mask_api.rs +++ b/src/simd_mask_api.rs @@ -18,6 +18,7 @@ /// /// `$vec` is the vector type whose comparisons produce this mask; it is used /// only for the doc examples. +#[allow(unused_macros)] // not every arm instantiates it (e.g. nightly-simd off x86) macro_rules! impl_bitmask_mask_api { ($mask:ident, $prim:ty, $lanes:expr, $vec:ident) => { impl $mask { diff --git a/tests/simd_u8x64_blend.rs b/tests/simd_u8x64_blend.rs new file mode 100644 index 00000000..da9434e7 --- /dev/null +++ b/tests/simd_u8x64_blend.rs @@ -0,0 +1,67 @@ +//! Parity for plan `simd-fruits-integration-v1` F7: `U8x64::mask_blend`, +//! `saturating_add` and `saturating_sub` against per-byte `u8` arithmetic, +//! on whichever arm the build compiled (the AVX2 arm realizes them as two +//! `U8x32` halves and a shuffle-free bit-to-byte mask expansion since F7). + +use ndarray::simd::U8x64; + +fn xs(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state +} + +#[test] +fn simd_u8x64_blend_report_arm() { + println!( + "simd_u8x64_blend: avx512f={} avx512bw={} avx2={} avx={} neon={} simd128={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx512bw"), + cfg!(target_feature = "avx2"), + cfg!(target_feature = "avx"), + cfg!(target_feature = "neon"), + cfg!(target_feature = "simd128"), + ); +} + +#[test] +fn mask_blend_takes_b_where_bit_set() { + let mut s = 0x1234_5678_9ABC_DEF1u64; + let mut masks: Vec = vec![0, u64::MAX, 1, 1 << 63, 0x5555_5555_5555_5555, 0xAAAA_AAAA_AAAA_AAAA]; + // every single bit, and every byte-aligned 0xFF window + masks.extend((0..64).map(|i| 1u64 << i)); + masks.extend((0..8).map(|q| 0xFFu64 << (8 * q))); + masks.extend((0..400).map(|_| xs(&mut s))); + let a: [u8; 64] = core::array::from_fn(|i| i as u8); + let b: [u8; 64] = core::array::from_fn(|i| 200u8.wrapping_add(i as u8)); + for &m in &masks { + let got = U8x64::mask_blend(m, U8x64::from_array(a), U8x64::from_array(b)).to_array(); + let want: [u8; 64] = core::array::from_fn(|i| if (m >> i) & 1 == 1 { b[i] } else { a[i] }); + assert_eq!(got, want, "mask_blend mask={m:#018x}"); + } +} + +#[test] +fn saturating_add_sub_match_u8() { + let mut s = 0xDEAD_BEEF_0BAD_F00Du64; + for round in 0..600 { + let a: [u8; 64] = core::array::from_fn(|i| match round { + 0 => 255, + 1 => 0, + 2 => i as u8 * 4, + _ => xs(&mut s) as u8, + }); + let b: [u8; 64] = core::array::from_fn(|i| match round { + 0 => 1, + 1 => 1, + 2 => 255 - i as u8, + _ => xs(&mut s) as u8, + }); + let (va, vb) = (U8x64::from_array(a), U8x64::from_array(b)); + let add: [u8; 64] = core::array::from_fn(|i| a[i].saturating_add(b[i])); + let sub: [u8; 64] = core::array::from_fn(|i| a[i].saturating_sub(b[i])); + assert_eq!(va.saturating_add(vb).to_array(), add, "saturating_add a={a:?} b={b:?}"); + assert_eq!(va.saturating_sub(vb).to_array(), sub, "saturating_sub a={a:?} b={b:?}"); + } +} From 7c51be918d70211e2288ef1e8e25c7554732f3b1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 15:06:53 +0000 Subject: [PATCH 08/13] tests: gate the simd-fruits parity tests on feature std ndarray::simd exists only with std; the tests/1.99.0 CI row builds the integration tests with --no-default-features and failed on the new files. Same #![cfg(feature = "std")] as tests/add_pm.rs. Verified: cargo test --no-run --no-default-features with -D warnings builds clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- tests/simd_float_classify.rs | 1 + tests/simd_lane_masks.rs | 1 + tests/simd_u8x64_blend.rs | 1 + tests/simd_widen_mul.rs | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/simd_float_classify.rs b/tests/simd_float_classify.rs index b3e60db2..a6dc0e71 100644 --- a/tests/simd_float_classify.rs +++ b/tests/simd_float_classify.rs @@ -8,6 +8,7 @@ //! a relative error of 2^-11 on normal inputs whose reciprocal is normal //! (the documented contract; AVX-512 `vrcp14` is 2^-14, AVX `vrcpps` //! 1.5·2^-12, the other arms are exact). +#![cfg(feature = "std")] use ndarray::simd::{F32x16, F32x8, F64x4, F64x8}; diff --git a/tests/simd_lane_masks.rs b/tests/simd_lane_masks.rs index 07117424..cfecfe6d 100644 --- a/tests/simd_lane_masks.rs +++ b/tests/simd_lane_masks.rs @@ -15,6 +15,7 @@ //! NaN in min/max is deliberately not graded: x86 `vminps` returns the second //! operand while the scalar arm's `f32::min` returns the non-NaN one; that //! difference predates this plan and is documented on the methods. +#![cfg(feature = "std")] use ndarray::simd::{F32Mask16, F32Mask8, F32x16, F32x8, F64Mask4, F64Mask8, F64x4, F64x8}; diff --git a/tests/simd_u8x64_blend.rs b/tests/simd_u8x64_blend.rs index da9434e7..2e39280c 100644 --- a/tests/simd_u8x64_blend.rs +++ b/tests/simd_u8x64_blend.rs @@ -2,6 +2,7 @@ //! `saturating_add` and `saturating_sub` against per-byte `u8` arithmetic, //! on whichever arm the build compiled (the AVX2 arm realizes them as two //! `U8x32` halves and a shuffle-free bit-to-byte mask expansion since F7). +#![cfg(feature = "std")] use ndarray::simd::U8x64; diff --git a/tests/simd_widen_mul.rs b/tests/simd_widen_mul.rs index 84f07f77..b9a63adb 100644 --- a/tests/simd_widen_mul.rs +++ b/tests/simd_widen_mul.rs @@ -3,6 +3,7 @@ //! `I16x32::mul_high` (high 16 bits of the 32-bit product), against `u64` / //! `u32` / `i32` arithmetic computed here, so the file grades whichever arm //! the build compiled. +#![cfg(feature = "std")] use ndarray::simd::{I16x32, U16x32, U32x16}; From 2cd32d72ec2b209c8b4ae4b7dad3c631ff65bdfc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 15:21:39 +0000 Subject: [PATCH 09/13] =?UTF-8?q?simd:=20F4=20=E2=80=94=20load=5Fpartial?= =?UTF-8?q?=20/=20store=5Fpartial=20(masked=20tails)=20on=20the=20chunk=20?= =?UTF-8?q?types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan simd-fruits-integration-v1 F4, drafted by Sonnet in one new file (src/simd_partial.rs), reviewed and verified here. load_partial(&[T]) takes lanes 0..min(len, LANES) and zero-fills the rest; store_partial(&mut [T]) writes only lanes 0..min(len, LANES). Types: F32x16, F64x8, F32x8, F64x4, U8x64, I32x16, U32x16. - AVX-512: _mm512_maskz_loadu_* / _mm512_mask_storeu_* with a low-n-bit k-mask (AVX-512BW for U8x64); masked-off lanes are never accessed. - AVX (F32x8 / F64x4): vmaskmovps / vmaskmovpd with an i < n lane mask. - every other arm/type: one copy_from_slice into a zeroed buffer + load. Verified: tests/simd_partial.rs 3/3 on native(v4), v3, avx, v2, aarch64 (qemu) and nightly-simd; disable run (mask enables lane n) crashes with SIGSEGV on the empty-slice case, so the correct mask provably never touches the dangling pointer; lib 2743 (v4) / 2692 (v3); doctests 14/14; masking- parity wasm, wasm-scalar, neon-qemu, avx-qemu PASS; clippy -D warnings, fmt, facet-stride guard, ternlog generator check and the no-default- features test build all clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 12 ++ src/lib.rs | 5 + src/simd_partial.rs | 364 ++++++++++++++++++++++++++++++++++++++++++ tests/simd_partial.rs | 70 ++++++++ 4 files changed, 451 insertions(+) create mode 100644 src/simd_partial.rs create mode 100644 tests/simd_partial.rs diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 710dc54f..0eb78c79 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,15 @@ +## 2026-11-17 — LANDED: simd-fruits F4 (masked tail load/store) — P0 F1/F2 + P1 F4–F7 complete + +`load_partial(&[T])` / `store_partial(&mut [T])` on F32x16, F64x8, F32x8, F64x4, U8x64, I32x16, U32x16 +(src/simd_partial.rs, one Sonnet worker). AVX-512: maskz_loadu / mask_storeu with k = low-n bits (BW for +U8x64); AVX 256-bit floats: vmaskmovps/pd; everything else: one memcpy into a zeroed buffer + full load. +tests/simd_partial.rs 3/3 on v4, v3, avx, v2, aarch64-qemu, nightly (every tail length 0..=LANES+1, +store writes nothing outside the window). Disable run (mask16 enables lane n) → SIGSEGV on the empty- +slice case: with the correct mask the dangling pointer of an empty slice is never dereferenced — direct +evidence that masked-off lanes are not accessed (the fault-safety property). Sonnet card: 1 run, 0 defects. +Local gate list now also: `cargo test --no-run --no-default-features` (RUSTFLAGS=-Dwarnings) and +`python3 tools/gen_ternlog_bodies.py --check`. + ## 2026-11-17 — LANDED: simd-fruits F7 + CI fix (AVX2 U8x64 burn-down; D-ADD-PM loose end closed) AVX2 `U8x64::saturating_add/sub` are two `U8x32` vpaddusb/vpsubusb halves; `mask_blend(u64)` expands diff --git a/src/lib.rs b/src/lib.rs index 2617d805..da387b8a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -240,6 +240,11 @@ mod dimension; #[macro_use] mod simd_mask_api; +/// Masked tail load/store (`load_partial` / `store_partial`) on the SIMD +/// chunk types (plan simd-fruits-integration-v1 F4). +#[cfg(feature = "std")] +mod simd_partial; + /// Portable SIMD types — `crate::simd::f32x16` today, `std::simd::f32x16` tomorrow. #[cfg(feature = "std")] #[allow(clippy::all, missing_docs, dead_code, unused_variables, unused_imports)] diff --git a/src/simd_partial.rs b/src/simd_partial.rs new file mode 100644 index 00000000..451d85d7 --- /dev/null +++ b/src/simd_partial.rs @@ -0,0 +1,364 @@ +//! Masked tail load/store (`load_partial` / `store_partial`) for the chunk types — plan +//! `.claude/plans/simd-fruits-integration-v1.md` F4. +//! +//! Every chunk loop over a slice ends with a tail shorter than one register. +//! `load_partial` reads lanes `0..min(s.len(), LANES)` and zero-fills the rest; +//! `store_partial` writes only lanes `0..min(s.len(), LANES)` and touches +//! nothing else of the slice. +//! +//! Two realizations, selected by `cfg`: +//! +//! * **A, portable** (every arm unless B applies): the tail is staged through a +//! zeroed stack array with one `copy_from_slice` memcpy, then +//! `from_array` / `to_array`. +//! * **B, native masked memory ops** (fault-free: masked-off lanes are never +//! read or written): +//! * AVX-512F: `F32x16`, `F64x8`, `I32x16`, `U32x16` use `maskz_loadu` / +//! `mask_storeu` with a low-`n`-bits `k` register; `U8x64` additionally +//! needs AVX-512BW. +//! * AVX: `F32x8` and `F64x4` use `vmaskmov` with a lane-index compare mask. +//! +//! The `nightly-simd` backend always takes realization A. + +use crate::simd::{F32x16, F32x8, F64x4, F64x8, I32x16, U32x16, U8x64}; + +/// Generates `load_partial` / `store_partial` for one type. The load and store +/// bodies are supplied as blocks so both realizations share docs and framing. +macro_rules! partial_impl { + ( + $ty:ident, $elem:ty, $lanes:expr, + [$one:literal $two:literal $three:literal $zero:literal $nine:literal $five:literal], + load($s:ident, $n:ident) $load:block, + store($this:ident, $t:ident, $m:ident) $store:block + ) => { + impl $ty { + #[doc = concat!( + "Loads lanes `0..min(s.len(), LANES)` from `s` and zeroes the rest.\n\n", + "Never reads past the end of `s`.\n\n", + "```\n", + "use ndarray::simd::", stringify!($ty), ";\n", + "let v = ", stringify!($ty), "::load_partial(&[", stringify!($one), ", ", + stringify!($two), ", ", stringify!($three), "]);\n", + "assert_eq!(v.to_array()[..4], [", stringify!($one), ", ", + stringify!($two), ", ", stringify!($three), ", ", stringify!($zero), "]);\n", + "```" + )] + #[inline(always)] + pub fn load_partial($s: &[$elem]) -> Self { + let $n = $s.len().min($lanes); + $load + } + + #[doc = concat!( + "Writes lanes `0..min(s.len(), LANES)` into `s`; nothing else is touched.\n\n", + "Elements of `s` at index `LANES` and beyond are left unchanged.\n\n", + "```\n", + "use ndarray::simd::", stringify!($ty), ";\n", + "let mut out = [", stringify!($nine), "; 3];\n", + stringify!($ty), "::splat(", stringify!($five), ").store_partial(&mut out);\n", + "assert_eq!(out, [", stringify!($five), "; 3]);\n", + "```" + )] + #[inline(always)] + pub fn store_partial(self, $t: &mut [$elem]) { + let $this = self; + let $m = $t.len().min($lanes); + $store + } + } + }; +} + +/// Realization A: stage the tail through a zeroed stack array (one memcpy). +#[allow(unused_macros)] // the AVX-512 arm realizes every type natively +macro_rules! partial_via_buffer { + ($ty:ident, $elem:ty, $lanes:expr, $doc:tt) => { + partial_impl!( + $ty, $elem, $lanes, $doc, + load(s, n) { + let mut buf = [<$elem as Default>::default(); $lanes]; + buf[..n].copy_from_slice(&s[..n]); + Self::from_array(buf) + }, + store(this, t, n) { + t[..n].copy_from_slice(&this.to_array()[..n]); + } + ); + }; +} + +// --------------------------------------------------------------------------- +// Native helpers (private): low-n-bit k masks. +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd")))] +mod k_masks { + use core::arch::x86_64::{__mmask16, __mmask8}; + + /// Low `n` bits set (`n <= 16`), without overflow at `n == 16`. + #[inline(always)] + pub(super) fn mask16(n: usize) -> __mmask16 { + if n >= 16 { + 0xFFFF + } else { + (1u16 << n) - 1 + } + } + + /// Low `n` bits set (`n <= 8`), without overflow at `n == 8`. + #[inline(always)] + pub(super) fn mask8(n: usize) -> __mmask8 { + if n >= 8 { + 0xFF + } else { + (1u8 << n) - 1 + } + } +} + +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx512f", + target_feature = "avx512bw", + not(feature = "nightly-simd") +))] +mod k_masks64 { + use core::arch::x86_64::__mmask64; + + /// Low `n` bits set (`n <= 64`), without overflow at `n == 64`. + #[inline(always)] + pub(super) fn mask64(n: usize) -> __mmask64 { + if n >= 64 { + u64::MAX + } else { + (1u64 << n) - 1 + } + } +} + +// --------------------------------------------------------------------------- +// F32x16 (AVX-512F masked move, else buffer) +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd")))] +partial_impl!( + F32x16, f32, 16, [1.0 2.0 3.0 0.0 9.0 5.0], + load(s, n) { + // SAFETY: the pointer comes from a live slice; only the first n <= s.len() + // lanes are enabled in k, and AVX-512 masked loads do not access (or fault + // on) disabled lanes; loadu needs no alignment. + unsafe { Self(core::arch::x86_64::_mm512_maskz_loadu_ps(k_masks::mask16(n), s.as_ptr())) } + }, + store(this, t, n) { + // SAFETY: the pointer comes from a live mutable slice; only the first + // n <= t.len() lanes are enabled in k, and AVX-512 masked stores do not + // access (or fault on) disabled lanes; storeu needs no alignment. + unsafe { + core::arch::x86_64::_mm512_mask_storeu_ps(t.as_mut_ptr(), k_masks::mask16(n), this.0) + } + } +); +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd"))))] +partial_via_buffer!(F32x16, f32, 16, [1.0 2.0 3.0 0.0 9.0 5.0]); + +// --------------------------------------------------------------------------- +// F64x8 +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd")))] +partial_impl!( + F64x8, f64, 8, [1.0 2.0 3.0 0.0 9.0 5.0], + load(s, n) { + // SAFETY: the pointer comes from a live slice; only the first n <= s.len() + // lanes are enabled in k, and AVX-512 masked loads do not access (or fault + // on) disabled lanes; loadu needs no alignment. + unsafe { Self(core::arch::x86_64::_mm512_maskz_loadu_pd(k_masks::mask8(n), s.as_ptr())) } + }, + store(this, t, n) { + // SAFETY: the pointer comes from a live mutable slice; only the first + // n <= t.len() lanes are enabled in k, and AVX-512 masked stores do not + // access (or fault on) disabled lanes; storeu needs no alignment. + unsafe { + core::arch::x86_64::_mm512_mask_storeu_pd(t.as_mut_ptr(), k_masks::mask8(n), this.0) + } + } +); +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd"))))] +partial_via_buffer!(F64x8, f64, 8, [1.0 2.0 3.0 0.0 9.0 5.0]); + +// --------------------------------------------------------------------------- +// I32x16 +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd")))] +partial_impl!( + I32x16, i32, 16, [1 2 3 0 9 5], + load(s, n) { + // SAFETY: the pointer comes from a live slice; only the first n <= s.len() + // lanes are enabled in k, and AVX-512 masked loads do not access (or fault + // on) disabled lanes; loadu needs no alignment. + unsafe { + Self(core::arch::x86_64::_mm512_maskz_loadu_epi32(k_masks::mask16(n), s.as_ptr().cast())) + } + }, + store(this, t, n) { + // SAFETY: the pointer comes from a live mutable slice; only the first + // n <= t.len() lanes are enabled in k, and AVX-512 masked stores do not + // access (or fault on) disabled lanes; storeu needs no alignment. + unsafe { + core::arch::x86_64::_mm512_mask_storeu_epi32( + t.as_mut_ptr().cast(), + k_masks::mask16(n), + this.0, + ) + } + } +); +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd"))))] +partial_via_buffer!(I32x16, i32, 16, [1 2 3 0 9 5]); + +// --------------------------------------------------------------------------- +// U32x16 +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd")))] +partial_impl!( + U32x16, u32, 16, [1 2 3 0 9 5], + load(s, n) { + // SAFETY: the pointer comes from a live slice; only the first n <= s.len() + // lanes are enabled in k, and AVX-512 masked loads do not access (or fault + // on) disabled lanes; loadu needs no alignment. + unsafe { + Self(core::arch::x86_64::_mm512_maskz_loadu_epi32(k_masks::mask16(n), s.as_ptr().cast())) + } + }, + store(this, t, n) { + // SAFETY: the pointer comes from a live mutable slice; only the first + // n <= t.len() lanes are enabled in k, and AVX-512 masked stores do not + // access (or fault on) disabled lanes; storeu needs no alignment. + unsafe { + core::arch::x86_64::_mm512_mask_storeu_epi32( + t.as_mut_ptr().cast(), + k_masks::mask16(n), + this.0, + ) + } + } +); +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd"))))] +partial_via_buffer!(U32x16, u32, 16, [1 2 3 0 9 5]); + +// --------------------------------------------------------------------------- +// U8x64 (AVX-512F + AVX-512BW masked byte move, else buffer) +// --------------------------------------------------------------------------- + +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx512f", + target_feature = "avx512bw", + not(feature = "nightly-simd") +))] +partial_impl!( + U8x64, u8, 64, [1 2 3 0 9 5], + load(s, n) { + // SAFETY: the pointer comes from a live slice; only the first n <= s.len() + // lanes are enabled in k, and AVX-512BW masked loads do not access (or + // fault on) disabled lanes; loadu needs no alignment. + unsafe { + Self(core::arch::x86_64::_mm512_maskz_loadu_epi8(k_masks64::mask64(n), s.as_ptr().cast())) + } + }, + store(this, t, n) { + // SAFETY: the pointer comes from a live mutable slice; only the first + // n <= t.len() lanes are enabled in k, and AVX-512BW masked stores do not + // access (or fault on) disabled lanes; storeu needs no alignment. + unsafe { + core::arch::x86_64::_mm512_mask_storeu_epi8( + t.as_mut_ptr().cast(), + k_masks64::mask64(n), + this.0, + ) + } + } +); +#[cfg(not(all( + target_arch = "x86_64", + target_feature = "avx512f", + target_feature = "avx512bw", + not(feature = "nightly-simd") +)))] +partial_via_buffer!(U8x64, u8, 64, [1 2 3 0 9 5]); + +// --------------------------------------------------------------------------- +// F32x8 (AVX vmaskmov, else buffer) +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "x86_64", target_feature = "avx", not(feature = "nightly-simd")))] +partial_impl!( + F32x8, f32, 8, [1.0 2.0 3.0 0.0 9.0 5.0], + load(s, n) { + use core::arch::x86_64::*; + // SAFETY: the pointer comes from a live slice; lane i is active iff + // i < n <= s.len(), and vmaskmov suppresses faults and does not access + // masked-off elements; no alignment needed. + unsafe { + let lt = _mm256_cmp_ps::<_CMP_LT_OQ>( + _mm256_setr_ps(0., 1., 2., 3., 4., 5., 6., 7.), + _mm256_set1_ps(n as f32), + ); + Self(_mm256_maskload_ps(s.as_ptr(), _mm256_castps_si256(lt))) + } + }, + store(this, t, n) { + use core::arch::x86_64::*; + // SAFETY: the pointer comes from a live mutable slice; lane i is active + // iff i < n <= t.len(), and vmaskmov suppresses faults and does not + // access masked-off elements; no alignment needed. + unsafe { + let lt = _mm256_cmp_ps::<_CMP_LT_OQ>( + _mm256_setr_ps(0., 1., 2., 3., 4., 5., 6., 7.), + _mm256_set1_ps(n as f32), + ); + _mm256_maskstore_ps(t.as_mut_ptr(), _mm256_castps_si256(lt), this.0) + } + } +); +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx", not(feature = "nightly-simd"))))] +partial_via_buffer!(F32x8, f32, 8, [1.0 2.0 3.0 0.0 9.0 5.0]); + +// --------------------------------------------------------------------------- +// F64x4 (AVX vmaskmov, else buffer) +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "x86_64", target_feature = "avx", not(feature = "nightly-simd")))] +partial_impl!( + F64x4, f64, 4, [1.0 2.0 3.0 0.0 9.0 5.0], + load(s, n) { + use core::arch::x86_64::*; + // SAFETY: the pointer comes from a live slice; lane i is active iff + // i < n <= s.len(), and vmaskmov suppresses faults and does not access + // masked-off elements; no alignment needed. + unsafe { + let lt = _mm256_cmp_pd::<_CMP_LT_OQ>( + _mm256_setr_pd(0., 1., 2., 3.), + _mm256_set1_pd(n as f64), + ); + Self(_mm256_maskload_pd(s.as_ptr(), _mm256_castpd_si256(lt))) + } + }, + store(this, t, n) { + use core::arch::x86_64::*; + // SAFETY: the pointer comes from a live mutable slice; lane i is active + // iff i < n <= t.len(), and vmaskmov suppresses faults and does not + // access masked-off elements; no alignment needed. + unsafe { + let lt = _mm256_cmp_pd::<_CMP_LT_OQ>( + _mm256_setr_pd(0., 1., 2., 3.), + _mm256_set1_pd(n as f64), + ); + _mm256_maskstore_pd(t.as_mut_ptr(), _mm256_castpd_si256(lt), this.0) + } + } +); +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx", not(feature = "nightly-simd"))))] +partial_via_buffer!(F64x4, f64, 4, [1.0 2.0 3.0 0.0 9.0 5.0]); diff --git a/tests/simd_partial.rs b/tests/simd_partial.rs new file mode 100644 index 00000000..54a55791 --- /dev/null +++ b/tests/simd_partial.rs @@ -0,0 +1,70 @@ +//! Parity for plan `simd-fruits-integration-v1` F4: `load_partial` / +//! `store_partial` on the seven chunk types, for every tail length +//! `0..=LANES` (and one longer slice), on whichever arm the build compiled. +//! +//! The store test also checks that nothing past `min(len, LANES)` is written: +//! the destination slice is a window into a larger, sentinel-filled buffer. +#![cfg(feature = "std")] + +use ndarray::simd::{F32x16, F32x8, F64x4, F64x8, I32x16, U32x16, U8x64}; + +#[test] +fn simd_partial_report_arm() { + println!( + "simd_partial: avx512f={} avx512bw={} avx2={} avx={} neon={} simd128={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx512bw"), + cfg!(target_feature = "avx2"), + cfg!(target_feature = "avx"), + cfg!(target_feature = "neon"), + cfg!(target_feature = "simd128"), + ); +} + +macro_rules! partial_parity { + ($vec:ident, $elem:ty, $lanes:expr, $val:expr, $sentinel:expr, $zero:expr) => {{ + let src: Vec<$elem> = (0usize..$lanes + 3).map($val).collect(); + for len in 0..=$lanes + 1 { + // load: lanes < len come from the slice, the rest are zero + let got = $vec::load_partial(&src[..len]).to_array(); + for i in 0..$lanes { + let want = if i < len { src[i] } else { $zero }; + assert_eq!(got[i], want, "{}::load_partial len {} lane {}", stringify!($vec), len, i); + } + // store: exactly min(len, LANES) elements written, neighbours untouched + let mut buf: Vec<$elem> = vec![$sentinel; $lanes + 6]; + $vec::from_array(core::array::from_fn(|i| src[i])).store_partial(&mut buf[2..2 + len]); + let n = len.min($lanes); + for (j, &x) in buf.iter().enumerate() { + let want = if j >= 2 && j < 2 + n { src[j - 2] } else { $sentinel }; + assert_eq!(x, want, "{}::store_partial len {} index {}", stringify!($vec), len, j); + } + } + }}; +} + +#[test] +fn load_store_partial_every_tail_length() { + partial_parity!(F32x16, f32, 16, |i| i as f32 + 0.5, -7.0f32, 0.0f32); + partial_parity!(F64x8, f64, 8, |i| i as f64 * 1.25, -7.0f64, 0.0f64); + partial_parity!(F32x8, f32, 8, |i| 100.0 - i as f32, -7.0f32, 0.0f32); + partial_parity!(F64x4, f64, 4, |i| i as f64 - 3.0, -7.0f64, 0.0f64); + partial_parity!(U8x64, u8, 64, |i| (i as u8).wrapping_mul(37).wrapping_add(1), 0xEEu8, 0u8); + partial_parity!(I32x16, i32, 16, |i| i as i32 * -1000 - 1, i32::MIN, 0i32); + partial_parity!(U32x16, u32, 16, |i| 0xF000_0000u32 | i as u32, 0xDEAD_BEEFu32, 0u32); +} + +#[test] +fn tail_loop_sums_unaligned_lengths() { + // The use the API exists for: a chunked reduction with a ragged tail. + for n in [0usize, 1, 15, 16, 17, 31, 33, 100, 255] { + let data: Vec = (0..n).map(|i| (i % 7) as f32).collect(); + let mut acc = F32x16::splat(0.0); + let (chunks, tail) = data.as_chunks::<16>(); + for c in chunks { + acc += F32x16::from_array(*c); + } + acc += F32x16::load_partial(tail); + assert_eq!(acc.reduce_sum(), data.iter().sum::(), "n = {n}"); + } +} From 0553c7b982bf85375b28890dd0c106bb45052bf0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 16:54:37 +0000 Subject: [PATCH 10/13] =?UTF-8?q?simd:=20revert=20the=20two=20breaking=20c?= =?UTF-8?q?hanges=20of=20F2=20=E2=80=94=20additions=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator ruling: breaking changes are not acceptable. This restores both exactly, keeping the vectorized internals. 1. AVX2 F32Mask16 / F64Mask8 are `pub u16` / `pub u8` bitmasks again, byte-identical struct definitions to master; hpc/splat3d/raster.rs is back to master. Compares still run per 256-bit half (vcmpp* packed with vmovmskp* via a crate-internal from_halves); select re-expands each half with F32Mask8/F64Mask4::from_bitmask and blends with vblendvp*. The new operators/any/all/none/from_bitmask come from impl_bitmask_mask_api!. 2. AVX2 simd_min / simd_max / reduce_min / reduce_max keep master's f32::min NaN semantics: the F32x8/F64x4 halves use vminp*/vmaxp* plus an unordered compare that blends in self where other is NaN; reductions are a NaN-ignoring tree and return +/-INFINITY for an all-NaN vector, the same as master's fold(INFINITY, f32::min). Verified: every pub fn/struct/type/use/mod/const line of master is present unchanged on the branch (14 files, 0 missing); new compatibility tests nan_min_max_reduce_* go red when the NaN blend is removed; lib 2743 (v4) / 2692 (v3); all five parity test files pass on native(v4), v3, avx, v2, aarch64 (qemu) and nightly-simd; masking-parity native, wasm, wasm-scalar, neon-qemu, avx-qemu, v2-qemu PASS; doctests 89/89; clippy -D warnings, fmt, facet-stride guard, ternlog generator check and no-default-features test build clean. Plan F2 and the blackboard record the correction and the additions-only rule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 19 ++ .claude/plans/simd-fruits-integration-v1.md | 10 ++ src/hpc/splat3d/raster.rs | 2 +- src/simd_avx2.rs | 188 ++++++-------------- src/simd_avx512.rs | 175 +++++++++++++----- tests/simd_lane_masks.rs | 66 +++++++ 6 files changed, 282 insertions(+), 178 deletions(-) diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 0eb78c79..4664b0a5 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,22 @@ +## 2026-11-17 — CORRECTION: simd-fruits breaking changes reverted (operator: "absolutely unacceptable") + +Two breaks had shipped on PR #435 and are removed: +1. AVX2 `F32Mask16(pub u16)` / `F64Mask8(pub u8)` had become private lane-mask pairs → restored byte-for- + byte (struct lines identical to master); `hpc/splat3d/raster.rs` reverted to master. The speedup stays: + compares = per-half `vcmpp*` packed with `vmovmskp*` (`from_halves`), `select` = `from_bitmask` + + `vblendvp*`; operators/any/all/none come from `impl_bitmask_mask_api!` (additive). +2. AVX2 `simd_min/max` had switched to vminps NaN rule → restored `f32::min` semantics: `F32x8/F64x4` + simd_min/max = `vminp*` + `cmpunord(other)` blend of `self`; reduce_* = NaN-ignoring tree, all-NaN → + ±INFINITY (= master's `fold(INFINITY, f32::min)`). +Evidence: every `pub fn/struct/type/use/mod/const` line of master exists unchanged on the branch (14 files, +0 missing); new compat tests `nan_min_max_reduce_*` in tests/simd_lane_masks.rs (disable run: drop the NaN +blend → both red); lib 2743 (v4) / 2692 (v3); all 5 parity test files on v4, v3, avx, v2, aarch64-qemu, +nightly; masking-parity 6 arms PASS; clippy/guard/gen-check/no-std clean. Nightly backend left exactly as +on master (its all-NaN reduce returns NaN — pre-existing, test exempts only that case on nightly). +RULE (recorded in plan F2): this plan is additions-only — no existing public field, signature or result +changes. Residual, not a break: new inherent methods (e.g. `is_nan`, `any`) take precedence over a +downstream trait method of the same name on these types. + ## 2026-11-17 — LANDED: simd-fruits F4 (masked tail load/store) — P0 F1/F2 + P1 F4–F7 complete `load_partial(&[T])` / `store_partial(&mut [T])` on F32x16, F64x8, F32x8, F64x4, U8x64, I32x16, U32x16 diff --git a/.claude/plans/simd-fruits-integration-v1.md b/.claude/plans/simd-fruits-integration-v1.md index e86c210f..ed23c010 100644 --- a/.claude/plans/simd-fruits-integration-v1.md +++ b/.claude/plans/simd-fruits-integration-v1.md @@ -41,6 +41,16 @@ Today `F32x8` has `splat/from_slice/from_array/to_array/copy_to_slice/reduce_sum - Witness: `F64x4`/`F32x8` parity tests and masking-parity on all arms; `codegen-witness avx2` shows `vcmppd`/`vcmpps` and no `vucomisd` in `F64x8::simd_le`. +> ⊘ **REVISED 2026-11-17 (operator: "the breaking changes are absolutely unacceptable").** +> The landed F2 first changed the AVX2 `F32Mask16`/`F64Mask8` from `pub u16`/`pub u8` to +> private lane-mask pairs, and switched AVX2 `simd_min`/`simd_max`/`reduce_*` from `f32::min` +> to `vminps` NaN semantics. Both were API/behaviour breaks and are reverted: the AVX2 masks +> keep their public bitmask field exactly as on master; compares are computed per 256-bit half +> (`vcmpp*` + `vmovmskp*`, no lane loop) and `select` re-expands with `F32Mask8::from_bitmask` +> + `vblendvp*`; min/max keep `f32::min` NaN semantics via `vminp*` + unordered-compare blend. +> **Rule from this: no public field, signature or observable result of an existing API may change +> in this plan — additions only.** The open question below is closed: answer "no". + **F2. Mask types that match each tier.** *(TD1–TD2; from pulp `m64s`, wide)* - AVX-512: masks stay `__mmask8`/`__mmask16` (unchanged). - AVX2: `F64Mask4(__m256d)`, `F32Mask8(__m256)`, `F64Mask8([__m256d; 2])` and diff --git a/src/hpc/splat3d/raster.rs b/src/hpc/splat3d/raster.rs index b76ced3a..2d3bc421 100644 --- a/src/hpc/splat3d/raster.rs +++ b/src/hpc/splat3d/raster.rs @@ -48,7 +48,7 @@ pub const T_SATURATION_EPS: f32 = 1e-4; /// Combine two F32Mask16 with bitwise AND (both conditions must be true). #[inline(always)] fn mask_and(a: F32Mask16, b: F32Mask16) -> F32Mask16 { - a & b + F32Mask16(a.0 & b.0) } // ════════════════════════════════════════════════════════════════════════════ diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 0deb36e0..d8985508 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -599,11 +599,11 @@ impl F32x16 { } #[inline(always)] pub fn simd_lt(self, other: Self) -> F32Mask16 { - F32Mask16(self.0.simd_lt(other.0), self.1.simd_lt(other.1)) + F32Mask16::from_halves(self.0.simd_lt(other.0), self.1.simd_lt(other.1)) } #[inline(always)] pub fn simd_le(self, other: Self) -> F32Mask16 { - F32Mask16(self.0.simd_le(other.0), self.1.simd_le(other.1)) + F32Mask16::from_halves(self.0.simd_le(other.0), self.1.simd_le(other.1)) } #[inline(always)] pub fn simd_gt(self, other: Self) -> F32Mask16 { @@ -615,11 +615,11 @@ impl F32x16 { } #[inline(always)] pub fn simd_eq(self, other: Self) -> F32Mask16 { - F32Mask16(self.0.simd_eq(other.0), self.1.simd_eq(other.1)) + F32Mask16::from_halves(self.0.simd_eq(other.0), self.1.simd_eq(other.1)) } #[inline(always)] pub fn simd_ne(self, other: Self) -> F32Mask16 { - F32Mask16(self.0.simd_ne(other.0), self.1.simd_ne(other.1)) + F32Mask16::from_halves(self.0.simd_ne(other.0), self.1.simd_ne(other.1)) } /// Gather 16 f32 values at `base_ptr.offset(indices[i])` — the same /// signature and contract as the AVX-512 backend's `_mm512_i32gather_ps` @@ -684,7 +684,7 @@ impl F32x16 { /// ``` #[inline(always)] pub fn is_nan(self) -> F32Mask16 { - F32Mask16(self.0.is_nan(), self.1.is_nan()) + F32Mask16::from_halves(self.0.is_nan(), self.1.is_nan()) } /// Lane set iff the lane is neither NaN nor ±Inf. /// @@ -697,7 +697,7 @@ impl F32x16 { /// ``` #[inline(always)] pub fn is_finite(self) -> F32Mask16 { - F32Mask16(self.0.is_finite(), self.1.is_finite()) + F32Mask16::from_halves(self.0.is_finite(), self.1.is_finite()) } /// Lane set iff the lane is +Inf or −Inf. /// @@ -711,7 +711,7 @@ impl F32x16 { /// ``` #[inline(always)] pub fn is_infinite(self) -> F32Mask16 { - F32Mask16(self.0.is_infinite(), self.1.is_infinite()) + F32Mask16::from_halves(self.0.is_infinite(), self.1.is_infinite()) } /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and ±0). /// @@ -827,68 +827,41 @@ impl Default for F32x16 { } } -/// 16-lane `F32x16` compare result on AVX2: two `F32Mask8` lane masks (one -/// per 256-bit half), so `select` is two `vblendvps` and `to_bitmask` two -/// `vmovmskps` — no per-lane loop. (Until plan simd-fruits-integration-v1 F2 -/// this was a `u16` bitmask, the AVX-512 shape, which forced every compare -/// and select through a scalar loop on this arm.) Read and combine it through -/// `to_bitmask` / `from_bitmask` / `& | ^ !`; the fields are private. +/// F32x16 compare result on AVX2: a packed bitmask, bit `i` = lane `i` (the +/// public representation is unchanged). Compares are computed per 256-bit +/// half with `vcmpp*` and packed with `vmovmskp*` ([`F32Mask16::from_halves`]); +/// `select` re-expands each half with `F32Mask8::from_bitmask` and blends with +/// `vblendvp*`, so no step loops over lanes. #[derive(Copy, Clone, Debug)] -pub struct F32Mask16(pub(crate) F32Mask8, pub(crate) F32Mask8); +pub struct F32Mask16(pub u16); impl F32Mask16 { - /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). - /// The one representation-independent reading of a compare result: every - /// backend stores its mask differently (`__mmask16`, two lane-mask - /// vectors, `u16`, `core::simd::Mask`), so callers combine and inspect - /// masks through this rather than the tuple fields. + /// The mask as a packed bitmask, LSB-first (bit `i` = lane `i`). /// - /// # Examples - /// Bit `i` is lane `i`: with lanes 0 and 15 below the threshold the - /// `simd_lt` mask reads `0b1000_0000_0000_0001`. /// ``` /// use ndarray::simd::F32x16; - /// let mut a = [10.0f32; 16]; - /// a[0] = -1.0; - /// a[15] = -1.0; - /// let m = F32x16::from_array(a).simd_lt(F32x16::splat(0.0)); - /// assert_eq!(m.to_bitmask(), 0b1000_0000_0000_0001); + /// let m = F32x16::splat(1.0).simd_lt(F32x16::splat(2.0)); + /// assert_eq!(m.to_bitmask(), u16::MAX); /// ``` #[inline(always)] pub fn to_bitmask(self) -> u16 { - u16::from(self.0.to_bitmask()) | (u16::from(self.1.to_bitmask()) << 8) + self.0 } - /// Lane `i` is set iff bit `i` of `bits` is set. - /// - /// ``` - /// use ndarray::simd::F32Mask16; - /// let m = F32Mask16::from_bitmask(0x8001); - /// assert_eq!(m.to_bitmask(), 0x8001); - /// ``` + /// Packs the lane masks of the two 256-bit halves (crate-internal). #[inline(always)] - pub fn from_bitmask(bits: u16) -> Self { - Self(F32Mask8::from_bitmask(bits as u8), F32Mask8::from_bitmask((bits >> 8) as u8)) + pub(crate) fn from_halves(lo: F32Mask8, hi: F32Mask8) -> Self { + Self(u16::from(lo.to_bitmask()) | (u16::from(hi.to_bitmask()) << 8)) } - /// Lane-wise `if mask { true_val } else { false_val }` (two `vblendvps`). + /// Lane-wise `if mask { true_val } else { false_val }`: each half's bits + /// are expanded to a lane mask and blended with `vblendvp*`. #[inline(always)] pub fn select(self, true_val: F32x16, false_val: F32x16) -> F32x16 { - F32x16(self.0.select(true_val.0, false_val.0), self.1.select(true_val.1, false_val.1)) - } - /// `true` if at least one lane is set. - #[inline(always)] - pub fn any(self) -> bool { - self.to_bitmask() != 0 - } - /// `true` if every lane is set. - #[inline(always)] - pub fn all(self) -> bool { - self.to_bitmask() == u16::MAX - } - /// `true` if no lane is set. - #[inline(always)] - pub fn none(self) -> bool { - self.to_bitmask() == 0 + F32x16( + F32Mask8::from_bitmask(self.0 as u8).select(true_val.0, false_val.0), + F32Mask8::from_bitmask((self.0 >> 8) as u8).select(true_val.1, false_val.1), + ) } } +impl_bitmask_mask_api!(F32Mask16, u16, 16, F32x16); /// 8×f64 via 2× AVX2 F64x4 (__m256d). Same API as simd_avx512::F64x8. #[derive(Copy, Clone)] @@ -971,31 +944,31 @@ impl F64x8 { } #[inline(always)] pub fn simd_ge(self, other: Self) -> F64Mask8 { - F64Mask8(self.0.simd_ge(other.0), self.1.simd_ge(other.1)) + F64Mask8::from_halves(self.0.simd_ge(other.0), self.1.simd_ge(other.1)) } #[inline(always)] pub fn simd_le(self, other: Self) -> F64Mask8 { - F64Mask8(self.0.simd_le(other.0), self.1.simd_le(other.1)) + F64Mask8::from_halves(self.0.simd_le(other.0), self.1.simd_le(other.1)) } /// Lane-wise `==` — ordered: false if either lane is NaN; `+0.0 == -0.0` (AVX-512 `_CMP_EQ_OQ`). #[inline(always)] pub fn simd_eq(self, other: Self) -> F64Mask8 { - F64Mask8(self.0.simd_eq(other.0), self.1.simd_eq(other.1)) + F64Mask8::from_halves(self.0.simd_eq(other.0), self.1.simd_eq(other.1)) } /// Lane-wise `!=` — unordered: true if either lane is NaN (AVX-512 `_CMP_NEQ_UQ`). #[inline(always)] pub fn simd_ne(self, other: Self) -> F64Mask8 { - F64Mask8(self.0.simd_ne(other.0), self.1.simd_ne(other.1)) + F64Mask8::from_halves(self.0.simd_ne(other.0), self.1.simd_ne(other.1)) } /// Lane-wise `<` — ordered: false if either lane is NaN (AVX-512 `_CMP_LT_OS`). #[inline(always)] pub fn simd_lt(self, other: Self) -> F64Mask8 { - F64Mask8(self.0.simd_lt(other.0), self.1.simd_lt(other.1)) + F64Mask8::from_halves(self.0.simd_lt(other.0), self.1.simd_lt(other.1)) } /// Lane-wise `>` — ordered: false if either lane is NaN (AVX-512 `other.simd_lt(self)`). #[inline(always)] pub fn simd_gt(self, other: Self) -> F64Mask8 { - F64Mask8(self.0.simd_gt(other.0), self.1.simd_gt(other.1)) + F64Mask8::from_halves(self.0.simd_gt(other.0), self.1.simd_gt(other.1)) } #[inline(always)] pub fn to_bits(self) -> U64x8 { @@ -1026,7 +999,7 @@ impl F64x8 { /// ``` #[inline(always)] pub fn is_nan(self) -> F64Mask8 { - F64Mask8(self.0.is_nan(), self.1.is_nan()) + F64Mask8::from_halves(self.0.is_nan(), self.1.is_nan()) } /// Lane set iff the lane is neither NaN nor ±Inf. /// @@ -1037,7 +1010,7 @@ impl F64x8 { /// ``` #[inline(always)] pub fn is_finite(self) -> F64Mask8 { - F64Mask8(self.0.is_finite(), self.1.is_finite()) + F64Mask8::from_halves(self.0.is_finite(), self.1.is_finite()) } /// Lane set iff the lane is +Inf or −Inf. /// @@ -1048,7 +1021,7 @@ impl F64x8 { /// ``` #[inline(always)] pub fn is_infinite(self) -> F64Mask8 { - F64Mask8(self.0.is_infinite(), self.1.is_infinite()) + F64Mask8::from_halves(self.0.is_infinite(), self.1.is_infinite()) } /// Magnitude bits of `self`, sign bit of `sign` (bit-exact, also for NaN and ±0). /// @@ -1164,54 +1137,41 @@ impl Default for F64x8 { } } -/// 8-lane `F64x8` compare result on AVX2: two `F64Mask4` lane masks (one per -/// 256-bit half). Same design and API as `F32Mask16` above; the fields are -/// private. +/// F64x8 compare result on AVX2: a packed bitmask, bit `i` = lane `i` (the +/// public representation is unchanged). Compares are computed per 256-bit +/// half with `vcmpp*` and packed with `vmovmskp*` ([`F64Mask8::from_halves`]); +/// `select` re-expands each half with `F64Mask4::from_bitmask` and blends with +/// `vblendvp*`, so no step loops over lanes. #[derive(Copy, Clone, Debug)] -pub struct F64Mask8(pub(crate) F64Mask4, pub(crate) F64Mask4); +pub struct F64Mask8(pub u8); impl F64Mask8 { - /// Lane `i` of the comparison is bit `i` of the result. + /// The mask as a packed bitmask, LSB-first (bit `i` = lane `i`). /// /// ``` /// use ndarray::simd::F64x8; - /// let m = F64x8::from_array([0.0, 2.0, 0.0, 2.0, 0.0, 2.0, 0.0, 2.0]).simd_lt(F64x8::splat(1.0)); - /// assert_eq!(m.to_bitmask(), 0b0101_0101); + /// let m = F64x8::splat(1.0).simd_lt(F64x8::splat(2.0)); + /// assert_eq!(m.to_bitmask(), u8::MAX); /// ``` #[inline(always)] pub fn to_bitmask(self) -> u8 { - self.0.to_bitmask() | (self.1.to_bitmask() << 4) + self.0 } - /// Lane `i` is set iff bit `i` of `bits` is set. - /// - /// ``` - /// use ndarray::simd::F64Mask8; - /// assert_eq!(F64Mask8::from_bitmask(0x81).to_bitmask(), 0x81); - /// ``` + /// Packs the lane masks of the two 256-bit halves (crate-internal). #[inline(always)] - pub fn from_bitmask(bits: u8) -> Self { - Self(F64Mask4::from_bitmask(bits & 0xF), F64Mask4::from_bitmask(bits >> 4)) + pub(crate) fn from_halves(lo: F64Mask4, hi: F64Mask4) -> Self { + Self(lo.to_bitmask() | (hi.to_bitmask() << 4)) } - /// Lane-wise `if mask { true_val } else { false_val }` (two `vblendvpd`). + /// Lane-wise `if mask { true_val } else { false_val }`: each half's bits + /// are expanded to a lane mask and blended with `vblendvp*`. #[inline(always)] pub fn select(self, true_val: F64x8, false_val: F64x8) -> F64x8 { - F64x8(self.0.select(true_val.0, false_val.0), self.1.select(true_val.1, false_val.1)) - } - /// `true` if at least one lane is set. - #[inline(always)] - pub fn any(self) -> bool { - self.to_bitmask() != 0 - } - /// `true` if every lane is set. - #[inline(always)] - pub fn all(self) -> bool { - self.to_bitmask() == u8::MAX - } - /// `true` if no lane is set. - #[inline(always)] - pub fn none(self) -> bool { - self.to_bitmask() == 0 + F64x8( + F64Mask4::from_bitmask(self.0 & 0xF).select(true_val.0, false_val.0), + F64Mask4::from_bitmask(self.0 >> 4).select(true_val.1, false_val.1), + ) } } +impl_bitmask_mask_api!(F64Mask8, u8, 8, F64x8); // ── Integer types: array-backed, use scalar ops (no AVX2 integer 512-bit) ── @@ -5273,42 +5233,6 @@ impl U32x16 { } } -// Lane-wise logic on the AVX2 512-bit mask pairs: one op per half. -macro_rules! impl_mask_pair_logic { - ($mask:ident) => { - impl BitAnd for $mask { - type Output = Self; - #[inline(always)] - fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0, self.1 & rhs.1) - } - } - impl BitOr for $mask { - type Output = Self; - #[inline(always)] - fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0, self.1 | rhs.1) - } - } - impl BitXor for $mask { - type Output = Self; - #[inline(always)] - fn bitxor(self, rhs: Self) -> Self { - Self(self.0 ^ rhs.0, self.1 ^ rhs.1) - } - } - impl Not for $mask { - type Output = Self; - #[inline(always)] - fn not(self) -> Self { - Self(!self.0, !self.1) - } - } - }; -} -impl_mask_pair_logic!(F32Mask16); -impl_mask_pair_logic!(F64Mask8); - // GEN-TERNLOG-BEGIN (tools/gen_ternlog_bodies.py — regenerate, do not hand-edit) /// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function /// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 14ddc49d..7094f15b 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -3224,66 +3224,106 @@ impl F32x8 { } } - /// Smallest lane, folded with `vminps`: on a NaN lane the result follows - /// `vminps` (the second operand wins), as on `F32x16::reduce_min`. + /// Smallest lane, with the semantics of + /// `iter().fold(f32::INFINITY, f32::min)`: NaN lanes are ignored and an + /// all-NaN vector gives `f32::INFINITY`. Tree of NaN-ignoring `vminps`. /// /// ``` /// use ndarray::simd::F32x8; - /// let v = F32x8::from_array([3.0, -1.0, 7.0, 2.0, 9.0, 0.5, 4.0, 8.0]); - /// assert_eq!(v.reduce_min(), -1.0); + /// let v = F32x8::from_array([f32::NAN, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]); + /// assert_eq!(v.reduce_min(), 1.0); + /// assert_eq!(F32x8::splat(f32::NAN).reduce_min(), f32::INFINITY); /// ``` #[inline(always)] pub fn reduce_min(self) -> f32 { - // SAFETY: AVX/SSE3 register-only ops; this type is only compiled into - // x86 arms that have AVX (`simd.rs` routes v2 to the scalar backend). + // SAFETY: AVX/SSE3/SSE4.1 register-only ops (x86 arms with AVX only). unsafe { - let m4 = _mm_min_ps(_mm256_castps256_ps128(self.0), _mm256_extractf128_ps(self.0, 1)); - let m2 = _mm_min_ps(m4, _mm_movehl_ps(m4, m4)); - _mm_cvtss_f32(_mm_min_ss(m2, _mm_movehdup_ps(m2))) + #[inline(always)] + unsafe fn nan_min(a: __m128, b: __m128) -> __m128 { + _mm_blendv_ps(_mm_min_ps(a, b), a, _mm_cmpunord_ps(b, b)) + } + let m4 = nan_min(_mm256_castps256_ps128(self.0), _mm256_extractf128_ps(self.0, 1)); + let m2 = nan_min(m4, _mm_movehl_ps(m4, m4)); + let r = _mm_cvtss_f32(nan_min(m2, _mm_movehdup_ps(m2))); + if r.is_nan() { + f32::INFINITY + } else { + r + } } } - /// Largest lane, folded with `vmaxps` (NaN: second operand wins). + /// Largest lane, with the semantics of + /// `iter().fold(f32::NEG_INFINITY, f32::max)`: NaN lanes are ignored and an + /// all-NaN vector gives `f32::NEG_INFINITY`. Tree of NaN-ignoring `vmaxps`. /// /// ``` /// use ndarray::simd::F32x8; - /// let v = F32x8::from_array([3.0, -1.0, 7.0, 2.0, 9.0, 0.5, 4.0, 8.0]); - /// assert_eq!(v.reduce_max(), 9.0); + /// let v = F32x8::from_array([f32::NAN, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]); + /// assert_eq!(v.reduce_max(), 7.0); + /// assert_eq!(F32x8::splat(f32::NAN).reduce_max(), f32::NEG_INFINITY); /// ``` #[inline(always)] pub fn reduce_max(self) -> f32 { - // SAFETY: as `reduce_min`. + // SAFETY: AVX/SSE3/SSE4.1 register-only ops (x86 arms with AVX only). unsafe { - let m4 = _mm_max_ps(_mm256_castps256_ps128(self.0), _mm256_extractf128_ps(self.0, 1)); - let m2 = _mm_max_ps(m4, _mm_movehl_ps(m4, m4)); - _mm_cvtss_f32(_mm_max_ss(m2, _mm_movehdup_ps(m2))) + #[inline(always)] + unsafe fn nan_max(a: __m128, b: __m128) -> __m128 { + _mm_blendv_ps(_mm_max_ps(a, b), a, _mm_cmpunord_ps(b, b)) + } + let m4 = nan_max(_mm256_castps256_ps128(self.0), _mm256_extractf128_ps(self.0, 1)); + let m2 = nan_max(m4, _mm_movehl_ps(m4, m4)); + let r = _mm_cvtss_f32(nan_max(m2, _mm_movehdup_ps(m2))); + if r.is_nan() { + f32::NEG_INFINITY + } else { + r + } } } - /// Lane-wise minimum, `vminps`: if either lane is NaN the lane of `other` - /// is returned (the same rule as `F32x16::simd_min` on AVX-512). + /// Lane-wise minimum with `f32::min` NaN semantics: if exactly one lane is + /// NaN the other lane is returned (NaN only if both are). Realized as + /// `vminps` plus one unordered compare and one `vblendvps` that picks + /// `self` where `other` is NaN (`vminps` alone would return `other`). /// /// ``` /// use ndarray::simd::F32x8; - /// let m = F32x8::splat(1.0).simd_min(F32x8::splat(2.0)); - /// assert_eq!(m.to_array(), [1.0; 8]); + /// let a = F32x8::from_array([f32::NAN, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]); + /// let r = a.simd_min(F32x8::splat(1.0)).to_array(); + /// assert_eq!(r[0], 1.0); // NaN lane ignored, like f32::min /// ``` #[inline(always)] pub fn simd_min(self, other: Self) -> Self { - // SAFETY: AVX register-only op (see `reduce_min`). - Self(unsafe { _mm256_min_ps(self.0, other.0) }) + // SAFETY: AVX register-only ops (this type is only compiled into x86 + // arms with AVX). + unsafe { + let m = _mm256_min_ps(self.0, other.0); + let other_nan = _mm256_cmp_ps::<_CMP_UNORD_Q>(other.0, other.0); + Self(_mm256_blendv_ps(m, self.0, other_nan)) + } } - /// Lane-wise maximum, `vmaxps` (NaN: the lane of `other` is returned). + /// Lane-wise maximum with `f32::max` NaN semantics: if exactly one lane is + /// NaN the other lane is returned (NaN only if both are). Realized as + /// `vmaxps` plus one unordered compare and one `vblendvps` that picks + /// `self` where `other` is NaN (`vmaxps` alone would return `other`). /// /// ``` /// use ndarray::simd::F32x8; - /// assert_eq!(F32x8::splat(1.0).simd_max(F32x8::splat(2.0)).to_array(), [2.0; 8]); + /// let a = F32x8::from_array([f32::NAN, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]); + /// let r = a.simd_max(F32x8::splat(1.0)).to_array(); + /// assert_eq!(r[0], 1.0); // NaN lane ignored, like f32::max /// ``` #[inline(always)] pub fn simd_max(self, other: Self) -> Self { - // SAFETY: AVX register-only op. - Self(unsafe { _mm256_max_ps(self.0, other.0) }) + // SAFETY: AVX register-only ops (this type is only compiled into x86 + // arms with AVX). + unsafe { + let m = _mm256_max_ps(self.0, other.0); + let other_nan = _mm256_cmp_ps::<_CMP_UNORD_Q>(other.0, other.0); + Self(_mm256_blendv_ps(m, self.0, other_nan)) + } } /// `self.simd_max(lo).simd_min(hi)`. @@ -3677,59 +3717,104 @@ impl F64x4 { } } - /// Smallest lane, folded with `vminpd` (NaN: second operand wins). + /// Smallest lane, with the semantics of + /// `iter().fold(f64::INFINITY, f64::min)`: NaN lanes are ignored and an + /// all-NaN vector gives `f64::INFINITY`. Tree of NaN-ignoring `vminpd`. /// /// ``` /// use ndarray::simd::F64x4; - /// assert_eq!(F64x4::from_array([3.0, -1.0, 7.0, 2.0]).reduce_min(), -1.0); + /// let v = F64x4::from_array([f64::NAN, 1.0, 2.0, 3.0]); + /// assert_eq!(v.reduce_min(), 1.0); + /// assert_eq!(F64x4::splat(f64::NAN).reduce_min(), f64::INFINITY); /// ``` #[inline(always)] pub fn reduce_min(self) -> f64 { - // SAFETY: AVX/SSE2 register-only ops (x86 arms with AVX only). + // SAFETY: AVX/SSE4.1 register-only ops (x86 arms with AVX only). unsafe { - let m2 = _mm_min_pd(_mm256_castpd256_pd128(self.0), _mm256_extractf128_pd(self.0, 1)); - _mm_cvtsd_f64(_mm_min_sd(m2, _mm_unpackhi_pd(m2, m2))) + #[inline(always)] + unsafe fn nan_min(a: __m128d, b: __m128d) -> __m128d { + _mm_blendv_pd(_mm_min_pd(a, b), a, _mm_cmpunord_pd(b, b)) + } + let m2 = nan_min(_mm256_castpd256_pd128(self.0), _mm256_extractf128_pd(self.0, 1)); + let r = _mm_cvtsd_f64(nan_min(m2, _mm_unpackhi_pd(m2, m2))); + if r.is_nan() { + f64::INFINITY + } else { + r + } } } - /// Largest lane, folded with `vmaxpd` (NaN: second operand wins). + /// Largest lane, with the semantics of + /// `iter().fold(f64::NEG_INFINITY, f64::max)`: NaN lanes are ignored and an + /// all-NaN vector gives `f64::NEG_INFINITY`. Tree of NaN-ignoring `vmaxpd`. /// /// ``` /// use ndarray::simd::F64x4; - /// assert_eq!(F64x4::from_array([3.0, -1.0, 7.0, 2.0]).reduce_max(), 7.0); + /// let v = F64x4::from_array([f64::NAN, 1.0, 2.0, 3.0]); + /// assert_eq!(v.reduce_max(), 3.0); + /// assert_eq!(F64x4::splat(f64::NAN).reduce_max(), f64::NEG_INFINITY); /// ``` #[inline(always)] pub fn reduce_max(self) -> f64 { - // SAFETY: as `reduce_min`. + // SAFETY: AVX/SSE4.1 register-only ops (x86 arms with AVX only). unsafe { - let m2 = _mm_max_pd(_mm256_castpd256_pd128(self.0), _mm256_extractf128_pd(self.0, 1)); - _mm_cvtsd_f64(_mm_max_sd(m2, _mm_unpackhi_pd(m2, m2))) + #[inline(always)] + unsafe fn nan_max(a: __m128d, b: __m128d) -> __m128d { + _mm_blendv_pd(_mm_max_pd(a, b), a, _mm_cmpunord_pd(b, b)) + } + let m2 = nan_max(_mm256_castpd256_pd128(self.0), _mm256_extractf128_pd(self.0, 1)); + let r = _mm_cvtsd_f64(nan_max(m2, _mm_unpackhi_pd(m2, m2))); + if r.is_nan() { + f64::NEG_INFINITY + } else { + r + } } } - /// Lane-wise minimum, `vminpd`: if either lane is NaN the lane of `other` - /// is returned (the same rule as `F64x8::simd_min` on AVX-512). + /// Lane-wise minimum with `f64::min` NaN semantics: if exactly one lane is + /// NaN the other lane is returned (NaN only if both are). Realized as + /// `vminpd` plus one unordered compare and one `vblendvpd` that picks + /// `self` where `other` is NaN (`vminpd` alone would return `other`). /// /// ``` /// use ndarray::simd::F64x4; - /// assert_eq!(F64x4::splat(1.0).simd_min(F64x4::splat(2.0)).to_array(), [1.0; 4]); + /// let a = F64x4::from_array([f64::NAN, 1.0, 2.0, 3.0]); + /// let r = a.simd_min(F64x4::splat(1.0)).to_array(); + /// assert_eq!(r[0], 1.0); // NaN lane ignored, like f64::min /// ``` #[inline(always)] pub fn simd_min(self, other: Self) -> Self { - // SAFETY: AVX register-only op. - Self(unsafe { _mm256_min_pd(self.0, other.0) }) + // SAFETY: AVX register-only ops (this type is only compiled into x86 + // arms with AVX). + unsafe { + let m = _mm256_min_pd(self.0, other.0); + let other_nan = _mm256_cmp_pd::<_CMP_UNORD_Q>(other.0, other.0); + Self(_mm256_blendv_pd(m, self.0, other_nan)) + } } - /// Lane-wise maximum, `vmaxpd` (NaN: the lane of `other` is returned). + /// Lane-wise maximum with `f64::max` NaN semantics: if exactly one lane is + /// NaN the other lane is returned (NaN only if both are). Realized as + /// `vmaxpd` plus one unordered compare and one `vblendvpd` that picks + /// `self` where `other` is NaN (`vmaxpd` alone would return `other`). /// /// ``` /// use ndarray::simd::F64x4; - /// assert_eq!(F64x4::splat(1.0).simd_max(F64x4::splat(2.0)).to_array(), [2.0; 4]); + /// let a = F64x4::from_array([f64::NAN, 1.0, 2.0, 3.0]); + /// let r = a.simd_max(F64x4::splat(1.0)).to_array(); + /// assert_eq!(r[0], 1.0); // NaN lane ignored, like f64::max /// ``` #[inline(always)] pub fn simd_max(self, other: Self) -> Self { - // SAFETY: AVX register-only op. - Self(unsafe { _mm256_max_pd(self.0, other.0) }) + // SAFETY: AVX register-only ops (this type is only compiled into x86 + // arms with AVX). + unsafe { + let m = _mm256_max_pd(self.0, other.0); + let other_nan = _mm256_cmp_pd::<_CMP_UNORD_Q>(other.0, other.0); + Self(_mm256_blendv_pd(m, self.0, other_nan)) + } } /// `self.simd_max(lo).simd_min(hi)`. diff --git a/tests/simd_lane_masks.rs b/tests/simd_lane_masks.rs index cfecfe6d..eeea3c03 100644 --- a/tests/simd_lane_masks.rs +++ b/tests/simd_lane_masks.rs @@ -279,3 +279,69 @@ fn compare_select_early_exit_loop() { }); assert_eq!(count.to_array(), want); } + +/// NaN handling of min/max/reduce must stay exactly what `f32::min` / +/// `fold(INFINITY, f32::min)` give: that is the behaviour the AVX2 and +/// scalar 512-bit types had before plan F2 (it is a compatibility contract, +/// not a free choice), and the new 256-bit types follow it on every arm. +macro_rules! nan_minmax_contract { + ($vec:ident, $elem:ty, $lanes:expr) => {{ + let pool: [$elem; 6] = [<$elem>::NAN, 1.0, -2.0, <$elem>::NAN, 3.0, -0.5]; + for seed in 0..12usize { + let a: [$elem; $lanes] = core::array::from_fn(|i| pool[(seed + i) % 6]); + let b: [$elem; $lanes] = core::array::from_fn(|i| pool[(seed * 5 + i * 7 + 1) % 6]); + let (va, vb) = ($vec::from_array(a), $vec::from_array(b)); + let (mn, mx) = (va.simd_min(vb).to_array(), va.simd_max(vb).to_array()); + for i in 0..$lanes { + let (wmin, wmax) = (a[i].min(b[i]), a[i].max(b[i])); + assert!( + (mn[i].is_nan() && wmin.is_nan()) || mn[i] == wmin, + "{}::simd_min lane {i}: {:?} {:?} got {:?} want {:?}", + stringify!($vec), + a[i], + b[i], + mn[i], + wmin + ); + assert!( + (mx[i].is_nan() && wmax.is_nan()) || mx[i] == wmax, + "{}::simd_max lane {i}: {:?} {:?} got {:?} want {:?}", + stringify!($vec), + a[i], + b[i], + mx[i], + wmax + ); + } + let rmin = a.iter().copied().fold(<$elem>::INFINITY, <$elem>::min); + let rmax = a.iter().copied().fold(<$elem>::NEG_INFINITY, <$elem>::max); + assert_eq!(va.reduce_min(), rmin, "{}::reduce_min {:?}", stringify!($vec), a); + assert_eq!(va.reduce_max(), rmax, "{}::reduce_max {:?}", stringify!($vec), a); + } + // The `nightly-simd` backend has always delegated to `core::simd`'s + // `reduce_min`/`reduce_max`, which return NaN for an all-NaN vector; + // that pre-existing behaviour is kept unchanged, so only the stable + // arms are held to the fold's ±INFINITY here. + if !cfg!(feature = "nightly-simd") { + let all_nan = $vec::splat(<$elem>::NAN); + assert_eq!(all_nan.reduce_min(), <$elem>::INFINITY, "{} all-NaN reduce_min", stringify!($vec)); + assert_eq!(all_nan.reduce_max(), <$elem>::NEG_INFINITY, "{} all-NaN reduce_max", stringify!($vec)); + } + }}; +} + +#[test] +fn nan_min_max_reduce_follow_f32_min_on_256_bit_types() { + nan_minmax_contract!(F32x8, f32, 8); + nan_minmax_contract!(F64x4, f64, 4); +} + +/// The 512-bit types on the arms whose master implementation was the +/// `f32::min` loop: AVX2, AVX-without-AVX2 and the scalar arm (x86-64-v2 and +/// non-x86 targets other than aarch64/wasm, whose native types are unchanged). +#[cfg(not(any(target_feature = "avx512f", target_arch = "aarch64", target_arch = "wasm32")))] +#[test] +fn nan_min_max_reduce_unchanged_on_avx2_and_scalar_512_bit_types() { + nan_minmax_contract!(F32x16, f32, 16); + nan_minmax_contract!(F64x8, f64, 8); +} From e830466f7937851634932c6b2166f560ca7913de Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 17:10:06 +0000 Subject: [PATCH 11/13] =?UTF-8?q?simd:=20remove=20recip=5Fapprox=20?= =?UTF-8?q?=E2=80=94=20facade=20methods=20must=20be=20IEEE=20bit-exact=20o?= =?UTF-8?q?n=20every=20CPU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator rule: every method is IEEE bit-exact and identical on every arm. recip_approx returned arm-dependent bits (1/3.0 gave 3eaaaa80 on AVX-512 vrcp14, 3eaaa000 on AVX vrcpps, 3eaaaaab on the exact arms), so it is removed from simd_avx512, simd_avx2, simd_scalar, simd_neon, simd_wasm and the nightly backend, with its tolerance test. recip (exact IEEE 1/x) stays. Plan F5 and the blackboard record the removal and the rule; plan F12 (fast/exact variants) is rejected for the same reason. Verified: the five parity test files pass on native(v4), v3, avx, v2, aarch64 (qemu) and nightly-simd; masking-parity wasm and neon-qemu PASS; clippy -D warnings, fmt and facet-stride guard clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 10 +++ .claude/plans/simd-fruits-integration-v1.md | 8 ++- src/simd_avx2.rs | 24 ------- src/simd_avx512.rs | 69 --------------------- src/simd_neon.rs | 22 ------- src/simd_nightly/f32_types.rs | 10 --- src/simd_nightly/f64_types.rs | 10 --- src/simd_scalar.rs | 5 -- src/simd_wasm.rs | 24 ------- tests/simd_float_classify.rs | 52 ++-------------- 10 files changed, 19 insertions(+), 215 deletions(-) diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 4664b0a5..e4d280e8 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,13 @@ +## 2026-11-17 — RULE (operator): every facade method is IEEE bit-exact, identical on every CPU; no round trips + +"Every method MUST be ieee bit exact. Any round trip must be prevented by all means. Mantissa bit must be +preserved." Applied: `recip_approx` removed from all arms (it returned arm-dependent bits). Audit of the +rest of this branch's additions: compares/masks, copysign, recip, sqrt, round, floor, mul_add (FMA3, or the +exact scalar FMA on AVX-without-FMA), mul_widen_even/mul_high (integer exact), load/store_partial (bit +copies) — all bit-exact and arm-identical by the parity tests. Pre-existing (master, untouched): AVX-512 +F32x16/F64x8 simd_min/max use raw vminps (NaN → second operand) while AVX2/scalar use f32::min; nightly +reduce_* all-NaN → NaN. Plan F12 (fast/exact split) rejected. + ## 2026-11-17 — CORRECTION: simd-fruits breaking changes reverted (operator: "absolutely unacceptable") Two breaks had shipped on PR #435 and are removed: diff --git a/.claude/plans/simd-fruits-integration-v1.md b/.claude/plans/simd-fruits-integration-v1.md index ed23c010..e6e5a951 100644 --- a/.claude/plans/simd-fruits-integration-v1.md +++ b/.claude/plans/simd-fruits-integration-v1.md @@ -94,9 +94,11 @@ Today `F32x8` has `splat/from_slice/from_array/to_array/copy_to_slice/reduce_sum `recip`, `recip_sqrt`. - AVX-512: `copysign` is one `vpternlogq` bit-select with a sign-bit mask operand. The immediate depends on operand order; derive it from the truth table and pin it in a test. - AVX2: `copysign` is an `and`/`andnot`/`or` sequence. -- Semantics: there are two flavours of reciprocal. `recip_approx` uses `vrcp14p[sd]` - (AVX-512, 2⁻¹⁴ relative error) or `vrcpps` (AVX2, f32 only, 1.5·2⁻¹²). `recip` is exact - (`1/x`, i.e. `vdivp[sd]`). Both precisions are written in the doc comments. +- ⊘ **REMOVED 2026-11-17 (operator: "Every method MUST be ieee bit exact").** `recip_approx` + (vrcp14 / vrcpps / exact by arm) returned different bits on different CPUs (1/3.0: + `3eaaaa80` v4, `3eaaa000` v3, `3eaaaaab` scalar) — an estimate has no place in the facade. + Only `recip` (exact IEEE `1/x`, identical on every arm) remains. The wide/pulp "fast vs exact" + split (F12) is rejected for the same reason. - Witness: parity tests over NaN, ±Inf, ±0 and denormals; scalar is the reference. **F6. Widening multiply and high-half multiply.** *(TD9; from wide `widening_mul`, `mul_keep_high`)* diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index d8985508..bf49b961 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -733,18 +733,6 @@ impl F32x16 { pub fn recip(self) -> Self { Self(self.0.recip(), self.1.recip()) } - /// Approximation of `1/self`. On AVX2 this is the 256-bit half's bound: - /// ≤ 1.5·2^-12 relative error for f32 (`vrcpps`), exact for f64. - /// - /// ``` - /// use ndarray::simd::F32x16; - /// let r = F32x16::splat(4.0).recip_approx().to_array(); - /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 2f32.powi(-11))); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - Self(self.0.recip_approx(), self.1.recip_approx()) - } } impl Add for F32x16 { @@ -1043,18 +1031,6 @@ impl F64x8 { pub fn recip(self) -> Self { Self(self.0.recip(), self.1.recip()) } - /// Approximation of `1/self`. On AVX2 this is the 256-bit half's bound: - /// ≤ 1.5·2^-12 relative error for f32 (`vrcpps`), exact for f64. - /// - /// ``` - /// use ndarray::simd::F64x8; - /// let r = F64x8::splat(4.0).recip_approx().to_array(); - /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 2f64.powi(-11))); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - Self(self.0.recip_approx(), self.1.recip_approx()) - } } impl Add for F64x8 { diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 7094f15b..0ec2dbd3 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -317,19 +317,6 @@ impl F32x16 { pub fn recip(self) -> Self { F32x16::splat(1.0) / self } - - /// Approximate reciprocal `1 / self`. `_mm512_rcp14_ps`: relative error <= 2^-14. - /// - /// ``` - /// use ndarray::simd::F32x16; - /// let r = F32x16::splat(4.0).recip_approx().to_array(); - /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 0.001)); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - // SAFETY: AVX-512F register-only op (vrcp14ps). - Self(unsafe { _mm512_rcp14_ps(self.0) }) - } } impl_bin_op!(F32x16, Add, add, _mm512_add_ps); @@ -625,19 +612,6 @@ impl F64x8 { pub fn recip(self) -> Self { F64x8::splat(1.0) / self } - - /// Approximate reciprocal `1 / self`. `_mm512_rcp14_pd`: relative error <= 2^-14. - /// - /// ``` - /// use ndarray::simd::F64x8; - /// let r = F64x8::splat(4.0).recip_approx().to_array(); - /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 0.001)); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - // SAFETY: AVX-512F register-only op (vrcp14pd). - Self(unsafe { _mm512_rcp14_pd(self.0) }) - } } impl_bin_op!(F64x8, Add, add, _mm512_add_pd); @@ -3545,28 +3519,6 @@ impl F32x8 { pub fn recip(self) -> Self { F32x8::splat(1.0) / self } - - /// Approximate reciprocal `1 / self`. With `avx512vl` enabled: `_mm256_rcp14_ps`, relative error <= 2^-14. - /// Otherwise: `_mm256_rcp_ps`, relative error <= 1.5 * 2^-12. - /// - /// ``` - /// use ndarray::simd::F32x8; - /// let r = F32x8::splat(4.0).recip_approx().to_array(); - /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 0.001)); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - #[cfg(target_feature = "avx512vl")] - { - // SAFETY: avx512vl is enabled at compile time (vrcp14ps ymm). - Self(unsafe { _mm256_rcp14_ps(self.0) }) - } - #[cfg(not(target_feature = "avx512vl"))] - { - // SAFETY: AVX register-only op (vrcpps). - Self(unsafe { _mm256_rcp_ps(self.0) }) - } - } } impl Add for F32x8 { @@ -4033,27 +3985,6 @@ impl F64x4 { pub fn recip(self) -> Self { F64x4::splat(1.0) / self } - - /// Approximate reciprocal `1 / self`. With `avx512vl` enabled: `_mm256_rcp14_pd`, relative error <= 2^-14. - /// Otherwise AVX has no f64 reciprocal estimate, so this is the exact `recip()`. - /// - /// ``` - /// use ndarray::simd::F64x4; - /// let r = F64x4::splat(4.0).recip_approx().to_array(); - /// assert!(r.iter().all(|&x| (x - 0.25).abs() <= 0.25 * 0.001)); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - #[cfg(target_feature = "avx512vl")] - { - // SAFETY: avx512vl is enabled at compile time (vrcp14pd ymm). - Self(unsafe { _mm256_rcp14_pd(self.0) }) - } - #[cfg(not(target_feature = "avx512vl"))] - { - self.recip() - } - } } impl Add for F64x4 { diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 6153cb1f..59fa5daa 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -799,17 +799,6 @@ pub mod aarch64_simd { pub fn recip(self) -> Self { Self::splat(1.0) / self } - - /// Approximate reciprocal; exact on NEON (no estimate is used; `vrecpe` alone is ~8 bits). - /// - /// ``` - /// use ndarray::simd::F32x16; - /// assert_eq!(F32x16::splat(4.0).recip_approx().to_array(), [0.25f32; 16]); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } } impl Add for F32x16 { @@ -1272,17 +1261,6 @@ pub mod aarch64_simd { pub fn recip(self) -> Self { Self::splat(1.0) / self } - - /// Approximate reciprocal; exact on NEON (no estimate is used; `vrecpe` alone is ~8 bits). - /// - /// ``` - /// use ndarray::simd::F64x8; - /// assert_eq!(F64x8::splat(4.0).recip_approx().to_array(), [0.25f64; 8]); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } } impl Add for F64x8 { diff --git a/src/simd_nightly/f32_types.rs b/src/simd_nightly/f32_types.rs index 4e2da6ef..2281f13a 100644 --- a/src/simd_nightly/f32_types.rs +++ b/src/simd_nightly/f32_types.rs @@ -474,11 +474,6 @@ impl F32x16 { pub fn recip(self) -> Self { Self(self.0.recip()) } - /// Approximate `1/self`; exact on the portable backend. - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } } // F5 (plan simd-fruits-integration-v1): classification, copysign, reciprocal. @@ -508,9 +503,4 @@ impl F32x8 { pub fn recip(self) -> Self { Self(self.0.recip()) } - /// Approximate `1/self`; exact on the portable backend. - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } } diff --git a/src/simd_nightly/f64_types.rs b/src/simd_nightly/f64_types.rs index 6052d676..dcd58fca 100644 --- a/src/simd_nightly/f64_types.rs +++ b/src/simd_nightly/f64_types.rs @@ -380,11 +380,6 @@ impl F64x8 { pub fn recip(self) -> Self { Self(self.0.recip()) } - /// Approximate `1/self`; exact on the portable backend. - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } } // F5 (plan simd-fruits-integration-v1): classification, copysign, reciprocal. @@ -414,9 +409,4 @@ impl F64x4 { pub fn recip(self) -> Self { Self(self.0.recip()) } - /// Approximate `1/self`; exact on the portable backend. - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } } diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 6c6d6280..8c3cc32f 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -205,11 +205,6 @@ macro_rules! impl_float_type { Self(core::array::from_fn(|i| 1.0 / self.0[i])) } - #[doc = concat!("Approximate `1/self`. Exact on the scalar arm (relative error 0, the same as `recip`).\n\n```\nuse ndarray::simd::", stringify!($name), ";\nlet r = ", stringify!($name), "::splat(4.0).recip_approx();\nassert_eq!(r.to_array(), [0.25; ", stringify!($lanes), "]);\n```")] - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } #[inline(always)] pub fn simd_lt(self, other: Self) -> $mask { diff --git a/src/simd_wasm.rs b/src/simd_wasm.rs index feb3a575..3541b835 100644 --- a/src/simd_wasm.rs +++ b/src/simd_wasm.rs @@ -390,18 +390,6 @@ pub mod wasm32_simd { pub fn recip(self) -> Self { Self::splat(1.0) / self } - - /// Reciprocal "approximation": exact on wasm (simd128 has no reciprocal - /// estimate), i.e. identical to [`Self::recip`]. - /// - /// ``` - /// use ndarray::simd::F32x16; - /// assert_eq!(F32x16::splat(4.0).recip_approx().to_array(), [0.25; 16]); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } } impl Add for F32x16 { @@ -838,18 +826,6 @@ pub mod wasm32_simd { pub fn recip(self) -> Self { Self::splat(1.0) / self } - - /// Reciprocal "approximation": exact on wasm (simd128 has no reciprocal - /// estimate), i.e. identical to [`Self::recip`]. - /// - /// ``` - /// use ndarray::simd::F64x8; - /// assert_eq!(F64x8::splat(4.0).recip_approx().to_array(), [0.25; 8]); - /// ``` - #[inline(always)] - pub fn recip_approx(self) -> Self { - self.recip() - } } impl Add for F64x8 { diff --git a/tests/simd_float_classify.rs b/tests/simd_float_classify.rs index a6dc0e71..4a9c3875 100644 --- a/tests/simd_float_classify.rs +++ b/tests/simd_float_classify.rs @@ -1,13 +1,12 @@ //! Parity for plan `simd-fruits-integration-v1` F5: `is_nan`, `is_finite`, -//! `is_infinite`, `copysign`, `recip` and `recip_approx` on `F32x16`, +//! `is_infinite`, `copysign` and `recip` on `F32x16`, //! `F64x8`, `F32x8` and `F64x4`. //! //! Expectations are computed lane by lane with the `f32`/`f64` methods of //! the same name, so this file grades whichever arm the build compiled. -//! `copysign` and `recip` must be bit-exact; `recip_approx` must be within -//! a relative error of 2^-11 on normal inputs whose reciprocal is normal -//! (the documented contract; AVX-512 `vrcp14` is 2^-14, AVX `vrcpps` -//! 1.5·2^-12, the other arms are exact). +//! Every result must be IEEE bit-exact and identical on every arm (the +//! polyfill contract): the masks bit for bit, `copysign` and `recip` by +//! `to_bits`. #![cfg(feature = "std")] use ndarray::simd::{F32x16, F32x8, F64x4, F64x8}; @@ -85,7 +84,6 @@ macro_rules! classify_parity { ); let cs = va.copysign(vs).to_array(); let rc = va.recip().to_array(); - let ra = va.recip_approx().to_array(); for i in 0..$lanes { assert_eq!( cs[i].to_bits(), @@ -102,17 +100,6 @@ macro_rules! classify_parity { rc[i], want ); - if a[i].is_normal() && want.is_normal() { - let rel = ((ra[i] - want) / want).abs(); - assert!( - rel <= 1.0 / 2048.0, - "{t}::recip_approx lane {i}: 1/{:?} got {:?} want {:?} rel {:e}", - a[i], - ra[i], - want, - rel - ); - } } } }; @@ -125,34 +112,3 @@ fn classification_copysign_recip_match_scalar() { classify_parity!(F32x8, f32, 8, lanes_f32); classify_parity!(F64x4, f64, 4, lanes_f64); } - -#[test] -fn recip_approx_dense_sweep_f32() { - // 4,096 normal f32 values spread over many binades and both signs. - let mut worst = 0.0f64; - for k in 0..256u32 { - let base: [f32; 16] = core::array::from_fn(|i| { - let e = (k % 60) as i32 - 30; - let m = 1.0 + (i as f32 + 16.0 * (k / 60) as f32) / 128.0; - let x = m * 2f32.powi(e); - if (k + i as u32).is_multiple_of(2) { - x - } else { - -x - } - }); - let got = F32x16::from_array(base).recip_approx().to_array(); - let got8 = F32x8::from_array(base[..8].try_into().unwrap()) - .recip_approx() - .to_array(); - for i in 0..16 { - let want = 1.0 / f64::from(base[i]); - worst = worst.max(((f64::from(got[i]) - want) / want).abs()); - if i < 8 { - worst = worst.max(((f64::from(got8[i]) - want) / want).abs()); - } - } - } - assert!(worst <= 1.0 / 2048.0, "recip_approx worst relative error {worst:e} > 2^-11"); - println!("recip_approx f32 worst relative error: {worst:e}"); -} From c62bc2c343683df4d4a01db6413adcf730ed2bfc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 17:14:03 +0000 Subject: [PATCH 12/13] docs: record what the BF16x16 / add_mul / as_chunks reference code teaches; withdraw plan F8 The operator pointed at these as the pattern to learn from, not as code to change. Blackboard entry states the pattern (widen once, compute exactly in the wide type, one certified rounding at the edge, fallbacks bit-identical to hardware, existing behaviour never altered). Plan F8 ("bf16 debt") is marked withdrawn. No source file is touched. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 19 +++++++++++++++++++ .claude/plans/simd-fruits-integration-v1.md | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/.claude/blackboard.md b/.claude/blackboard.md index e4d280e8..905b4872 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,22 @@ +## 2026-11-17 — LEARNED from the reference code (operator pointed at BF16x16, F16/F32 + add_mul, as_chunks) + +Read-only study; nothing in these files was changed. I had mislabelled them as "debt" (plan F8, now withdrawn). +- `BF16x16` (simd_avx512.rs): bf16 is a STORAGE format. Widening bf16→f32 is lossless (a shift); the one + lossy step, f32→bf16, has exactly one definition (Intel SDM RNE) and the AVX-512-F software path is + certified byte-identical to `_mm512_cvtneps_pbh` on every input, so every CPU gives the same bits. The + older truncating `f32_to_bf16_scalar` was LEFT IN PLACE for its callers; the new rule got a new name. +- `add_mul_f32/f64` (simd_ops.rs, simd_runtime/add_mul.rs): compute stays in the wide format; each lane is + ONE IEEE rounding (hardware FMA, or the exact software FMA of simd_fma where there is none), and the tail + uses the same single-rounded op, so body and tail and every arm agree bit for bit. No narrowing between ops. +- `as_chunks::()` (simd_soa.rs, hpc/renderer.rs, hpc/statistics.rs): a chunk is a `&[T; N]` — a pointer + cast, no copy, no index arithmetic — handed to `from_array(*chunk)`; the remainder is a separate slice. +Pattern: widen once at the edge, compute exactly in the wide type, narrow once at the other edge with one +certified rounding; a software fallback is only acceptable if it is bit-identical to the hardware form and a +test proves it; existing behaviour is never altered — a new semantics gets a new name. +Checked this branch's additions against it: all are bit-exact and arm-identical by test. One thing to know, +not changed: on AVX2 a compare result is packed to the public bitmask and re-expanded by `select` (lossless, +but a pack/expand per use) — that is the cost of keeping `F64Mask8(pub u8)` unchanged. + ## 2026-11-17 — RULE (operator): every facade method is IEEE bit-exact, identical on every CPU; no round trips "Every method MUST be ieee bit exact. Any round trip must be prevented by all means. Mantissa bit must be diff --git a/.claude/plans/simd-fruits-integration-v1.md b/.claude/plans/simd-fruits-integration-v1.md index e6e5a951..a4ec93a1 100644 --- a/.claude/plans/simd-fruits-integration-v1.md +++ b/.claude/plans/simd-fruits-integration-v1.md @@ -125,6 +125,10 @@ Today `F32x8` has `splat/from_slice/from_array/to_array/copy_to_slice/reduce_sum ### P2 — the AVX-512-only features this host has and no competitor uses well +> ⊘ **WITHDRAWN 2026-11-17 (operator: "I didn't ask for changing BF16, I told you where to learn +> from").** F8 is not to be executed. `BF16x16`, `F16`/`F32` + `add_mul` and the `as_chunks` call +> sites are the REFERENCE for how this facade is built, not debt (see blackboard, same date). + **F8. Fix the bf16 debt.** *(§ 2b)* - Remove the name collision: `simd_half::BF16x16([u16;16])` becomes either an alias or a re-export of the hardware type on `avx512bf16`, and a scalar type elsewhere. One name From c05a8729d84556ae42bd9e51c762f4ad2d443330 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 17:34:35 +0000 Subject: [PATCH 13/13] ci: IEEE 754 bit-exact gate on every realization row (parity group 26) Adds check_ieee_bit_exact to crates/simd-masking-parity, the program every row of the SIMD realization matrix already runs. For F32x16, F64x8, F32x8 and F64x4 it compares + - * /, sqrt, single-rounded mul_add, round (ties to even), floor, abs, copysign, recip, the six compares, is_nan / is_finite / is_infinite and simd_min / simd_max lane by lane with the scalar IEEE operation (specials plus random bit patterns), and the slice add_mul_f32 / add_mul_f64 at ten remainder lengths. Every result is folded into an FNV-1a digest that must equal IEEE_GOLDEN, one constant for all realizations, so a green row proves the same bits as every other row. The native binary prints the digest. Not pinned (documented in the group comment): NaN sign/payload, min/max of +0 against -0, min/max with a NaN operand on the 512-bit types (pre-existing AVX-512 vminps behaviour, untouched), and reductions. Verified: digest 0xee52c9e7819d32c3 printed on native (v4), pinned v3, avx-qemu, v2-qemu, neon-qemu and nightly; wasm, wasm-scalar and wasm-relaxed return rc=0 against the same constant. Disable run: making F32x16::recip an estimate (vrcp14) fails with code 0xC00A. No library source is changed; the workflow file only gains a comment. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01XAqKMQaYXab73XQdjqZfAW --- .claude/blackboard.md | 16 ++ .github/workflows/simd-matrix.yaml | 9 ++ crates/simd-masking-parity/src/lib.rs | 215 ++++++++++++++++++++++++- crates/simd-masking-parity/src/main.rs | 4 + 4 files changed, 243 insertions(+), 1 deletion(-) diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 905b4872..71e1017e 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,19 @@ +## 2026-11-17 — IEEE bit-exact gate in CI: parity group 26 `check_ieee_bit_exact` + `IEEE_GOLDEN` + +`crates/simd-masking-parity` gains one check group, so every existing realization row of +`.github/workflows/simd-matrix.yaml` runs it (no new job). Per lane vs the scalar IEEE op: + - * /, sqrt, +single-rounded mul_add, round (ties-even), floor, abs, copysign, recip, six compares, is_nan/is_finite/ +is_infinite, simd_min/max, on F32x16, F64x8, F32x8, F64x4 (600 rounds each: specials + random bit patterns +incl. subnormals/NaN/Inf), plus slice add_mul_f32/f64 at 10 remainder lengths. All results fold into an +FNV-1a digest that must equal `IEEE_GOLDEN = 0xEE52_C9E7_819D_32C3`. +Measured: identical digest printed on native v4, pinned v3, avx-qemu, v2-qemu, neon-qemu and nightly; +wasm, wasm-scalar, wasm-relaxed return rc=0 (they compare against the same constant). Disable run: F32x16:: +recip → vrcp14 gives code 0xC00A (exit 1). Not pinned, by IEEE or by pre-existing behaviour (documented in +the group's comment): NaN sign/payload (canonicalized); min/max of +0 vs -0; min/max with a NaN operand on +the 512-bit types (AVX-512 = raw vminps, others ignore NaN — master behaviour, untouched); reductions +(summation order). The parity crate has 4 pre-existing clippy hits (same count before this change; not CI- +gated); none in the new section. + ## 2026-11-17 — LEARNED from the reference code (operator pointed at BF16x16, F16/F32 + add_mul, as_chunks) Read-only study; nothing in these files was changed. I had mislabelled them as "debt" (plan F8, now withdrawn). diff --git a/.github/workflows/simd-matrix.yaml b/.github/workflows/simd-matrix.yaml index 0a485d66..268bd6d0 100644 --- a/.github/workflows/simd-matrix.yaml +++ b/.github/workflows/simd-matrix.yaml @@ -52,6 +52,15 @@ permissions: # backend selected the instruction it is REQUIRED to select at opt-level 3. # Neither replaces the other. # +# IEEE 754 bit-exactness rides on every one of these rows: the parity +# program's `check_ieee_bit_exact` group compares each float method of the +# facade (`+ - * /`, sqrt, single-rounded FMA, round-half-even, floor, abs, +# copysign, 1/x, compares, classification, slice `add_mul`) lane by lane with +# the scalar IEEE operation AND folds every result into one digest that must +# equal `IEEE_GOLDEN` — the same constant on every realization. A row that +# passes therefore proves "same bits as every other row", not just "same as +# its own scalar". The native rows print `ieee-digest=…` in the log. +# # The scalar realization has no host of its own: it is what `simd.rs` selects # on wasm32 WITHOUT `+simd128`, so the `scalar` row is a wasm32 build with the # feature off, run under node. `nightly` is the `core::simd` realization diff --git a/crates/simd-masking-parity/src/lib.rs b/crates/simd-masking-parity/src/lib.rs index 37caac8b..14bcf43a 100644 --- a/crates/simd-masking-parity/src/lib.rs +++ b/crates/simd-masking-parity/src/lib.rs @@ -44,6 +44,7 @@ //! `selfcheck()` (the wasm cdylib export, driven by `run.mjs`) both call //! [`run`]. +use ndarray::simd::{add_mul_f32, add_mul_f64, mul_add_f32, mul_add_f64, F32x8, F64x4}; use ndarray::simd::{ blend_i32, count_gt_i32_at, count_set_at, eq_f32_to_mask, eq_f64_to_mask, eq_i32_to_mask, eq_i32_to_mask_under, eq_u32_strided_to_mask, eq_u32_to_mask, eq_u32_to_mask_under, eq_u32_via_to_mask, eq_u64_to_mask, eq_u8_to_mask, @@ -66,7 +67,7 @@ use ndarray::simd::{ }; /// Number of check groups [`run`] executes (for the log line only). -pub const CHECKS: usize = 25; +pub const CHECKS: usize = 26; /// The wasm export: identical to [`run`], `extern "C"` so `run.mjs` can call it. #[no_mangle] @@ -83,6 +84,7 @@ pub fn run() -> u32 { check_f64x8_compare, check_i8x16_u8x16_lanes, check_care_match16, check_narrow_ladder, check_mul_add_single_rounding, check_fixedpoint_requant, check_nan_presence_bijection, check_sql_null_predicates, check_lut9, check_lut_u8, check_position_folds, check_count_set_at, + check_ieee_bit_exact, ]; for g in groups { if let Err(code) = g() { @@ -3302,3 +3304,214 @@ fn check_sql_null_predicates() -> Result<(), u32> { } Ok(()) } + +// ============================================================================ +// IEEE 754 bit-exactness: the polyfill contract for the float facade +// ============================================================================ +// +// Every float method of the facade must return the SAME BITS on every CPU: +// each operation is one correctly rounded IEEE 754 operation, never an +// estimate, never a narrow-compute-widen round trip. This group checks that +// two ways, on whichever realization the build compiled: +// +// 1. lane by lane against the scalar IEEE operation (`+ - * /`, `sqrt`, +// single-rounded FMA, round-half-even, `floor`, `abs`, `copysign`, +// `1/x`, the six compares and the three classifications), and +// 2. by folding every result into one digest that must equal +// [`IEEE_GOLDEN`] — the same constant on AVX-512, AVX2, AVX, x86-64-v2, +// NEON, wasm simd128, wasm scalar and the `core::simd` backend. A +// realization whose scalar reference itself were off would still be +// caught by the constant. +// +// What IEEE 754 leaves open, and this group therefore does not pin: +// * NaN sign and payload: any NaN counts as the one canonical NaN. +// * `simd_min` / `simd_max` of `+0` and `-0`: the lane is skipped. +// * `simd_min` / `simd_max` with a NaN operand on the 512-bit types: the +// AVX-512 realization is the raw `vminps`/`vmaxps` (second operand), the +// others ignore the NaN. Pre-existing and left untouched; skipped here. +// The 256-bit types are checked with NaN operands (NaN ignored, as +// `f32::min`). +// Reductions (`reduce_sum`, …) are outside this group: their summation order +// is a property of the realization. + +/// Digest every realization must reproduce (see [`ieee_digest`]). +pub const IEEE_GOLDEN: u64 = 0xEE52_C9E7_819D_32C3; + +/// FNV-1a over the canonical result bits of every operation in the group. +struct Fnv(u64); +impl Fnv { + fn word(&mut self, x: u64) { + for b in x.to_le_bytes() { + self.0 ^= u64::from(b); + self.0 = self.0.wrapping_mul(0x0000_0100_0000_01B3); + } + } +} + +fn canon32(x: f32) -> u64 { + if x.is_nan() { + 0x7FC0_0000 + } else { + u64::from(x.to_bits()) + } +} +fn canon64(x: f64) -> u64 { + if x.is_nan() { + 0x7FF8_0000_0000_0000 + } else { + x.to_bits() + } +} + +const IEEE_SPECIALS: [f64; 20] = [ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + 1.5, + 2.5, + -2.5, + 3.0, + 1e-3, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + 0.1, + -7.75, + 16777217.0, + 1e30, + -1e-30, + 4.0, + 1e-45, +]; + +macro_rules! ieee_group { + ($vec:ident, $elem:ident, $lanes:expr, $canon:ident, $fma:ident, $from_bits:expr, $wide_nan_minmax:expr, + $rng:ident, $h:ident, $code:expr) => {{ + for round in 0..600usize { + let gen = |r: &mut SplitMix64, salt: usize| -> [$elem; $lanes] { + core::array::from_fn(|i| { + if round % 3 == 0 { + IEEE_SPECIALS[(round + i * 7 + salt * 5) % IEEE_SPECIALS.len()] as $elem + } else { + $from_bits(r.next()) + } + }) + }; + let (a, b, c) = (gen(&mut $rng, 0), gen(&mut $rng, 1), gen(&mut $rng, 2)); + let (va, vb, vc) = ($vec::from_array(a), $vec::from_array(b), $vec::from_array(c)); + // (result lanes, scalar IEEE reference lanes) + let ops: [([$elem; $lanes], [$elem; $lanes]); 11] = [ + ((va + vb).to_array(), core::array::from_fn(|i| a[i] + b[i])), + ((va - vb).to_array(), core::array::from_fn(|i| a[i] - b[i])), + ((va * vb).to_array(), core::array::from_fn(|i| a[i] * b[i])), + ((va / vb).to_array(), core::array::from_fn(|i| a[i] / b[i])), + (va.sqrt().to_array(), a.map($elem::sqrt)), + (va.mul_add(vb, vc).to_array(), core::array::from_fn(|i| $fma(a[i], b[i], c[i]))), + (va.round().to_array(), a.map($elem::round_ties_even)), + (va.floor().to_array(), a.map($elem::floor)), + (va.abs().to_array(), a.map($elem::abs)), + (va.copysign(vb).to_array(), core::array::from_fn(|i| a[i].copysign(b[i]))), + (va.recip().to_array(), a.map(|x| 1.0 / x)), + ]; + for (k, (got, want)) in ops.iter().enumerate() { + for i in 0..$lanes { + // abs and copysign are bit operations: exact even on NaN. + let (g, w) = if k == 8 || k == 9 { + (u64::from(got[i].to_bits()), u64::from(want[i].to_bits())) + } else { + ($canon(got[i]), $canon(want[i])) + }; + if g != w { + return Err($code + k as u32); + } + $h.word(g); + } + } + let bit = |f: &dyn Fn(usize) -> bool| -> u64 { (0..$lanes).filter(|&i| f(i)).fold(0, |m, i| m | (1 << i)) }; + let masks: [(u64, u64); 9] = [ + (va.simd_eq(vb).to_bitmask().into(), bit(&|i| a[i] == b[i])), + (va.simd_ne(vb).to_bitmask().into(), bit(&|i| a[i] != b[i])), + (va.simd_lt(vb).to_bitmask().into(), bit(&|i| a[i] < b[i])), + (va.simd_le(vb).to_bitmask().into(), bit(&|i| a[i] <= b[i])), + (va.simd_gt(vb).to_bitmask().into(), bit(&|i| a[i] > b[i])), + (va.simd_ge(vb).to_bitmask().into(), bit(&|i| a[i] >= b[i])), + (va.is_nan().to_bitmask().into(), bit(&|i| a[i].is_nan())), + (va.is_finite().to_bitmask().into(), bit(&|i| a[i].is_finite())), + (va.is_infinite().to_bitmask().into(), bit(&|i| a[i].is_infinite())), + ]; + for (k, (g, w)) in masks.iter().enumerate() { + if g != w { + return Err($code + 0x20 + k as u32); + } + $h.word(*g); + } + let (mn, mx) = (va.simd_min(vb).to_array(), va.simd_max(vb).to_array()); + for i in 0..$lanes { + let mixed_zero = a[i] == 0.0 && b[i] == 0.0 && a[i].to_bits() != b[i].to_bits(); + let nan = a[i].is_nan() || b[i].is_nan(); + if mixed_zero || (nan && !$wide_nan_minmax) { + continue; + } + if $canon(mn[i]) != $canon(a[i].min(b[i])) || $canon(mx[i]) != $canon(a[i].max(b[i])) { + return Err($code + 0x40); + } + $h.word($canon(mn[i])); + $h.word($canon(mx[i])); + } + } + }}; +} + +/// Runs the IEEE group and returns its digest, or the code of the first lane +/// that differs from the scalar IEEE operation. +pub fn ieee_digest() -> Result { + let mut rng = SplitMix64(0x1EEE_0754_0000_0001); + let mut h = Fnv(0xCBF2_9CE4_8422_2325); + let f32b = |x: u64| f32::from_bits(x as u32); + let f64b = |x: u64| f64::from_bits(x); + ieee_group!(F32x16, f32, 16, canon32, mul_add_f32, f32b, false, rng, h, 0xC000); + ieee_group!(F64x8, f64, 8, canon64, mul_add_f64, f64b, false, rng, h, 0xC100); + ieee_group!(F32x8, f32, 8, canon32, mul_add_f32, f32b, true, rng, h, 0xC200); + ieee_group!(F64x4, f64, 4, canon64, mul_add_f64, f64b, true, rng, h, 0xC300); + + // Slice FMA `acc[i] += a[i] * b[i]`: the 16/8-lane body and the scalar + // tail must both be the single-rounded FMA, at every remainder length. + for n in [0usize, 1, 7, 8, 15, 16, 17, 31, 33, 100] { + let a32: Vec = (0..n).map(|_| f32b(rng.next())).collect(); + let b32: Vec = (0..n).map(|_| f32b(rng.next())).collect(); + let mut acc32: Vec = (0..n).map(|_| f32b(rng.next())).collect(); + let want32: Vec = (0..n) + .map(|i| mul_add_f32(a32[i], b32[i], acc32[i])) + .collect(); + add_mul_f32(&mut acc32, &a32, &b32); + let a64: Vec = (0..n).map(|_| f64b(rng.next())).collect(); + let b64: Vec = (0..n).map(|_| f64b(rng.next())).collect(); + let mut acc64: Vec = (0..n).map(|_| f64b(rng.next())).collect(); + let want64: Vec = (0..n) + .map(|i| mul_add_f64(a64[i], b64[i], acc64[i])) + .collect(); + add_mul_f64(&mut acc64, &a64, &b64); + for i in 0..n { + if canon32(acc32[i]) != canon32(want32[i]) { + return Err(0xC400); + } + if canon64(acc64[i]) != canon64(want64[i]) { + return Err(0xC401); + } + h.word(canon32(acc32[i])); + h.word(canon64(acc64[i])); + } + } + Ok(h.0) +} + +/// The float facade is IEEE 754 bit-exact and returns the same bits on every +/// realization (see the section comment above for the exact scope). +fn check_ieee_bit_exact() -> Result<(), u32> { + if ieee_digest()? != IEEE_GOLDEN { + return Err(0xC4FF); + } + Ok(()) +} diff --git a/crates/simd-masking-parity/src/main.rs b/crates/simd-masking-parity/src/main.rs index ca7fabb7..2b774910 100644 --- a/crates/simd-masking-parity/src/main.rs +++ b/crates/simd-masking-parity/src/main.rs @@ -13,6 +13,10 @@ fn main() { cfg!(target_feature = "simd128"), cfg!(feature = "nightly-simd"), ); + match simd_masking_parity::ieee_digest() { + Ok(d) => println!("ieee-digest={d:#018x} (golden {:#018x})", simd_masking_parity::IEEE_GOLDEN), + Err(code) => println!("ieee-digest: lane differs from the scalar IEEE operation, code = {code:#x}"), + } let rc = simd_masking_parity::run(); if rc != 0 { eprintln!("simd-masking-parity FAILED: code = {rc:#x} (see src/lib.rs)");