diff --git a/.gitattributes b/.gitattributes index 650fc33..68ec940 100644 --- a/.gitattributes +++ b/.gitattributes @@ -28,3 +28,9 @@ vendor/afm/** -text # which format a later fixture takes. And here rather than as a `core.autocrlf false` step in CI, # because that would give the runner bytes no Windows clone of this repository gets. fixtures/** -text + +# The recognizer's models, fixture images and expected answers are bytes (docs/35-OCR-SCOPE.md). +# Its dictionary and expected.jsonl are ASCII with LF endings, so git would take them for text, and a +# CRLF checkout would move the dictionary's pinned sha256 in the recognizer's identity. +ocr/models/** -text +ocr/tests/data/** -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60dd5e6..8a57c09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,6 +220,39 @@ jobs: - name: deny run: cargo deny check + # The recognizer behind `--ocr` (docs/35-OCR-SCOPE.md): its own workspace and its own + # pinned toolchain, which the script installs and asserts. `ocr-platforms` below runs its + # fixture test on every supported CPU family; this step keeps it in the gate. + - name: ocr + run: ci/ocr.sh + + # ----------------------------------------------------------------------------------------- + # docs/35-OCR-SCOPE.md §10.1 — the recognizer answers with the same bytes on every platform + # ----------------------------------------------------------------------------------------- + # + # Its fixture test compares every answer's bytes with ocr/tests/data/expected.jsonl, written on + # Apple silicon. Linux x86-64, Linux arm64, Windows x86-64 and Intel macOS each run it: four + # vector widths' worth of SIMD kernels (rten picks AVX-512 where the runner has it) and three + # operating systems' maths libraries, none of which may move a byte. + ocr-platforms: + if: github.event_name != 'schedule' + name: ocr same bytes · ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master branch, 2026-09-24 + with: + toolchain: "1.98.0" + - name: The recognizer's fixtures, byte for byte + working-directory: ocr + shell: bash + run: cargo test --release --locked + # ----------------------------------------------------------------------------------------- # docs/history/03-V0-SCOPE.md §5 — every exit criterion, as its own named job # ----------------------------------------------------------------------------------------- diff --git a/CHANGELOG.md b/CHANGELOG.md index b28774c..cb47486 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,58 @@ milestone documents ([`05`](docs/history/05-MILESTONES.md), [`09`](docs/history/ ## [Unreleased] +### ParseBench runs reuse the score of an unchanged output (decision #121) + +- **`ETHOS_BENCH_LEDGER=`** makes the ParseBench adapter store each result under the SHA-256 of + what ParseBench's scorer receives, and reuse it for an example whose inputs repeat. The scores are + the same, checked against fresh scoring (0 of 2,553 apart), and an `--ocr` run with a few pages + changed scores in 38 s instead of 298. A held-out split was measured too, and not built: at this + benchmark's size it could not tell a typical decision from chance. +- The adapter's README pins ParseBench at `1f5b37a`, where its numbers were measured, and states a + full run's time as measured, about eight minutes. + +### A scanned page and an image file are read, and say so: OCR, v3 (decision #120) + +`extract --ocr` and `markdown --source --ocr` read text no text layer carries. Without the flag +nothing changes but the profile hash: the default profile gained a field, `recognizer`, set to +`not_pinned`, so its `profile_sha256` moved, and every artifact's with it, as every release's does. + +- **A page that shows no text and places exactly one image is read by a recognizer**, and each + word it reads is a `text_run` of derivation `recognized`, under the OCR profile, whose + `profile_sha256` differs from the default's. The trigger is a reason code, never a score. A page + that needs OCR and is not read — several images, an inline image, a placement that is not + axis-aligned, a filter or colour space the recognizer does not decode, an image a mask shapes — + is counted under `scanned-page-not-recognized` with its reason. Nothing recognized is merged + with drawn text: the seal refuses a page holding both, so contract §6's placement rule is + mechanical. +- **A PNG or JPEG file is a one-page document**, its page the file's own pixel grid, one pixel to + one point, declared under `image-resolution-not-read`; the source is the file. Without `--ocr` + an image file is refused by name, with the flag that reads it. `ground` refuses it by name: + `ethos.grounding.v1` admits PDF and the office sources only (decision #16). +- **The recognizer is `ethos-parser-ocr`**, a separate binary in `ocr/`, on Rust 1.98.0 while the + engine stays on 1.88: PP-OCRv5 mobile (detection and English recognition, 12.7 MB of models, + Apache-2.0, built into the binary) on rten 0.26. Its output is the same bytes on every supported + CPU: rten's sums are made order-fixed in a vendored `rten-vecmath`, and Linux x86-64 and arm64, + Windows x86-64 with AVX-512 and macOS arm64 and x86-64 answer identically, at one thread and at + the default. An x86-64 CPU without AVX2 and FMA is refused by name. The engine finds it beside + its own executable or at `ETHOS_PARSER_OCR`; without it `--ocr` fails by name and never falls + back to an artifact read without it. No confidence is written. +- **On the wire:** the locator variant `recognized` (the run's place on its page, the image object + it was read from, the word's corners in the image's pixels); the profile's `recognizer`; the + source media types `image/png` and `image/jpeg`; five limitation codes, + `scanned-page-not-recognized`, `recognized-words-outside-visible-box`, + `recognized-text-orientation-not-corrected` and `recognized-text-has-no-font`, and + `image-resolution-not-read`. In the library: `extract_recognized`, `extract_image`, + `image_media_type`, the `Recognize` trait and the protocol's types. +- **Measured.** ParseBench with `--ocr`: 52.10 -> 53.94 overall, 75 pages up and none down + (content faithfulness 0.6750 -> 0.7524). Without the flag, the eight gate documents and + opendataloader-bench's 200 extract byte-identical but the profile hash; with it, the same 208 + hold the same nodes. About 1.9 s and 1.35 GB peak per 200-dpi page on an Apple M4 Pro. +- **Not read yet:** composite pages, pages that need a renderer, CCITT, JBIG2 and JPEG 2000 + images, photos, text turned within an image. OCR moves ahead of assist, so it is v3 and assist + v4. `ci/gate.sh` gains a tenth step, `ci/ocr.sh`, and CI an `ocr-platforms` job that runs the + recognizer's byte-identity fixtures on five runners. + ### A footnote mark set against its label joins its line, an amount that opens with its sign opens its cell, and small numbers beside amounts key a table (decision #119) Folded into `whitespace-tracks-v16`, which no release carries yet, so no rule id and no diff --git a/README.md b/README.md index 7c8f14f..e6c5eeb 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ ethos-parser ground repr.json > grounding.json | Command | What it does | Exit codes | | --- | --- | --- | | `classify` | Counts and reason codes for a PDF — is it scanned? is the layout hard? | 0 / 1 / 2 | -| `extract` | The document as text runs with locators (`DocumentRepresentation v0`) | 0 / 2 | +| `extract` | The document as text runs with locators (`DocumentRepresentation v0`). `--ocr` also reads scanned pages, and PNG or JPEG files, through the recognizer `ethos-parser-ocr` ([`docs/35-OCR-SCOPE.md`](docs/35-OCR-SCOPE.md)) | 0 / 2 | | `ground` | Turn a representation into `ethos.grounding.v1`, the citation format | 0 / 2 | | `markdown` | Markdown plus a map from every character back to a node. Takes a representation, or `--source ` to read a document and project it in one process — the same bytes, without the record on disk between the two stages | 0 / 2 | | `html` | The same thing in HTML, projected from the document rather than from the Markdown | 0 / 2 | @@ -168,7 +168,7 @@ stdout never changes. ## Formats it reads PDF, DOCX, XLSX, PPTX, ODT, ODS, ODP, RTF and EPUB — all into the same record, with the same -fingerprint and the same JSON shape. +fingerprint and the same JSON shape. With `--ocr`, PNG and JPEG too, each read as one page. The format is decided by looking at the bytes, never the file extension. A `report.bin` that is really a PDF still reads; a `.docx` full of something else is a clear error. Anything the engine @@ -176,9 +176,9 @@ has no reader for is refused by name: ``` $ ethos-parser extract data.csv -engine: unsupported media type: these bytes state no format this engine reads. Three signatures +engine: unsupported media type: these bytes state no format this engine reads. Five signatures were looked for at byte 0 and none is present: a ZIP local file header ... an RTF brace group ... -and a PDF header. Nothing else was consulted — not the file's name ... +a PDF header; and a PNG's or a JPEG's ... Nothing else was consulted — not the file's name ... ``` CSV is deliberately absent. A CSV parse would produce real text, but the engine would have to diff --git a/ci/forbidden-tokens.sh b/ci/forbidden-tokens.sh index fab2f5e..976743d 100755 --- a/ci/forbidden-tokens.sh +++ b/ci/forbidden-tokens.sh @@ -45,8 +45,8 @@ case "$mode" in # 0` — a verdict its own evidence contradicts. pattern='confidence|quality_score|trust_score|is_good' why='docs/01-CONTRACT.md §9 forbids any public confidence field, score, grade, or quality -summary — at any version, including "diagnostic only". The v2.1 OCR lane may record a -processor-reported uncertainty, and when it does it goes on a recognition type under its own +summary — at any version, including "diagnostic only". The OCR lane (v3) writes none; if a +later slice records a processor-reported uncertainty, it goes on a recognition type under its own profile, never on these.' ;; verification) @@ -66,7 +66,7 @@ the failure this whole project is arranged to prevent.' ;; esac -files=$(find crates -path '*/src/*' -name '*.rs' | sort) +files=$(find crates ocr -path '*/src/*' -name '*.rs' -not -path 'ocr/vendor/*' | sort) if [ -z "$files" ]; then echo "::error::no crate sources found; the scan would pass vacuously" >&2 exit 1 diff --git a/ci/gate.sh b/ci/gate.sh index 12a9bc5..55702df 100755 --- a/ci/gate.sh +++ b/ci/gate.sh @@ -157,7 +157,7 @@ elif [ -x ../ethos-oracle/target/release/ethos ]; then fi step=0 -total=9 +total=10 announce() { step=$((step + 1)) printf '\n\033[1m[%d/%d] %s\033[0m\n' "$step" "$total" "$1" @@ -190,6 +190,9 @@ cargo deny check announce 'sdk suites — node and python, the version guards that had never run' ci/sdk-suites.sh +announce 'ocr — the recognizer, in its own workspace on its own toolchain' +ci/ocr.sh + printf '\n\033[1;32mgate: all %d checks passed.\033[0m\n' "$total" printf 'Not run, by design: the oracle build, the toolchain tripwire, v0-fuzz-smoke,\n' printf 'deny-policy-is-enforced, and the labelled re-runs of the suite above.\n' diff --git a/ci/ocr.sh b/ci/ocr.sh new file mode 100755 index 0000000..0999a22 --- /dev/null +++ b/ci/ocr.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# The recognizer behind `--ocr` (docs/35-OCR-SCOPE.md): fmt, clippy and its tests, in its own +# workspace on its own pinned toolchain (ocr/rust-toolchain.toml, 1.98.0; the engine stays on +# 1.88.0). Its fixture test compares every answer's bytes with ocr/tests/data/expected.jsonl, +# written on Apple silicon, so on any other CPU this is the same-bytes check of §10.1. +# +# Release mode: the fixture test runs both models over five pages, which a debug build of rten +# takes minutes to do. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../ocr" +want="$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+' rust-toolchain.toml | head -1)" +rustup toolchain install "$want" --profile minimal --component rustfmt,clippy >/dev/null +got="$(cargo --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)" +echo "ocr/rust-toolchain.toml wants $want; cargo reports $got" +[ "$want" = "$got" ] || { echo "ocr: toolchain pin not in force" >&2; exit 1; } +cargo fmt --check +cargo clippy --release --all-targets --locked -- -D warnings +cargo test --release --locked diff --git a/crates/ethos-parser-cli/src/main.rs b/crates/ethos-parser-cli/src/main.rs index 11b685a..7528769 100644 --- a/crates/ethos-parser-cli/src/main.rs +++ b/crates/ethos-parser-cli/src/main.rs @@ -39,6 +39,7 @@ #![forbid(unsafe_code)] mod mcp; +mod recognizer; use std::io::Write; use std::path::{Path, PathBuf}; @@ -314,6 +315,15 @@ struct ExtractArgs { /// request. #[arg(long, value_name = "N")] max_pages: Option, + + /// Read every page that shows no text and places one image — a scan — with the recognizer, + /// `ethos-parser-ocr` (beside this binary, or named by `ETHOS_PARSER_OCR`). Its words are runs + /// of derivation `recognized`, under the OCR profile, so the artifact's `profile_sha256` + /// differs from one made without it (`docs/35-OCR-SCOPE.md`). Pages that show text are read as + /// they always are; nothing recognized is ever merged with them. A PNG or JPEG file is read as + /// one page, its own pixel grid. + #[arg(long)] + ocr: bool, } #[derive(clap::Args)] @@ -421,6 +431,12 @@ struct MarkdownArgs { /// record — to ground a claim, to locate a quote, to keep — runs `extract` and keeps it. #[arg(long, value_name = "FILE")] source: Option, + + /// With `--source`, read the source through the recognizer, as `extract` does with the same + /// flag: its scanned pages, or the image file. The output is byte-identical to that extract + /// piped into this subcommand. + #[arg(long, requires = "source")] + ocr: bool, } #[derive(clap::Args)] @@ -735,7 +751,34 @@ fn run_extract(args: ExtractArgs) -> ExitCode { } profile.page_budget = ethos_parser_core::PageBudget::AtMost(n); } - emit_representation(representation_for_bytes(&head, &profile)) + if args.ocr { + emit_representation(recognized_representation(&head, profile)) + } else { + emit_representation(representation_for_bytes(&head, &profile)) + } +} + +/// `--ocr`: the OCR profile (with any page budget the caller set), the recognizer found and +/// pinned, and a PDF read through it. Office bytes are refused by name: they have no scanned +/// pages to read. +fn recognized_representation( + head: &[u8], + base: Profile, +) -> Result { + if routes_to_office(head) { + return Err(EngineError::Unsupported { + what: "option".into(), + detail: "--ocr reads the scanned pages of a PDF; these bytes go to the office reader, \ + which has no pages to scan" + .into(), + }); + } + let profile = Profile { + page_budget: base.page_budget, + ..Profile::ocr_v0() + }; + let mut process = recognizer::Process::find()?; + representation_for(head, &profile, Some(&mut process)) } /// Route bytes to the reader their own signatures name, and return the canonical @@ -752,10 +795,35 @@ fn run_extract(args: ExtractArgs) -> ExitCode { pub(crate) fn representation_for_bytes( head: &[u8], profile: &Profile, +) -> Result { + representation_for(head, profile, None) +} + +/// [`representation_for_bytes`], reading a PDF's scanned pages through `recognizer` when one is +/// given (`--ocr`, `docs/35-OCR-SCOPE.md`). +fn representation_for( + head: &[u8], + profile: &Profile, + recognizer: Option<&mut dyn ethos_parser_pdf::Recognize>, ) -> Result { if routes_to_office(head) { return ethos_parser_office::read(head); } + // v3 (`docs/35-OCR-SCOPE.md` §4): a PNG or JPEG states its format at byte 0, and is a page only + // a recognizer reads, so it is read with `--ocr` and refused by name without it. + if let Some(media_type) = ethos_parser_pdf::image_media_type(head) { + let Some(recognizer) = recognizer else { + return Err(EngineError::Unsupported { + what: "media type".into(), + detail: format!( + "these bytes are an image file ({media_type}), and an image file's text is \ + read only by the recognizer: run with --ocr" + ), + }); + }; + let extract = ethos_parser_pdf::extract_image(head, profile, recognizer)?; + return ethos_parser_pdf::to_representation(&extract, profile); + } // **v2-S10: the branch that was missing, and the last member of the shape S8 named.** // @@ -767,8 +835,9 @@ pub(crate) fn representation_for_bytes( // for its whole class rather than one more member; this is the class S8 left, and it is the // last one there is: bytes carrying no signature, no container and no declaration. // - // **This is not a seventh predicate and nothing new is sniffed.** It is the negation of the - // six above plus the one question the PDF reader already answers about itself. No comma is + // **This is not another predicate and nothing new is sniffed.** It is the negation of the + // office six and the image signature above (v3), plus the one question the PDF reader already + // answers about itself. No comma is // counted, no line is measured, no extension is read, and no signature is looked for past // byte 0. Comma-separated text cannot be told from prose without a reader, and a detector that // guessed would claim every comma file — so this engine refuses to name a format it did not @@ -790,7 +859,10 @@ pub(crate) fn representation_for_bytes( // consumes. The stage artifact remains the library's return type, so M3's acceptance suite // still asserts on the thing the parser actually produces. let doc = Document::open_bytes(head, profile)?; - let extract = ethos_parser_pdf::extract(&doc, profile)?; + let extract = match recognizer { + Some(r) => ethos_parser_pdf::extract_recognized(&doc, profile, r)?, + None => ethos_parser_pdf::extract(&doc, profile)?, + }; ethos_parser_pdf::to_representation(&extract, profile) } @@ -823,10 +895,10 @@ fn routes_to_office(head: &[u8]) -> bool { fn no_format_stated() -> EngineError { EngineError::Unsupported { what: "media type".into(), - detail: "these bytes state no format this engine reads. Three signatures were looked for \ + detail: "these bytes state no format this engine reads. Five signatures were looked for \ at byte 0 and none is present: a ZIP local file header, the container a DOCX, \ - XLSX, PPTX, ODT, ODS, ODP or EPUB arrives in; an RTF brace group; and a PDF \ - header. Nothing else was consulted — not the file's name, and no signature at any \ + XLSX, PPTX, ODT, ODS, ODP or EPUB arrives in; an RTF brace group; a PDF header; \ + and a PNG's or a JPEG's, an image file `--ocr` reads. Nothing else was consulted — not the file's name, and no signature at any \ other offset. A file this engine has no reader for is refused by name rather than \ handed to a reader it never named, because a refusal that guesses a format is a \ claim nobody measured. `docs/CAPABILITY.md` lists what is read and what is not." @@ -978,9 +1050,13 @@ fn run_locate(args: LocateArgs) -> ExitCode { fn representation_to_project( path: Option<&std::path::Path>, source: Option<&std::path::Path>, + ocr: bool, ) -> Result { if let Some(source) = source { let bytes = read_source(source)?; + if ocr { + return recognized_representation(&bytes, Profile::default()); + } return representation_for_bytes(&bytes, &Profile::default()); } // clap holds the other arm: exactly one of the two is present. @@ -996,10 +1072,11 @@ fn representation_to_project( } fn run_markdown(args: MarkdownArgs) -> ExitCode { - let repr = match representation_to_project(args.path.as_deref(), args.source.as_deref()) { - Ok(r) => r, - Err(e) => return fail(&e), - }; + let repr = + match representation_to_project(args.path.as_deref(), args.source.as_deref(), args.ocr) { + Ok(r) => r, + Err(e) => return fail(&e), + }; let profile = Profile::default(); let profile_sha256 = match profile.profile_sha256() { diff --git a/crates/ethos-parser-cli/src/recognizer.rs b/crates/ethos-parser-cli/src/recognizer.rs new file mode 100644 index 0000000..52df4e5 --- /dev/null +++ b/crates/ethos-parser-cli/src/recognizer.rs @@ -0,0 +1,166 @@ +// Copyright 2026 The ethos-parser maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! `--ocr`'s recognizer: `ethos-parser-ocr`, a separate binary on its own toolchain, which this +//! process finds, asks for its identity, and talks to over its stdin and stdout +//! (`docs/35-OCR-SCOPE.md` §5). +//! +//! **Found, never fallen back from.** `ETHOS_PARSER_OCR` names it when set, and a path that is not +//! a file is a named error; otherwise it is `ethos-parser-ocr` beside this executable. A missing +//! recognizer fails `--ocr` by name rather than producing an artifact without OCR +//! (`docs/07-VERIFY-BOUNDARY.md` §8.6). The `serve` process starts on the first scanned page, so a +//! document with none costs one `identity` call. + +use std::io::{BufRead, BufReader, Write}; +use std::path::PathBuf; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; + +use ethos_parser_core::{EngineError, RecognizerIdentity}; +use ethos_parser_pdf::{Recognition, RecognitionRequest, Recognize}; + +/// The environment variable that names the recognizer binary. +pub(crate) const ENV: &str = "ETHOS_PARSER_OCR"; + +/// A running (or not yet started) `ethos-parser-ocr serve`. +pub(crate) struct Process { + path: PathBuf, + identity: RecognizerIdentity, + serving: Option<(Child, ChildStdin, BufReader)>, +} + +fn missing(part: String) -> EngineError { + EngineError::MissingPart { part } +} + +impl Process { + /// Find the recognizer and read its identity. + pub(crate) fn find() -> Result { + let name = if cfg!(windows) { + "ethos-parser-ocr.exe" + } else { + "ethos-parser-ocr" + }; + let path = match std::env::var_os(ENV) { + Some(p) => PathBuf::from(p), + None => std::env::current_exe() + .map_err(|e| EngineError::Io { + detail: e.to_string(), + })? + .with_file_name(name), + }; + if !path.is_file() { + return Err(missing(format!( + "recognizer `{}`: --ocr reads scanned pages with ethos-parser-ocr, which must sit \ + beside ethos-parser or be named by {ENV}", + path.display() + ))); + } + let out = Command::new(&path) + .arg("identity") + .output() + .map_err(|e| missing(format!("recognizer `{}`: {e}", path.display())))?; + if !out.status.success() { + return Err(missing(format!( + "recognizer `{}` did not state its identity: {}", + path.display(), + String::from_utf8_lossy(&out.stderr).trim() + ))); + } + let identity = serde_json::from_slice(&out.stdout).map_err(|e| EngineError::Malformed { + what: "recognizer identity".into(), + detail: e.to_string(), + })?; + Ok(Self { + path, + identity, + serving: None, + }) + } + + fn serving(&mut self) -> Result<&mut (Child, ChildStdin, BufReader), EngineError> { + if self.serving.is_none() { + let mut child = Command::new(&self.path) + .arg("serve") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| missing(format!("recognizer `{}`: {e}", self.path.display())))?; + let stdin = child.stdin.take().expect("piped"); + let stdout = BufReader::new(child.stdout.take().expect("piped")); + self.serving = Some((child, stdin, stdout)); + } + Ok(self.serving.as_mut().expect("started above")) + } + + /// The recognizer stopped answering: say why, from its exit status and what it wrote. + fn stopped(&mut self) -> EngineError { + let Some((mut child, stdin, _)) = self.serving.take() else { + return EngineError::Io { + detail: "the recognizer is not running".into(), + }; + }; + drop(stdin); + let status = child.wait().ok(); + let mut stderr = String::new(); + if let Some(mut e) = child.stderr.take() { + let _ = std::io::Read::read_to_string(&mut e, &mut stderr); + } + let said = stderr.trim(); + if status.and_then(|s| s.code()) == Some(3) { + return EngineError::Unsupported { + what: "cpu".into(), + detail: said.to_string(), + }; + } + EngineError::Io { + detail: format!("the recognizer stopped: {said}"), + } + } +} + +impl Recognize for Process { + fn identity(&self) -> &RecognizerIdentity { + &self.identity + } + + fn read(&mut self, request: &RecognitionRequest) -> Result { + let header = serde_json::to_string(request).map_err(|e| EngineError::Malformed { + what: "recognizer request".into(), + detail: e.to_string(), + })?; + let (_, stdin, stdout) = self.serving()?; + let sent = writeln!(stdin, "{header}") + .and_then(|()| stdin.write_all(&request.data)) + .and_then(|()| stdin.flush()); + let mut line = String::new(); + let read = sent.and_then(|()| stdout.read_line(&mut line)); + if !matches!(read, Ok(n) if n > 0) { + return Err(self.stopped()); + } + serde_json::from_str(line.trim_end()).map_err(|e| EngineError::Malformed { + what: "recognizer answer".into(), + detail: e.to_string(), + }) + } +} + +impl Drop for Process { + fn drop(&mut self) { + if let Some((mut child, stdin, _)) = self.serving.take() { + drop(stdin); + let _ = child.wait(); + } + } +} diff --git a/crates/ethos-parser-cli/tests/draft_schemas.rs b/crates/ethos-parser-cli/tests/draft_schemas.rs index c454e56..2479662 100644 --- a/crates/ethos-parser-cli/tests/draft_schemas.rs +++ b/crates/ethos-parser-cli/tests/draft_schemas.rs @@ -292,6 +292,27 @@ fn artifacts() -> Vec<(String, &'static str, Value)> { let repr = ethos_parser_pdf::to_representation(&extract, profile).expect("represents"); representations.push((name, repr)); } + // v3 (decision #120): a scan and an image file, read by a recognizer that is a function, so the + // OCR profile and the recognized locator — with an image object and without — are walked too. + let ocr = Profile::ocr_v0(); + out.push(("ocr profile".into(), "profile", as_json!(ocr))); + let scan = Document::open_bytes(&scanned_pdf(), &ocr).expect("the scan opens"); + for (name, extract) in [ + ( + "scan.pdf", + ethos_parser_pdf::extract_recognized(&scan, &ocr, &mut Reads::new()), + ), + ( + "scan.png", + ethos_parser_pdf::extract_image(b"\x89PNG\r\n\x1a\n", &ocr, &mut Reads::new()), + ), + ] { + let extract = extract.expect("the recognizer reads it"); + out.push((name.into(), "extract", as_json!(extract))); + let repr = ethos_parser_pdf::to_representation(&extract, &ocr).expect("represents"); + representations.push((name.into(), repr)); + } + let office = repo_root().join("fixtures/office"); let mut documents: Vec = std::fs::read_dir(&office) .expect("fixtures/office is readable") @@ -324,6 +345,80 @@ fn artifacts() -> Vec<(String, &'static str, Value)> { out } +/// A one-page scan: a 200 x 100 point page that paints one 20 x 10 gray image over itself and +/// shows no text. +fn scanned_pdf() -> Vec { + use lopdf::{dictionary, Object, Stream}; + let mut doc = lopdf::Document::with_version("1.7"); + let pages = doc.new_object_id(); + let image = doc.add_object(Stream::new( + dictionary! { + "Type" => "XObject", "Subtype" => "Image", "Width" => 20, "Height" => 10, + "ColorSpace" => "DeviceGray", "BitsPerComponent" => 8, + }, + vec![255u8; 200], + )); + let content = doc.add_object(Stream::new( + dictionary! {}, + b"q 200 0 0 100 0 0 cm /Im0 Do Q".to_vec(), + )); + let page = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages, + "MediaBox" => vec![0.into(), 0.into(), 200.into(), 100.into()], + "Resources" => dictionary! { "XObject" => dictionary! { "Im0" => image } }, + "Contents" => content, + }); + doc.objects.insert( + pages, + Object::Dictionary( + dictionary! { "Type" => "Pages", "Kids" => vec![page.into()], "Count" => 1 }, + ), + ); + let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages }); + doc.trailer.set("Root", catalog); + let mut bytes = Vec::new(); + doc.save_to(&mut bytes).expect("saves"); + bytes +} + +/// A recognizer that is a function: "Hello world" off any image, as a 20 x 10 pixel one. +struct Reads(ethos_parser_core::RecognizerIdentity); + +impl Reads { + fn new() -> Self { + Reads(ethos_parser_core::RecognizerIdentity::ethos_parser_ocr()) + } +} + +impl ethos_parser_pdf::Recognize for Reads { + fn identity(&self) -> ðos_parser_core::RecognizerIdentity { + &self.0 + } + fn read( + &mut self, + request: ðos_parser_pdf::RecognitionRequest, + ) -> Result { + let word = |text: &str, x0: i32, x1: i32| ethos_parser_pdf::RecognizedWord { + text: text.into(), + quad: [[x0, 2], [x1, 2], [x1, 8], [x0, 8]], + }; + Ok(ethos_parser_pdf::Recognition { + ocr_response: 1, + id: request.id.clone(), + identity: self.0.clone(), + width: 20, + height: 10, + lines: vec![ethos_parser_pdf::RecognizedLine { + text: "Hello world".into(), + quad: [[2, 2], [18, 2], [18, 8], [2, 8]], + words: vec![word("Hello", 2, 9), word("world", 11, 18)], + }], + refused: None, + }) + } +} + /// The three projections of `repr`, each where the engine produces one. fn project( name: &str, diff --git a/crates/ethos-parser-cli/tests/fake-ocr.sh b/crates/ethos-parser-cli/tests/fake-ocr.sh new file mode 100755 index 0000000..e831d2b --- /dev/null +++ b/crates/ethos-parser-cli/tests/fake-ocr.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# A recognizer that reads nothing, for the CLI's tests (v3, docs/35-OCR-SCOPE.md §5): it states the +# identity in FAKE_OCR_IDENTITY and answers every request with FAKE_OCR_ANSWER, after reading the +# request's bytes. The real recognizer's protocol is held by ocr/tests/fixtures.rs. +case "$1" in +identity) printf '%s\n' "$FAKE_OCR_IDENTITY" ;; +serve) + while IFS= read -r header; do + n=$(printf '%s' "$header" | sed 's/.*"bytes":\([0-9]*\).*/\1/') + head -c "$n" >/dev/null + printf '%s\n' "$FAKE_OCR_ANSWER" + done + ;; +*) exit 2 ;; +esac diff --git a/crates/ethos-parser-cli/tests/markdown_cli.rs b/crates/ethos-parser-cli/tests/markdown_cli.rs index f0c643e..5c8acf7 100644 --- a/crates/ethos-parser-cli/tests/markdown_cli.rs +++ b/crates/ethos-parser-cli/tests/markdown_cli.rs @@ -375,9 +375,10 @@ fn no_cli_path_emits_markdown_without_its_map() { .map(|w| w.trim_end_matches(',')) .collect(); assert!( - flags - .iter() - .all(|f| matches!(*f, "--help" | "--version" | "--diagnostics" | "--source")), + flags.iter().all(|f| matches!( + *f, + "--help" | "--version" | "--diagnostics" | "--source" | "--ocr" + )), "`ethos-parser markdown` grew a flag: {flags:?}. Any option that could suppress the Anchor Map \ is the one thing this version exists to prevent (docs/history/10-V11-SCOPE.md law 1), so a new \ flag here is a deliberate decision that needs its own evidence." @@ -398,6 +399,17 @@ fn no_cli_path_emits_markdown_without_its_map() { source_help.contains("byte-identical"), "--source must document the equality that admits it, in the text a caller reads: {source_help:?}" ); + // **`--ocr` is admitted on the same evidence** (v3, `docs/35-OCR-SCOPE.md`): it decides how the + // record is read — `extract --ocr`'s reading, in this process — and suppresses nothing. + // [`the_ocr_source_path_equals_the_two_step_path`] asserts the bytes. + let ocr_help = options + .split("--ocr") + .nth(1) + .expect("the flag is declared, so clap documents it"); + assert!( + ocr_help.contains("byte-identical"), + "--ocr must document the equality that admits it: {ocr_help:?}" + ); // And the artifact itself always carries both halves. let dir = scratch("paired"); @@ -1651,6 +1663,72 @@ fn the_source_path_equals_the_two_step_path() { } } +/// **`--ocr` is `extract --ocr` in this process, and the bytes say so** (v3, +/// `docs/35-OCR-SCOPE.md`). The recognizer is `tests/fake-ocr.sh`, which states the identity the +/// OCR profile pins and reads "Hello world" off any image; the source is an image file, which the +/// engine does not decode, so its signature is all of it. +#[cfg(unix)] +#[test] +fn the_ocr_source_path_equals_the_two_step_path() { + let identity = serde_json::to_value(ethos_parser_core::RecognizerIdentity::ethos_parser_ocr()) + .expect("serializes"); + let word = |t: &str, x0: i32, x1: i32| serde_json::json!({"text": t, "quad": [[x0, 20], [x1, 20], [x1, 60], [x0, 60]]}); + let answer = serde_json::json!({ + "ocr_response": 1, "id": "fake", "identity": identity, "width": 200, "height": 100, + "lines": [{"text": "Hello world", "quad": [[20, 20], [180, 20], [180, 60], [20, 60]], + "words": [word("Hello", 20, 90), word("world", 110, 180)]}], + "refused": null + }); + let fake = repo_root().join("crates/ethos-parser-cli/tests/fake-ocr.sh"); + let with_ocr = |args: &[&str]| -> Output { + Command::new(env!("CARGO_BIN_EXE_ethos-parser")) + .args(args) + .env("ETHOS_PARSER_OCR", &fake) + .env("FAKE_OCR_IDENTITY", identity.to_string()) + .env("FAKE_OCR_ANSWER", answer.to_string()) + .output() + .expect("the engine binary runs") + }; + let dir = scratch("ocr-source-equals"); + let image = dir.join("scan.png"); + std::fs::write(&image, b"\x89PNG\r\n\x1a\n and the rest of the file").expect("write"); + let image = image.to_str().unwrap(); + + let extracted = with_ocr(&["extract", "--ocr", image]); + assert_eq!( + extracted.status.code(), + Some(0), + "extract --ocr: {}", + String::from_utf8_lossy(&extracted.stderr) + ); + let record: Value = serde_json::from_slice(&extracted.stdout).expect("a record"); + assert_eq!( + record["representation"]["source"]["media_type"], + "image/png" + ); + let repr_path = dir.join("scan.json"); + std::fs::write(&repr_path, &extracted.stdout).expect("write the record"); + + let two_step = engine(&["markdown", repr_path.to_str().unwrap()]); + let one_step = with_ocr(&["markdown", "--source", image, "--ocr"]); + assert_eq!(two_step.status.code(), Some(0)); + assert_eq!( + one_step.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&one_step.stderr) + ); + assert_eq!( + one_step.stdout, two_step.stdout, + "markdown --source --ocr differs from extract --ocr then markdown" + ); + let md: Value = serde_json::from_slice(&one_step.stdout).expect("an artifact"); + assert!( + md["markdown"].as_str().unwrap().contains("Hello world"), + "{md}" + ); +} + /// **Neither input, or both, is a usage error — and a source that states no format is the /// extractor's own refusal.** /// diff --git a/crates/ethos-parser-cli/tests/no_format_cli.rs b/crates/ethos-parser-cli/tests/no_format_cli.rs index 73cbc98..9dc3f22 100644 --- a/crates/ethos-parser-cli/tests/no_format_cli.rs +++ b/crates/ethos-parser-cli/tests/no_format_cli.rs @@ -150,7 +150,12 @@ fn the_refusal_names_what_was_looked_for_and_claims_no_format() { stderr.contains("these bytes state no format this engine reads"), "the cause, stated: {stderr}" ); - for looked_for in ["ZIP local file header", "RTF brace group", "PDF header"] { + for looked_for in [ + "ZIP local file header", + "RTF brace group", + "PDF header", + "a PNG's or a JPEG's", + ] { assert!( stderr.contains(looked_for), "the refusal must name {looked_for} as something looked for: {stderr}" @@ -178,6 +183,51 @@ fn the_refusal_names_what_was_looked_for_and_claims_no_format() { let _ = std::fs::remove_dir_all(&dir); } +/// **An image file states its format** (v3, `docs/35-OCR-SCOPE.md` §4): a PNG or a JPEG is not +/// refused as bytes that state none, and its refusal without `--ocr` names the flag that reads it. +#[test] +fn an_image_file_without_ocr_names_the_flag_that_reads_it() { + let dir = tempdir(); + for (name, bytes, media_type) in [ + ("scan.png", &b"\x89PNG\r\n\x1a\n and more"[..], "image/png"), + ("scan.jpg", &b"\xFF\xD8\xFF\xE0 and more"[..], "image/jpeg"), + ] { + let stderr = refuse(&dir, name, bytes); + assert!( + stderr.contains(media_type) && stderr.contains("--ocr"), + "{name}: {stderr}" + ); + assert!( + !stderr.contains("state no format"), + "{name}: an image file states one: {stderr}" + ); + } + let _ = std::fs::remove_dir_all(&dir); +} + +/// **`--ocr` without its recognizer fails by name**, never into an artifact read without it +/// (`docs/07-VERIFY-BOUNDARY.md` §8.6). +#[test] +fn ocr_without_its_recognizer_fails_by_name() { + let dir = tempdir(); + let path = dir.join("scan.png"); + std::fs::write(&path, b"\x89PNG\r\n\x1a\n and more").expect("write the file"); + let missing = dir.join("no-such-recognizer"); + let out = Command::new(env!("CARGO_BIN_EXE_ethos-parser")) + .args(["extract", "--ocr", path.to_str().expect("utf-8 path")]) + .env("ETHOS_PARSER_OCR", &missing) + .output() + .expect("the engine binary runs"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(2), "{stderr}"); + assert!(out.stdout.is_empty(), "no artifact without the recognizer"); + assert!( + stderr.contains("ETHOS_PARSER_OCR") && stderr.contains("no-such-recognizer"), + "{stderr}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + /// **The wrong cause is gone.** These bytes never aimed at the PDF reader. /// /// The defect v2-S6 fixed for an `.ods`, v2-S8 for an `.rtf` and for the ZIP *shape*, and this diff --git a/crates/ethos-parser-cli/tests/public_api.rs b/crates/ethos-parser-cli/tests/public_api.rs index 60d4b49..438c53e 100644 --- a/crates/ethos-parser-cli/tests/public_api.rs +++ b/crates/ethos-parser-cli/tests/public_api.rs @@ -423,6 +423,11 @@ const CORE: &[&str] = &[ "TextRunAttributes", "VerifierBinary", "VerifierPin", + // v3 (decision #120, docs/35-OCR-SCOPE.md): the recognizer pin and identity on the profile, + // and the address a recognized run carries. + "RecognizedLocator", + "RecognizerIdentity", + "RecognizerPin", "XrefRepair", "assurance", "c14n", @@ -521,6 +526,23 @@ const PDF: &[&str] = &[ "exit", "exit_code", "extract", + // v3 (decision #120, docs/35-OCR-SCOPE.md): `--ocr`. The stage that reads scanned pages + // through a recognizer, the trait a recognizer implements, and the protocol's shapes, which + // the CLI's client speaks and a test's recognizer answers in. + "extract_recognized", + "extract_image", + "image_media_type", + "Recognize", + "RecognitionRequest", + "Recognition", + "RecognizedLine", + "RecognizedWord", + "RecognizedSource", + "ImageSpec", + "PdfImageSpec", + "Colour", + "Base", + "FlateParms", "limitations", "to_representation", ]; diff --git a/crates/ethos-parser-core/src/assurance.rs b/crates/ethos-parser-core/src/assurance.rs index 1312f5b..48c9413 100644 --- a/crates/ethos-parser-core/src/assurance.rs +++ b/crates/ethos-parser-core/src/assurance.rs @@ -250,6 +250,19 @@ pub mod codes { /// `q /Xf1 Do Q` emits zero nodes with `pages_failed: 0`, and nothing distinguishes it from a /// blank page. pub const FORM_XOBJECTS_NOT_DESCENDED: &str = "form-xobjects-not-descended"; + /// A page that showed no text and placed an image was not read by the recognizer the profile + /// pins (v3, `docs/35-OCR-SCOPE.md` §7). Page-scoped, its reason named in the detail. + pub const SCANNED_PAGE_NOT_RECOGNIZED: &str = "scanned-page-not-recognized"; + /// Recognized words left unwritten because the page's visible box does not show where they lie. + pub const RECOGNIZED_WORDS_OUTSIDE_VISIBLE_BOX: &str = "recognized-words-outside-visible-box"; + /// Under the OCR profile: text turned within an upright image is not turned before it is read. + pub const RECOGNIZED_TEXT_ORIENTATION_NOT_CORRECTED: &str = + "recognized-text-orientation-not-corrected"; + /// Under the OCR profile: a recognized run has no font, so no emphasis and no font size. + pub const RECOGNIZED_TEXT_HAS_NO_FONT: &str = "recognized-text-has-no-font"; + /// An image file read as one page that is its pixel grid, one pixel to one point (v3, + /// `docs/35-OCR-SCOPE.md` §4): the resolution it declares is not read. + pub const IMAGE_RESOLUTION_NOT_READ: &str = "image-resolution-not-read"; /// Marked-content ids opened inside form XObjects bind nothing /// (`docs/30-FORM-XOBJECTS-SCOPE.md` §7). /// diff --git a/crates/ethos-parser-core/src/lib.rs b/crates/ethos-parser-core/src/lib.rs index 612eedc..a0d1d03 100644 --- a/crates/ethos-parser-core/src/lib.rs +++ b/crates/ethos-parser-core/src/lib.rs @@ -121,13 +121,14 @@ pub use markdown::{ }; pub use outlines::OutlineRecord; pub use profile::{ - profile_sha256, BackendIdentity, Capabilities, PageBudget, Profile, RasterDpi, TableDetection, - VerifierPin, XrefRepair, CMAP_DATA_VERSION, DOCX_READING_ORDER_RULE_V2, DOCX_TEXT_CODE_RULE_V3, - EPUB_READING_ORDER_RULE_V1, EPUB_TEXT_CODE_RULE_V2, FIGURE_RULE_V1, FORM_ANNOTATION_RULE_V2, - FURNITURE_RULE_V1, HEADING_INFERENCE_RULE_V4, HEADING_INFERENCE_RULE_V5, - HEADING_INFERENCE_RULE_V6, HEADING_INFERENCE_RULE_V7, LAYOUT_UNIT_RULE_V1, LAYOUT_UNIT_RULE_V2, - LAYOUT_UNIT_RULE_V3, LAYOUT_UNIT_RULE_V4, LAYOUT_UNIT_RULE_V5, LAYOUT_UNIT_RULE_V6, NOT_RUN, - OBSERVATION_RULE_V3, OBSERVATION_RULE_V4, ODP_READING_ORDER_RULE_V1, ODP_TEXT_CODE_RULE_V2, + profile_sha256, BackendIdentity, Capabilities, PageBudget, Profile, RasterDpi, + RecognizerIdentity, RecognizerPin, TableDetection, VerifierPin, XrefRepair, CMAP_DATA_VERSION, + DOCX_READING_ORDER_RULE_V2, DOCX_TEXT_CODE_RULE_V3, EPUB_READING_ORDER_RULE_V1, + EPUB_TEXT_CODE_RULE_V2, FIGURE_RULE_V1, FORM_ANNOTATION_RULE_V2, FURNITURE_RULE_V1, + HEADING_INFERENCE_RULE_V4, HEADING_INFERENCE_RULE_V5, HEADING_INFERENCE_RULE_V6, + HEADING_INFERENCE_RULE_V7, LAYOUT_UNIT_RULE_V1, LAYOUT_UNIT_RULE_V2, LAYOUT_UNIT_RULE_V3, + LAYOUT_UNIT_RULE_V4, LAYOUT_UNIT_RULE_V5, LAYOUT_UNIT_RULE_V6, NOT_RUN, OBSERVATION_RULE_V3, + OBSERVATION_RULE_V4, ODP_READING_ORDER_RULE_V1, ODP_TEXT_CODE_RULE_V2, ODS_READING_ORDER_RULE_V1, ODS_TEXT_CODE_RULE_V2, ODT_READING_ORDER_RULE_V1, ODT_TEXT_CODE_RULE_V2, PPTX_READING_ORDER_RULE_V1, PPTX_TEXT_CODE_RULE_V3, READING_ORDER_RULE_V0, READING_ORDER_RULE_V1, READING_ORDER_RULE_V2, READING_ORDER_RULE_V3, @@ -166,9 +167,9 @@ pub use representation::{ OfficeParagraphAttributes, OfficeRunAttributes, OfficeSlideRunAttributes, PageRecord, PaintedRect, PdfArtifactLocator, PdfImageLocator, PdfLocator, PdfObjectLocator, PdfObjectNumber, PdfTaggedLocator, PptxLocator, ProcessingRun, ProcessorIdentity, Reading, - RepresentationPayload, RtfLocator, RtfParagraphAttributes, RtfParagraphBreak, Script, - SourceIdentity, StructuralLocator, SynthesizedAt, TextFinding, TextRunAttributes, XlsxLocator, - REPRESENTATION_ARTIFACT_TYPE, REPRESENTATION_SCHEMA_VERSION, + RecognizedLocator, RepresentationPayload, RtfLocator, RtfParagraphAttributes, + RtfParagraphBreak, Script, SourceIdentity, StructuralLocator, SynthesizedAt, TextFinding, + TextRunAttributes, XlsxLocator, REPRESENTATION_ARTIFACT_TYPE, REPRESENTATION_SCHEMA_VERSION, }; /// The crate name, asserted by the M0 harness to prove the workspace links. diff --git a/crates/ethos-parser-core/src/markdown.rs b/crates/ethos-parser-core/src/markdown.rs index 08875aa..a2376b3 100644 --- a/crates/ethos-parser-core/src/markdown.rs +++ b/crates/ethos-parser-core/src/markdown.rs @@ -1168,8 +1168,7 @@ pub(crate) fn overprinted_runs<'a>( let mut read: Vec<(&str, bool, bool)> = Vec::new(); let mut page = None; for (k, node) in nodes.iter().enumerate() { - let (Some(a), crate::NativeLocator::Pdf(at)) = - (text_run_attributes(node), &node.native_locator) + let (Some(a), Some(at)) = (text_run_attributes(node), node.native_locator.run_locator()) else { continue; }; @@ -1177,10 +1176,11 @@ pub(crate) fn overprinted_runs<'a>( continue; }; let (font, size, x, y) = (a.font_id.as_str(), a.font_size, at.origin_x, at.origin_y); - let step = match nodes.get(k + 1).map(|n| &n.native_locator) { - Some(crate::NativeLocator::Pdf(next)) if next.origin_y == y => { - next.origin_x.saturating_sub(x) - } + let step = match nodes + .get(k + 1) + .and_then(|n| n.native_locator.run_locator()) + { + Some(next) if next.origin_y == y => next.origin_x.saturating_sub(x), _ => stated, }; let advance = if step.saturating_mul(3) >= stated { @@ -1342,8 +1342,11 @@ pub(crate) fn heading_number_continues( .filter(|a| a.inferred_heading) .map(|a| a.inferred_heading_level) }; - let drawn_after = match (&prev.native_locator, &node.native_locator) { - (crate::NativeLocator::Pdf(a), crate::NativeLocator::Pdf(b)) => b.origin_x > a.origin_x, + let drawn_after = match ( + prev.native_locator.run_locator(), + node.native_locator.run_locator(), + ) { + (Some(a), Some(b)) => b.origin_x > a.origin_x, _ => false, }; let one_line = level(prev).is_some() @@ -1352,10 +1355,10 @@ pub(crate) fn heading_number_continues( && line_key(prev) == line_key(node) && drawn_after; let bold = |n: &crate::Node| text_run_attributes(n).is_some_and(|a| a.bold); - let origin = match &node.native_locator { - crate::NativeLocator::Pdf(b) => Some((b.origin_x, b.origin_y)), - _ => None, - }; + let origin = node + .native_locator + .run_locator() + .map(|b| (b.origin_x, b.origin_y)); let beside = (level(prev).is_some() || bold(prev)) && (level(prev).is_some() || level(node).is_some() || bold(node)) && prev_box @@ -1398,9 +1401,10 @@ pub(crate) fn unit_gap_continues( pitch: &PitchReference, ) -> bool { let unit = |n: &crate::Node| text_run_attributes(n).and_then(|a| a.layout_unit); - let (crate::NativeLocator::Pdf(a), crate::NativeLocator::Pdf(b)) = - (&prev.native_locator, &node.native_locator) - else { + let (Some(a), Some(b)) = ( + prev.native_locator.run_locator(), + node.native_locator.run_locator(), + ) else { return false; }; let along = prev.parent == node.parent @@ -1791,9 +1795,10 @@ pub(crate) fn sequence_join( if open.element.is_some() && open.element == key.element { let measure = text_run_attributes(prev).and_then(|t| pitch.get(&(t.font_id.clone(), t.font_size)))?; - let (crate::NativeLocator::Pdf(a), crate::NativeLocator::Pdf(b)) = - (&prev.native_locator, &node.native_locator) - else { + let (Some(a), Some(b)) = ( + prev.native_locator.run_locator(), + node.native_locator.run_locator(), + ) else { return None; }; if line_key(prev).is_some_and(|line| same_line(line, node, *measure)) { @@ -1854,9 +1859,7 @@ fn same_parent(a: &crate::Node, b: &crate::Node) -> bool { /// could not say so. Author structure is untouched: an `Extracted` locator keys exactly as it /// has since v2.2-S1. pub(crate) fn group_key(node: &crate::Node) -> Option { - let crate::NativeLocator::Pdf(loc) = &node.native_locator else { - return None; - }; + let loc = node.native_locator.run_locator()?; let (kind, mcid, element) = match node.structural_locator.as_ref()? { // No projection rule id moves for this arm: no input that existed before it projects // differently — every locator a reader minted before auto-tagging is `Extracted` — and @@ -1929,9 +1932,7 @@ pub(crate) fn same_line(line: LineKey, node: &crate::Node, measure: FontMeasure) } pub(crate) fn line_key(node: &crate::Node) -> Option { - let crate::NativeLocator::Pdf(loc) = &node.native_locator else { - return None; - }; + let loc = node.native_locator.run_locator()?; Some(LineKey { page: loc.page, region: region_of(node), @@ -1981,8 +1982,7 @@ pub(crate) fn pitch_reference(nodes: &[crate::Node]) -> PitchReference { let mut per_font_space: std::collections::HashMap<(String, i64), Vec> = std::collections::HashMap::new(); for node in nodes { - let (Some(a), crate::NativeLocator::Pdf(loc)) = - (text_run_attributes(node), &node.native_locator) + let (Some(a), Some(loc)) = (text_run_attributes(node), node.native_locator.run_locator()) else { continue; }; @@ -2077,9 +2077,7 @@ pub(crate) fn pitch_reference(nodes: &[crate::Node]) -> PitchReference { /// tolerance above, one glyph of permitted overlap below. The `AC` cell-pitch run above is /// refused by a factor of eighteen and is nowhere near the widened bound. pub(crate) fn ink_reach(node: &crate::Node, pitch: &PitchReference) -> Option<(i64, FontMeasure)> { - let crate::NativeLocator::Pdf(loc) = &node.native_locator else { - return None; - }; + let loc = node.native_locator.run_locator()?; let advance = loc.advance?; if advance < 0 { return None; @@ -2134,9 +2132,10 @@ pub(crate) fn ink_sequenced( a_measure: FontMeasure, b: &crate::Node, ) -> bool { - let (crate::NativeLocator::Pdf(x), crate::NativeLocator::Pdf(y)) = - (&a.native_locator, &b.native_locator) - else { + let (Some(x), Some(y)) = ( + a.native_locator.run_locator(), + b.native_locator.run_locator(), + ) else { return false; }; // 1. Drawn after, not over. `dx == 0` is exact overprint and `dx < 0` a backward jump; both @@ -2235,8 +2234,8 @@ pub(crate) fn read_stretches<'a>( }) .map_or(nodes.len(), |n| start + n); let mut drawn: Vec<&crate::Node> = nodes[start..end].iter().collect(); - drawn.sort_by_key(|n| match &n.native_locator { - crate::NativeLocator::Pdf(loc) => loc.origin_x, + drawn.sort_by_key(|n| match n.native_locator.run_locator() { + Some(loc) => loc.origin_x, _ => 0, }); let mut line_ink: Option<(&crate::Node, i64, FontMeasure)> = None; @@ -2300,9 +2299,10 @@ pub(crate) const INK_EPSILON_CENTIPOINTS: i64 = 12; /// space is 243 — and every such pair opened a block mid-word: 22 116 pairs on 102 of that /// corpus's 506 text documents. pub(crate) fn ink_contiguous(a: &crate::Node, b: &crate::Node, pitch: &PitchReference) -> bool { - let (crate::NativeLocator::Pdf(x), crate::NativeLocator::Pdf(y)) = - (&a.native_locator, &b.native_locator) - else { + let (Some(x), Some(y)) = ( + a.native_locator.run_locator(), + b.native_locator.run_locator(), + ) else { return false; }; let Some(advance) = x.advance else { @@ -2335,8 +2335,11 @@ pub(crate) fn ink_contiguous(a: &crate::Node, b: &crate::Node, pitch: &PitchRefe /// page drew; a wrong join invents one. Given `docs/01-CONTRACT.md`'s posture on fabrication, those /// are not comparable costs. pub(crate) fn on_different_lines(a: &crate::Node, b: &crate::Node) -> bool { - match (&a.native_locator, &b.native_locator) { - (crate::NativeLocator::Pdf(x), crate::NativeLocator::Pdf(y)) => x.origin_y != y.origin_y, + match ( + a.native_locator.run_locator(), + b.native_locator.run_locator(), + ) { + (Some(x), Some(y)) => x.origin_y != y.origin_y, _ => false, } } @@ -3838,9 +3841,10 @@ pub(crate) fn cell_runs_abut( && !prev.reading_text().ends_with(char::is_whitespace) && !node.reading_text().starts_with(char::is_whitespace); } - let (crate::NativeLocator::Pdf(x), crate::NativeLocator::Pdf(y)) = - (&prev.native_locator, &node.native_locator) - else { + let (Some(x), Some(y)) = ( + prev.native_locator.run_locator(), + node.native_locator.run_locator(), + ) else { return false; }; y.origin_x > x.origin_x diff --git a/crates/ethos-parser-core/src/profile.rs b/crates/ethos-parser-core/src/profile.rs index 99e1c23..50c7358 100644 --- a/crates/ethos-parser-core/src/profile.rs +++ b/crates/ethos-parser-core/src/profile.rs @@ -2772,6 +2772,74 @@ pub struct Profile { pub xref_repair: XrefRepair, /// Which verifier this run is bound to. New at v0.1, [`VerifierPin::NotPinned`] by default. pub verifier: VerifierPin, + /// Which recognizer this run's `Recognized` runs come from (v3, `docs/35-OCR-SCOPE.md`). + /// [`RecognizerPin::NotPinned`] by default; [`Profile::ocr_v0`] pins one. + pub recognizer: RecognizerPin, +} + +/// Which recognizer a profile's `Recognized` runs come from, if any (v3, `docs/35-OCR-SCOPE.md` +/// §5). +/// +/// [`RecognizerPin::NotPinned`] is the default and a real statement rather than a missing field: +/// *no recognizer read any pixels for this run.* Every artifact made without `--ocr` carries it, +/// and the OCR profile differs from the base one here and nowhere else (contract §6, rule 4). +/// +/// **A recognizer is pinned by the identity it reports, not by its binary's digest.** Unlike +/// [`VerifierPin`], whose binary is the one whose bytes answered, the recognizer is built per +/// platform and gives the same bytes on every one of them (`docs/35-OCR-SCOPE.md` §10.1), so a +/// binary digest would make the same document's artifact differ between machines that read it +/// identically. Its identity names what decides its output — its version, its pipeline rule, the +/// digests of the models it holds, and the runtime — and the engine refuses an answer from a +/// recognizer reporting any other, so one profile never holds two recognizers' output. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +// Adjacently tagged, as `VerifierPin` is. +#[serde( + rename_all = "snake_case", + tag = "mode", + content = "identity", + deny_unknown_fields +)] +pub enum RecognizerPin { + /// No recognizer was consulted. + NotPinned, + /// This recognizer, and only it, read the pages this profile's artifacts recognize. + Pinned(RecognizerIdentity), +} + +/// A recognizer's identity, as `ethos-parser-ocr identity` reports it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecognizerIdentity { + /// The recognizer's name: `ethos-parser-ocr`. + pub recognizer: String, + /// Its version. + pub version: String, + /// The rule id of every step around its models; moves whenever one changes what it writes. + pub rule: String, + /// SHA-256 of the detection model it holds. + pub det_sha256: String, + /// SHA-256 of the recognition model it holds. + pub rec_sha256: String, + /// SHA-256 of the recognition model's alphabet. + pub dict_sha256: String, + /// The runtime that runs the models. + pub runtime: String, +} + +impl RecognizerIdentity { + /// `ethos-parser-ocr` 0.1.0: PP-OCRv5 mobile on rten 0.26 with fixed-order sums (`ocr/`). The + /// strings are `ocr/src/main.rs`'s, and a test holds the two together. + pub fn ethos_parser_ocr() -> Self { + Self { + recognizer: "ethos-parser-ocr".into(), + version: "0.1.0".into(), + rule: "ppocrv5-mobile-v1".into(), + det_sha256: "4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae".into(), + rec_sha256: "c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8".into(), + dict_sha256: "e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6".into(), + runtime: "rten 0.26.0; rten-vecmath 0.26.0 with fixed-order sums".into(), + } + } } impl Default for Profile { @@ -2806,11 +2874,23 @@ impl Default for Profile { raster_dpi: RasterDpi::NotEmitted, xref_repair: XrefRepair::Pad19To20V1, verifier: VerifierPin::NotPinned, + recognizer: RecognizerPin::NotPinned, } } } impl Profile { + /// The profile a PDF is read under with `--ocr` (v3, `docs/35-OCR-SCOPE.md`): the base + /// profile with [`RecognizerIdentity::ethos_parser_ocr`] pinned, so its `profile_sha256` differs + /// and an artifact holding recognized text is never comparable with one that does not by + /// accident (contract §6, rule 4). + pub fn ocr_v0() -> Self { + Self { + recognizer: RecognizerPin::Pinned(RecognizerIdentity::ethos_parser_ocr()), + ..Self::default() + } + } + /// The profile a page-less OOXML word-processing document is read under (v2-S2). /// /// **A separate profile so the two artifacts are provably non-comparable.** `14-V2-SCOPE.md` @@ -3497,13 +3577,13 @@ mod tests { object.keys().map(String::as_str).collect() } - // Pinned, so the verifier's one struct variant is on the wire too. + // Pinned, so the verifier's and the recognizer's struct variants are on the wire too. let profile = Profile { verifier: VerifierPin::pinned( "ethos 0.6.0", Sha256Hex::from_hex(&"0".repeat(64)).expect("hex"), ), - ..Profile::default() + ..Profile::ocr_v0() }; let bytes = profile.canonical_bytes().expect("canonical"); let wire: serde_json::Value = serde_json::from_slice(&bytes).expect("JSON"); @@ -3540,6 +3620,7 @@ mod tests { raster_dpi, xref_repair, verifier, + recognizer, }) ); assert_eq!( @@ -3590,6 +3671,50 @@ mod tests { keys(&wire["verifier"]["identity"]), declared!(&profile.verifier => VerifierPin::Pinned { version, sha256 }) ); + let RecognizerPin::Pinned(identity) = &profile.recognizer else { + panic!("the OCR profile pins its recognizer"); + }; + assert_eq!( + keys(&wire["recognizer"]["identity"]), + declared!(identity => RecognizerIdentity { + recognizer, + version, + rule, + det_sha256, + rec_sha256, + dict_sha256, + runtime, + }) + ); + } + + /// The pinned recognizer is the one in `ocr/`: its name, version, rule, model digests and + /// runtime, read from its own source, so the identity the engine demands and the identity the + /// recognizer reports cannot drift apart unnoticed. They live in two workspaces on two + /// toolchains, so no type can hold them together; this test does. + #[test] + fn the_pinned_recognizer_is_the_one_in_ocr() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let main = std::fs::read_to_string(root.join("ocr/src/main.rs")).expect("ocr/src/main.rs"); + let manifest = + std::fs::read_to_string(root.join("ocr/Cargo.toml")).expect("ocr/Cargo.toml"); + let constant = |name: &str| -> String { + let line = main + .lines() + .find(|l| l.starts_with(&format!("pub const {name}: &str = \""))) + .unwrap_or_else(|| panic!("ocr/src/main.rs states no `{name}`")); + line.split('"').nth(1).expect("a quoted value").to_string() + }; + let id = RecognizerIdentity::ethos_parser_ocr(); + assert_eq!(id.rule, constant("RULE")); + assert_eq!(id.det_sha256, constant("DET_SHA256")); + assert_eq!(id.rec_sha256, constant("REC_SHA256")); + assert_eq!(id.dict_sha256, constant("DICT_SHA256")); + assert_eq!(id.runtime, constant("RUNTIME")); + assert!(main.contains(&format!("recognizer: \"{}\",", id.recognizer))); + assert!(manifest + .lines() + .any(|l| l == format!("version = \"{}\"", id.version))); } /// Every field on `Profile` that is mutated below changes the hash when it changes. @@ -3705,6 +3830,7 @@ mod tests { unicode_data_version: _, xref_repair: _, verifier: _, + recognizer: _, } = &base; /// One named single-field mutation. @@ -3828,6 +3954,14 @@ mod tests { ) }), ), + ( + // v3 (decision #120). Which recognizer read the pixels is part of the run's + // identity: an OCR artifact must never be comparable with a plain one by accident. + "recognizer", + Box::new(|p: &mut Profile| { + p.recognizer = RecognizerPin::Pinned(RecognizerIdentity::ethos_parser_ocr()) + }), + ), ( "backend.name", Box::new(|p: &mut Profile| p.backend.name = "other-backend".into()), @@ -3981,12 +4115,13 @@ mod tests { // `figure_rule` and `capabilities.figures` the same way, and forty-five since decision // #54, which added `right_to_left_rule` the same way. Forty-six since decision #66, which // added `table_detection.lines` the same way, and forty-seven since decision #90, which - // added `table_detection.pies` the same way. + // added `table_detection.pies` the same way. Forty-eight since decision #120, which added + // `recognizer` the same way. assert_eq!( mutations.len(), - 47, - "{} single-field mutation(s); forty-seven is the number at decision #90, which added \ - `table_detection.pies` with its mutation", + 48, + "{} single-field mutation(s); forty-eight is the number at decision #120, which added \ + `recognizer` with its mutation", mutations.len() ); @@ -4041,7 +4176,7 @@ mod tests { let bytes = Profile::default().canonical_bytes().unwrap(); assert_eq!( String::from_utf8(bytes).unwrap(), - r#"{"backend":{"components":{"flate2":"1.1.9","read-fonts":"0.36.0","skrifa":"0.39.0","unicode-bidi":"0.3.18"},"name":"lopdf","version":"0.44.0"},"capabilities":{"annotations":true,"char_offsets":true,"figures":true,"form_fields":true,"html":true,"images":true,"markdown":true,"measured_ink_boxes":true,"multi_column_reading_order":true,"outlines":true,"page_screenshots":false,"spans":true,"structural_locators":true,"tables":true},"classify_sample_pages":8,"cmap_data_version":"annex-d-encodings-5","coordinate_system":{"origin":"top-left","unit":"centipoint"},"figure_rule":"figure-regions-v1","font_metrics_data_version":"core14-afm-2","form_annotation_rule":"form-annotations-v2","furniture_rule":"margin-bands-v1","heading_inference_rule":"type-size-v7","html_rule":"html-blocks-v13","layout_unit_rule":"line-units-v6","locate_rule":"locate-scalar-exact-v1","markdown_rule":"markdown-blocks-v13","observation_rule":"page-observations-v4","outline_rule":"outlines-v3","page_budget":{"mode":"unlimited"},"parser_version":"0.66.0-dev.1","quantum_per_point":100,"raster_dpi":{"mode":"not_emitted"},"reading_order_rule":"gutter-columns-v4","right_to_left_rule":"right-to-left-lines-v1","struct_tree_rule":"struct-tree-v3","table_detection":{"charts":"bar-labels-v3","lines":"point-labels-v1","pies":"slice-labels-v1","ruled":"ruled-rects-v14","stroke_ruled":"stroke-ruled-v2","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v16","unruled":"unruled-align-v1"},"text_box_rule":"advance-over-font-envelope-v2","text_code_rule":"declared-font-codes-v4","unicode_data_version":"std-unicode-16.0.0","verifier":{"mode":"not_pinned"},"xref_repair":{"mode":"pad-19-to-20-v1"}}"#, + r#"{"backend":{"components":{"flate2":"1.1.9","read-fonts":"0.36.0","skrifa":"0.39.0","unicode-bidi":"0.3.18"},"name":"lopdf","version":"0.44.0"},"capabilities":{"annotations":true,"char_offsets":true,"figures":true,"form_fields":true,"html":true,"images":true,"markdown":true,"measured_ink_boxes":true,"multi_column_reading_order":true,"outlines":true,"page_screenshots":false,"spans":true,"structural_locators":true,"tables":true},"classify_sample_pages":8,"cmap_data_version":"annex-d-encodings-5","coordinate_system":{"origin":"top-left","unit":"centipoint"},"figure_rule":"figure-regions-v1","font_metrics_data_version":"core14-afm-2","form_annotation_rule":"form-annotations-v2","furniture_rule":"margin-bands-v1","heading_inference_rule":"type-size-v7","html_rule":"html-blocks-v13","layout_unit_rule":"line-units-v6","locate_rule":"locate-scalar-exact-v1","markdown_rule":"markdown-blocks-v13","observation_rule":"page-observations-v4","outline_rule":"outlines-v3","page_budget":{"mode":"unlimited"},"parser_version":"0.66.0-dev.1","quantum_per_point":100,"raster_dpi":{"mode":"not_emitted"},"reading_order_rule":"gutter-columns-v4","recognizer":{"mode":"not_pinned"},"right_to_left_rule":"right-to-left-lines-v1","struct_tree_rule":"struct-tree-v3","table_detection":{"charts":"bar-labels-v3","lines":"point-labels-v1","pies":"slice-labels-v1","ruled":"ruled-rects-v14","stroke_ruled":"stroke-ruled-v2","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v16","unruled":"unruled-align-v1"},"text_box_rule":"advance-over-font-envelope-v2","text_code_rule":"declared-font-codes-v4","unicode_data_version":"std-unicode-16.0.0","verifier":{"mode":"not_pinned"},"xref_repair":{"mode":"pad-19-to-20-v1"}}"#, "the v0 profile changed. Expected causes: a crate version bump (parser_version is \ part of identity, so a new build IS a new profile — that is by design), or a new \ field. Update this vector and say why in the commit. Unexpected cause: something \ @@ -5239,11 +5374,15 @@ mod tests { fill outlines as a polygon as the rectangles it outlines. Nothing else.\n\n\ Moved for decision #112, `sha256:c65b6986…` -> `sha256:0903edb3…`: \ `table_detection.tracks` `whitespace-tracks-v15` -> `-v16`, which reads a header cell \ - set over several columns as a group head spanning them. Nothing else." + set over several columns as a group head spanning them. Nothing else.\n\n\ + Moved for decision #120, `sha256:0903edb3…` -> `sha256:8c2f8672…`: the new \ + `recognizer` pin, `not_pinned` here and pinned only by the OCR profile \ + (`docs/35-OCR-SCOPE.md`), so an artifact holding recognized text is never \ + comparable with one that does not by accident. Nothing else." ); assert_eq!( Profile::default().profile_sha256().unwrap().to_string(), - "sha256:0903edb3a54b36e8fcaee98f7351321e0ac197727b072e7b6c25fb59134f59d0" + "sha256:8c2f86723df4d63209022e6545f24a029bc305d59d3f0bd3c049b672f4184d19" ); } diff --git a/crates/ethos-parser-core/src/representation.rs b/crates/ethos-parser-core/src/representation.rs index 3584ea0..271084d 100644 --- a/crates/ethos-parser-core/src/representation.rs +++ b/crates/ethos-parser-core/src/representation.rs @@ -105,7 +105,8 @@ pub struct SourceIdentity { /// This read *"Always `application/pdf` at v0"* until v2-S13.5, which was true of v0 and had /// stopped describing the field: `ethos-parser-office` writes eight more into it — the OOXML three, /// the ODF three, `application/rtf` and `application/epub+zip`. A caller must not treat this - /// as a constant. + /// as a constant. And since v3 (decision #120) `image/png` or `image/jpeg`: an image file read + /// with `--ocr` as one page, its own pixel grid. /// /// **Not to be confused with `ethos_parser_grounding::Source::media_type`**, which IS pinned to /// `application/pdf` by the `ethos.grounding.v1` schema and is correctly documented as such. @@ -272,6 +273,16 @@ pub enum NativeLocator { /// The address is the spine item's package path plus a position in that document. See /// [`EpubLocator`] for why the path comes from the **spine** and not from the ZIP. Epub(EpubLocator), + /// A recognized run's address: where on its page it lies, and where in an image a recognizer + /// read it (v3, `docs/35-OCR-SCOPE.md` §6). + /// + /// **A variant of its own rather than [`Self::Pdf`], because nothing here was drawn.** A glyph + /// run's origin and advance are the content stream's; a recognized word's are computed from the + /// box a recognizer read in an image's pixels, through the matrix the page painted the image + /// with. They are stated as a text layer would state them — see [`RecognizedLocator::run`] — + /// so every rule that reads a run's place reads this one's ([`Self::run_locator`]); and the + /// pixels the recognizer read are here too, so the claim can be checked against the image. + Recognized(RecognizedLocator), } impl NativeLocator { @@ -284,7 +295,7 @@ impl NativeLocator { /// page in it, which is a thing the reader either read or did not. pub fn is_paginated(&self) -> bool { match self { - Self::Pdf(_) | Self::PdfObject(_) | Self::PdfImage(_) => true, + Self::Pdf(_) | Self::PdfObject(_) | Self::PdfImage(_) | Self::Recognized(_) => true, // **Eight page-less variants, and each has its own reason.** The arm below carried // running ordinals — "the sharpest of the four", then "the sharpest of the six" a // dozen lines under it — which were true at v2-S5 and v2-S7 and had been wrong ever @@ -342,7 +353,7 @@ impl NativeLocator { /// The part this address lives in, for a page-less format. `None` when paginated. pub fn part(&self) -> Option<&str> { match self { - Self::Pdf(_) | Self::PdfObject(_) | Self::PdfImage(_) => None, + Self::Pdf(_) | Self::PdfObject(_) | Self::PdfImage(_) | Self::Recognized(_) => None, Self::Docx(d) => Some(d.part.as_str()), Self::Xlsx(x) => Some(x.part.as_str()), Self::Pptx(p) => Some(p.part.as_str()), @@ -367,7 +378,11 @@ impl NativeLocator { /// be a bijection between. pub fn names_a_part(&self) -> bool { match self { - Self::Pdf(_) | Self::PdfObject(_) | Self::PdfImage(_) | Self::Rtf(_) => false, + Self::Pdf(_) + | Self::PdfObject(_) + | Self::PdfImage(_) + | Self::Recognized(_) + | Self::Rtf(_) => false, Self::Docx(_) | Self::Xlsx(_) | Self::Pptx(_) @@ -380,6 +395,18 @@ impl NativeLocator { | Self::Epub(_) => true, } } + + /// A text run's place on its page — page, baseline origin, advance — whether the page drew it + /// ([`Self::Pdf`]) or a recognizer read it ([`Self::Recognized`]); `None` for every other + /// address. The one accessor the rules that join, order and measure runs read a place + /// through, so a recognized run takes part in them as a drawn one does. + pub fn run_locator(&self) -> Option<&PdfLocator> { + match self { + Self::Pdf(l) => Some(l), + Self::Recognized(r) => Some(&r.run), + _ => None, + } + } } /// A run's address inside an OOXML word-processing part (v2-S2). @@ -836,6 +863,23 @@ pub struct PdfObjectNumber { pub generation: u32, } +/// Where a recognized run lies (v3, `docs/35-OCR-SCOPE.md` §6): on its page, as a text layer would +/// state it, and in the image it was read from, as the recognizer reported it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecognizedLocator { + /// The run's place on its page: the foot of its line under the word's left edge as the + /// origin, and the word's width as the advance — the numbers a text layer written by a + /// recognizer states (Tesseract's, which Phase 0's measurements read). **Computed** from + /// [`Self::pixels`] through the matrix the page painted the image with, never drawn. + pub run: PdfLocator, + /// The image XObject the run was read from. `null` for an image file, whose page is the image. + pub image: Option, + /// The word's corners in the image's own pixel grid, as the recognizer read them: the + /// top-left, top-right, bottom-right and bottom-left of its text, in reading orientation. + pub pixels: [[i32; 2]; 4], +} + /// The `/Rect` an annotation or widget declares, or why there is none. /// /// **Deliberately not [`crate::derivation::GeometryPresence`].** That type means *measured ink*, @@ -2785,7 +2829,45 @@ impl DocumentRepresentation { self.check_cell_runs_are_declared_nodes(&seen_nodes)?; self.check_attributes_agree_with_kind()?; self.check_readings_move_characters()?; + self.check_recognized_placement()?; + + Ok(()) + } + /// Contract §6's placement rule, checked rather than trusted (`docs/35-OCR-SCOPE.md` §6). + /// + /// A recognized run is a text run of derivation `Recognized` under a + /// [`NativeLocator::Recognized`] address, and nothing else carries either; and no page holds + /// both a `Recognized` node and an `Extracted` text run. A recognizer authors only where the + /// deterministic reader found no text layer at all, and is never merged into one — the failure + /// `docs/07-VERIFY-BOUNDARY.md` §8.3 names. + fn check_recognized_placement(&self) -> Result<(), EngineError> { + let mut extracted_pages = std::collections::BTreeSet::new(); + for node in &self.representation.nodes { + let recognized = node.derivation == DerivationClass::Recognized; + let addressed = matches!(node.native_locator, NativeLocator::Recognized(_)); + if recognized != addressed || (recognized && node.kind != NodeKind::TextRun) { + return Err(Self::malformed(format!( + "node `{}`: a recognized run is a text run of derivation `recognized` under a \ + `recognized` address, and nothing else is either", + node.id + ))); + } + if node.kind == NodeKind::TextRun && node.derivation == DerivationClass::Extracted { + extracted_pages.insert(&node.parent); + } + } + for node in &self.representation.nodes { + if node.derivation == DerivationClass::Recognized + && extracted_pages.contains(&node.parent) + { + return Err(Self::malformed(format!( + "node `{}` is recognized text on a page holding extracted text; a recognizer \ + authors only where the page has no text layer at all (contract §6)", + node.id + ))); + } + } Ok(()) } @@ -3278,6 +3360,66 @@ mod tests { DocumentRepresentation::seal(p, geometry) } + /// Contract §6's placement rule is the seal's (`docs/35-OCR-SCOPE.md` §6): a recognized run + /// seals on a page of its own, and is refused beside extracted text, under a drawn run's + /// address, or as anything but a text run. + #[test] + fn recognized_text_seals_only_where_no_text_was_extracted() { + let recognized = |n: Node| Node { + derivation: DerivationClass::Recognized, + native_locator: NativeLocator::Recognized(RecognizedLocator { + run: n + .native_locator + .run_locator() + .expect("a run's place") + .clone(), + image: Some(PdfObjectNumber { + object: 4, + generation: 0, + }), + pixels: [[0, 0], [10, 0], [10, 5], [0, 5]], + }), + ..n + }; + let rows = |nodes: &[&Node]| -> Vec { + let r = QRect::new(7200, 4800, 8200, 7200).unwrap(); + nodes + .iter() + .map(|n| geom(n, GeometryPresence::Measured(r))) + .collect() + }; + let mut alloc = IdAllocator::new(Profile::default().profile_sha256().unwrap()); + let (p1, p2) = (page(&mut alloc, 1), page(&mut alloc, 2)); + let drawn = node(&mut alloc, &p1.id, 1, 1); + let read = recognized(node(&mut alloc, &p2.id, 1, 2)); + let g = rows(&[&drawn, &read]); + seal( + vec![drawn.clone(), read.clone()], + vec![p1.clone(), p2.clone()], + g, + ) + .expect("a recognized page beside a drawn one seals"); + + let beside = recognized(node(&mut alloc, &p1.id, 2, 1)); + let g = rows(&[&drawn, &beside]); + let e = seal(vec![drawn.clone(), beside], vec![p1.clone()], g) + .expect_err("recognized text on a page holding extracted text"); + assert!(e.to_string().contains("no text layer at all"), "{e}"); + + let laundered = Node { + derivation: DerivationClass::Extracted, + ..read.clone() + }; + let g = rows(&[&laundered]); + assert!(seal(vec![laundered], vec![p2.clone()], g).is_err()); + let unaddressed = Node { + native_locator: drawn.native_locator.clone(), + ..read + }; + let g = rows(&[&unaddressed]); + assert!(seal(vec![unaddressed], vec![p2], g).is_err()); + } + /// The spliced emit route and the full serialization are one byte stream — /// and a parsed artifact, whose cache is cold, equals its minted original. #[test] diff --git a/crates/ethos-parser-office/tests/erasure_counters.rs b/crates/ethos-parser-office/tests/erasure_counters.rs index 059ec7c..d0372a3 100644 --- a/crates/ethos-parser-office/tests/erasure_counters.rs +++ b/crates/ethos-parser-office/tests/erasure_counters.rs @@ -82,7 +82,7 @@ fn readers() -> Vec { const PDF_EXTRACT: &str = "../../ethos-parser-pdf/src/extract.rs"; /// The `ethos-parser-pdf` counters, which do not share `ethos-parser-office`'s naming. -const PDF_COUNTERS: [&str; 22] = [ +const PDF_COUNTERS: [&str; 23] = [ "unclaimed_tree_items", "mcids_unbound", "composite_fonts", @@ -138,6 +138,10 @@ const PDF_COUNTERS: [&str; 22] = [ // states, the count `unmapped-codes-read-from-font` names. Caught the same way, by the targeted // run that included this file. "font_read_runs", + // Decision #120 (`docs/35-OCR-SCOPE.md`). The recognized words whose centre falls outside + // their page's visible box, the count `recognized-words-outside-visible-box` names. Caught the + // same way, by the first full gate after it landed. + "recognized_outside", ]; /// An accumulation that cannot wrap: the crate's saturating fold, or `saturating_add` itself. diff --git a/crates/ethos-parser-pdf/src/content.rs b/crates/ethos-parser-pdf/src/content.rs index 2618c53..4bcd726 100644 --- a/crates/ethos-parser-pdf/src/content.rs +++ b/crates/ethos-parser-pdf/src/content.rs @@ -511,6 +511,11 @@ pub struct Interpreter<'a> { /// stream to digest, because its samples live in the content stream itself. Counted so the /// absence of a node for it is visible on the artifact. pub inline_images: u32, + /// Text-showing operators run on this page, a form's each time the page draws it — what + /// `classify` counts as `text_operators`, and the half of its `scanned` reason the OCR trigger + /// reads (`docs/35-OCR-SCOPE.md` §4). Counted before the operands are read, so a string this + /// reader cannot decode still counts: the page has a text layer, whatever it says. + pub text_operators: u32, /// The index of the operation [`Self::run_inner`] is dispatching, recorded before every /// dispatch and read by [`Self::show`] (auto-tagging S2). /// @@ -583,6 +588,7 @@ impl<'a> Interpreter<'a> { images: Vec::new(), unresolved_xobjects: 0, inline_images: 0, + text_operators: 0, xobjects: None, op_index: 0, forms: None, @@ -893,20 +899,24 @@ impl<'a> Interpreter<'a> { // --- text showing: all four, including the one pdf-inspector omits --- ShowText => { + self.text_operators = self.text_operators.saturating_add(1); let bytes = string_operand(operands, 0)?; self.show(&bytes, &[])?; } ShowTextAdjusted => { + self.text_operators = self.text_operators.saturating_add(1); let items = array_operand(operands, 0)?; self.show_adjusted(&items)?; } NextLineShowText => { + self.text_operators = self.text_operators.saturating_add(1); self.ts.next_line(); self.text_drawn_since_placement = false; let bytes = string_operand(operands, 0)?; self.show(&bytes, &[])?; } NextLineShowTextSpacing => { + self.text_operators = self.text_operators.saturating_add(1); // `aw ac string "` — sets word and char spacing, then behaves as `'`. self.ts.word_spacing = num(operands, 0)?; self.ts.char_spacing = num(operands, 1)?; diff --git a/crates/ethos-parser-pdf/src/extract.rs b/crates/ethos-parser-pdf/src/extract.rs index 2a044a6..0c68169 100644 --- a/crates/ethos-parser-pdf/src/extract.rs +++ b/crates/ethos-parser-pdf/src/extract.rs @@ -613,6 +613,14 @@ struct PageYield { /// that hole. Rewriting only emitted entities would close it and renumber /// every id after it. local_ids: IdAllocator, + /// v3 (`docs/35-OCR-SCOPE.md` §4): the page the OCR trigger took, on its first reading under a + /// profile that pins a recognizer. + candidate: Option, + /// v3: why a page the trigger would read was not, by name — the trigger's own refusal or the + /// recognizer's. Set only under a profile that pins a recognizer, on a page that showed no text. + ocr_refusal: Option, + /// v3: recognized words not written because no viewer shows where they lie. + recognized_outside: u32, } /// One page's whole extraction, exactly the body the sequential loop ran, with two @@ -634,6 +642,7 @@ fn extract_page( page_id: lopdf::ObjectId, content: &crate::budget::ContentBudget, limit: usize, + words: Option<&[crate::recognize::PageWord]>, ) -> Result, EngineError> { let mut alloc = IdAllocator::new(profile_sha256.clone()); let mut encoding_dropped_runs: u32 = 0; @@ -678,6 +687,9 @@ fn extract_page( // content order until the right-to-left rule reads it, before `reorder_page`. Never on a // `TextRun`. let mut code_chars: Vec> = Vec::new(); + // v3: the OCR trigger's answer for this page, decided after its images are read. + let mut candidate: Option = None; + let mut ocr_refusal: Option = None; // The gate: the document declares no author structure (see `no_author_structure`) — or, since // decision #94, its tree tags no heading — and the profile names the rule; any other id, // `not-run-for-this-format` included, runs nothing. @@ -971,6 +983,65 @@ fn extract_page( // computed from evidence the page supplies: the text rendering mode the content // stream set, and the visible box the page declares. findings: run_findings(shown.render_mode, origin_x, origin_y, &visible), + recognized: None, + }); + } + + // v3 (`docs/35-OCR-SCOPE.md` §6): a scanned page read again with the words its recognizer + // read, as runs of derivation `Recognized` — beside no drawn run, since the trigger took + // only a page that showed no text, and before every rule that reads runs, so reading + // order, lines, blocks, headings and tables read them as they read drawn ones. Their + // place is the one a text layer states: the line's foot under the word, the word's + // width, the line's height as the size. No font, no codes, no operation drew them. + for w in words.into_iter().flatten() { + op_indices.push(usize::MAX); + ems.push(Some(w.font_size).filter(|em| *em > 0)); + glyph_starts.push(w.origin_x); + ink_ends.push(Some(w.rect.x1())); + code_chars.push(vec![1; w.text.chars().count()]); + clipped.push(false); + runs.push(TextRun { + id: alloc.next(IdKind::Span)?, + text: w.text.clone(), + char_codes: Vec::new(), + scalar_code_mismatch: !w.text.is_empty(), + synthesized: Vec::new(), + font_id: String::new(), + font_size: w.font_size, + region: None, + block: None, + layout_unit: None, + inferred_heading: false, + inferred_heading_level: None, + bold: false, + italic: false, + weight: None, + script: None, + furniture: None, + reading: None, + locator: PdfLocator { + page: page_number, + origin_x: w.origin_x, + origin_y: w.origin_y, + advance: w.advance, + }, + // As a drawn run's, above: a word whose centre is on the page and whose box runs + // past its edge keeps its text, not a box the seal would refuse the document for. + geometry: if geom.contains(w.rect) { + ethos_parser_core::GeometryPresence::Measured(w.rect) + } else { + ethos_parser_core::GeometryPresence::Absent( + ethos_parser_core::GeometryAbsence::MeasuredOffPage, + ) + }, + mcid: None, + structural: None, + derivation: DerivationClass::Recognized, + findings: Vec::new(), + recognized: Some(crate::nodes::RecognizedSource { + image: w.image, + pixels: w.pixels, + }), }); } @@ -1801,6 +1872,36 @@ fn extract_page( }); } + // v3 (`docs/35-OCR-SCOPE.md` §4): the trigger, a reason code and never a score. A page + // that showed no text and placed an image is `classify`'s `scanned`; under a profile that + // pins a recognizer it is read when it placed exactly one image, and refused by name when + // it did not. Decided on the first reading only, and only under that profile, so no other + // artifact pays for it. + if words.is_none() + && matches!( + profile.recognizer, + ethos_parser_core::RecognizerPin::Pinned(_) + ) + && interp.text_operators == 0 + { + let placed = interp.images.len() + interp.inline_images as usize; + if interp.images.len() == 1 && interp.inline_images == 0 && images.len() == 1 { + let placement = &interp.images[0]; + candidate = Some(crate::recognize::Candidate { + page_id, + object: placement.object, + corners: placement.corners.map(|(x, y)| geom.to_top_left(x, y)), + visible: (visible.x0, visible.y0, visible.x1, visible.y1), + }); + } else if interp.inline_images > 0 { + ocr_refusal = Some("inline-image".into()); + } else if placed > 1 { + ocr_refusal = Some("several-images".into()); + } else if placed == 1 || interp.unresolved_xobjects > 0 { + ocr_refusal = Some("image-not-loaded".into()); + } + } + for run in &runs { for f in &run.findings { *findings_seen.entry(f.as_code()).or_insert(0) += 1; @@ -1862,6 +1963,9 @@ fn extract_page( findings_seen, font_limitations, local_ids: alloc, + candidate, + ocr_refusal, + recognized_outside: 0, })) } @@ -1943,6 +2047,150 @@ pub fn extract(doc: &Document, profile: &Profile) -> Result Result { + extract_counted( + doc, + profile, + &crate::budget::ContentBudget::default(), + Some(recognizer), + None, + ) + .map(|(artifact, _)| artifact) +} + +/// An image file's one page, read before the page existed: what [`extract_counted`] sets on it +/// and states as its source. +struct ImageFile { + media_type: &'static str, + sha256: Sha256Hex, + words: Vec, + outside: u32, +} + +/// An image file, PNG or JPEG, read under a profile that pins `recognizer` as a one-page document +/// (v3, `docs/35-OCR-SCOPE.md` §4): the recognizer reads the file whole, and its words are runs of +/// derivation `Recognized` on a page that is the image's own pixel grid, one pixel to one point. +/// The source is the file. No image node is written and no locator names an image object: the +/// page is the image. +/// +/// # Errors +/// +/// [`EngineError::Unsupported`] for bytes that are not a PNG or JPEG, under a profile that does not +/// pin `recognizer`, and for an image the recognizer refuses to read — an image file has no other +/// page to count the refusal against; what `recognizer` returns when it fails. +pub fn extract_image( + bytes: &[u8], + profile: &Profile, + recognizer: &mut dyn crate::recognize::Recognize, +) -> Result { + let media_type = + crate::recognize::image_media_type(bytes).ok_or_else(|| EngineError::Unsupported { + what: "media type".into(), + detail: "an image file is read when it is a PNG or a JPEG, and these bytes state \ + neither" + .into(), + })?; + let pinned = pinned_recognizer(profile, Some(recognizer.identity()))?; + let answer = recognizer.read(&crate::recognize::RecognitionRequest { + ocr_request: 1, + id: "image".into(), + image: crate::recognize::ImageSpec::File, + orientation: 0, + bytes: bytes.len() as u64, + data: bytes.to_vec(), + })?; + if Some(&answer.identity) != pinned.as_ref() { + return Err(EngineError::Unsupported { + what: "recognizer".into(), + detail: format!( + "the image was answered by `{} {}`, not the recognizer the profile pins", + answer.identity.recognizer, answer.identity.version + ), + }); + } + if let Some(why) = &answer.refused { + return Err(EngineError::Unsupported { + what: "image".into(), + detail: format!("the recognizer did not read this {media_type}: `{why}`"), + }); + } + let doc = Document::open_bytes( + &crate::recognize::blank_page(answer.width, answer.height)?, + profile, + )?; + let page_id = doc + .pages() + .first() + .map(|&(_, id)| id) + .ok_or_else(|| EngineError::Malformed { + what: "image".into(), + detail: "the page an image file is set on has no page".into(), + })?; + let place = crate::recognize::whole_page(page_id, answer.width, answer.height); + let (words, outside) = crate::recognize::place(&place, &answer, None)?; + let file = ImageFile { + media_type, + sha256: Sha256Hex::parse(format!( + "sha256:{}", + ethos_parser_core::c14n::sha256_hex_bytes(bytes) + ))?, + words, + outside, + }; + extract_counted( + &doc, + profile, + &crate::budget::ContentBudget::default(), + Some(recognizer), + Some(file), + ) + .map(|(artifact, _)| artifact) +} + +/// The recognizer `profile` pins, checked against the one given (v3, `docs/35-OCR-SCOPE.md` §5): +/// an artifact whose profile names one recognizer and whose runs another read would be the +/// laundering the pin exists to prevent. +fn pinned_recognizer( + profile: &Profile, + given: Option<ðos_parser_core::RecognizerIdentity>, +) -> Result, EngineError> { + match (&profile.recognizer, given) { + (ethos_parser_core::RecognizerPin::NotPinned, None) => Ok(None), + (ethos_parser_core::RecognizerPin::Pinned(id), Some(r)) if r == id => Ok(Some(id.clone())), + (ethos_parser_core::RecognizerPin::Pinned(id), _) => Err(EngineError::Unsupported { + what: "recognizer".into(), + detail: format!( + "the profile reads scanned pages through `{} {}` (`{}`), and {}", + id.recognizer, + id.version, + id.rule, + if given.is_some() { + "another recognizer answered" + } else { + "none was given" + } + ), + }), + (ethos_parser_core::RecognizerPin::NotPinned, Some(_)) => Err(EngineError::Unsupported { + what: "recognizer".into(), + detail: "a recognizer was given to a profile that pins none".into(), + }), + } +} + /// [`extract`], returning beside the artifact where each run came from in its page's content /// stream (auto-tagging S2). /// @@ -1956,16 +2204,25 @@ pub(crate) fn extract_with_positions( doc: &Document, profile: &Profile, ) -> Result<(ExtractArtifact, RunPositions), EngineError> { - extract_counted(doc, profile, &crate::budget::ContentBudget::default()) + extract_counted( + doc, + profile, + &crate::budget::ContentBudget::default(), + None, + None, + ) } /// [`extract_with_positions`], charging `content` for the pages it reads: one budget per reading, -/// which a test can hand in already spent. +/// which a test can hand in already spent. `image_file` is set by [`extract_image`] alone. fn extract_counted( doc: &Document, profile: &Profile, content: &crate::budget::ContentBudget, + recognizer: Option<&mut dyn crate::recognize::Recognize>, + image_file: Option, ) -> Result<(ExtractArtifact, RunPositions), EngineError> { + let pinned = pinned_recognizer(profile, recognizer.as_ref().map(|r| r.identity()))?; let profile_sha256 = profile .profile_sha256() .map_err(|e| EngineError::Malformed { @@ -1992,6 +2249,10 @@ fn extract_counted( let mut doc_em_tally = crate::headings::EmTally::default(); let mut positions = RunPositions::new(); let mut limitations = lim::extract_limitations(); + // v3: what a recognized run does not carry, on every artifact made under the OCR profile. + if pinned.is_some() { + limitations.extend(lim::recognized_text_limitations()); + } // A repaired open is never silent: every artifact derived from one says so. if let Some(padded) = doc.xref_entries_padded() { limitations.push(lim::xref_entry_padded(padded)); @@ -2059,6 +2320,7 @@ fn extract_counted( let mut inline_images: u32 = 0; let mut unresolved_xobjects: u32 = 0; let mut undescended_xobjects: u32 = 0; + let mut recognized_outside: u32 = 0; let mut form_mcids: u32 = 0; let mut findings_seen: std::collections::BTreeMap<&'static str, u32> = std::collections::BTreeMap::new(); @@ -2143,7 +2405,7 @@ fn extract_counted( // object graph, built in `Document::open_bytes` before this function runs — a cost no budget // consulted here can lower. See `docs/measurements/memory-ceiling/` §5 and §15. use rayon::prelude::*; - let read = |page_number, page_id, limit| { + let read = |page_number, page_id, limit, words: Option<&[crate::recognize::PageWord]>| { extract_page( doc, profile, @@ -2154,6 +2416,7 @@ fn extract_counted( page_id, content, limit, + words, ) }; // A page denser than `HEAVY_PAGE_OPERATIONS` is left by the parallel read and read below, alone @@ -2166,7 +2429,12 @@ fn extract_counted( if !budget.admits(page_number) { return (page_number, page_id, None); } - let outcome = read(page_number, page_id, crate::budget::HEAVY_PAGE_OPERATIONS); + let outcome = read( + page_number, + page_id, + crate::budget::HEAVY_PAGE_OPERATIONS, + None, + ); (page_number, page_id, Some(outcome)) }) .collect::>() @@ -2174,8 +2442,13 @@ fn extract_counted( .map(|(page_number, page_id, outcome)| { let outcome = outcome.map(|result| match result { Ok(Some(page)) => Ok(page), - Ok(None) => read(page_number, page_id, crate::budget::MAX_PAGE_OPERATIONS)? - .ok_or_else(|| crate::budget::page_operations_past_ceiling(page_number)), + Ok(None) => read( + page_number, + page_id, + crate::budget::MAX_PAGE_OPERATIONS, + None, + )? + .ok_or_else(|| crate::budget::page_operations_past_ceiling(page_number)), Err(e) => Err(e), }); (page_number, outcome) @@ -2189,6 +2462,83 @@ fn extract_counted( doc.tounicode().refusal()?; content.refusal()?; + // v3 (`docs/35-OCR-SCOPE.md` §5): every page the trigger took, asked of the recognizer in page + // order — one at a time, because it uses every core itself — and read again with its words. + let mut outcomes = outcomes; + if let (Some(recognizer), Some(pinned)) = (recognizer, &pinned) { + for (page_number, outcome) in &mut outcomes { + let Some(Ok(y)) = outcome else { + continue; + }; + let Some(candidate) = y.candidate.take() else { + continue; + }; + let request = match crate::recognize::request(doc.inner(), &candidate, *page_number) { + Ok(r) => r, + Err(why) => { + y.ocr_refusal = Some(why); + continue; + } + }; + let answer = recognizer.read(&request)?; + if &answer.identity != pinned { + return Err(EngineError::Unsupported { + what: "recognizer".into(), + detail: format!( + "page {page_number} was answered by `{} {}`, not the pinned `{} {}`", + answer.identity.recognizer, + answer.identity.version, + pinned.recognizer, + pinned.version + ), + }); + } + if let Some(why) = answer.refused.clone() { + y.ocr_refusal = Some(why); + continue; + } + if let crate::recognize::ImageSpec::Pdf(spec) = &request.image { + if (answer.width, answer.height) != (spec.width, spec.height) { + y.ocr_refusal = Some("image-size-mismatch".into()); + continue; + } + } + let image = ethos_parser_core::PdfObjectNumber { + object: candidate.object.0, + generation: u32::from(candidate.object.1), + }; + let (words, outside) = crate::recognize::place(&candidate, &answer, Some(image))?; + if !words.is_empty() { + let again = read( + *page_number, + candidate.page_id, + crate::budget::MAX_PAGE_OPERATIONS, + Some(&words), + )? + .ok_or_else(|| crate::budget::page_operations_past_ceiling(*page_number))?; + *y = again; + } + y.recognized_outside = declare(y.recognized_outside, outside); + } + } + // An image file's one page, which was read before it was made ([`extract_image`]). + if let Some(file) = &image_file { + if let (Some((page_number, Some(Ok(y)))), Some(&(_, page_id))) = + (outcomes.first_mut(), doc.pages().first()) + { + if !file.words.is_empty() { + *y = read( + *page_number, + page_id, + crate::budget::MAX_PAGE_OPERATIONS, + Some(&file.words), + )? + .ok_or_else(|| crate::budget::page_operations_past_ceiling(*page_number))?; + } + y.recognized_outside = declare(y.recognized_outside, file.outside); + } + } + for (page_number, outcome) in outcomes { let Some(result) = outcome else { page_states.push(PageStateEntry { @@ -2335,6 +2685,10 @@ fn extract_counted( inline_images = declare(inline_images, y.inline_images); unresolved_xobjects = declare(unresolved_xobjects, y.unresolved_xobjects); undescended_xobjects = declare(undescended_xobjects, y.undescended_xobjects); + recognized_outside = declare(recognized_outside, y.recognized_outside); + if let Some(why) = &y.ocr_refusal { + limitations.push(lim::scanned_page_not_recognized(page_number, why)); + } form_mcids = declare(form_mcids, y.form_mcids); if y.undescended_xobjects > 0 { limitations.push(lim::form_xobjects_not_descended_on_page( @@ -2595,6 +2949,14 @@ fn extract_counted( if undescended_xobjects > 0 { limitations.push(lim::form_xobjects_not_descended(undescended_xobjects)); } + if recognized_outside > 0 { + limitations.push(lim::recognized_words_outside_visible_box( + recognized_outside, + )); + } + if image_file.is_some() { + limitations.push(lim::image_resolution_not_read()); + } if form_mcids > 0 { limitations.push(lim::form_xobject_mcids_not_bound(form_mcids)); } @@ -2655,9 +3017,15 @@ fn extract_counted( parser_version: profile.parser_version.clone(), profile_sha256, }, - source: SourceRef { - media_type: "application/pdf".to_string(), - sha256: Sha256Hex::parse(doc.source_sha256().as_str())?, + source: match image_file { + Some(file) => SourceRef { + media_type: file.media_type.to_string(), + sha256: file.sha256, + }, + None => SourceRef { + media_type: "application/pdf".to_string(), + sha256: Sha256Hex::parse(doc.source_sha256().as_str())?, + }, }, reading_order_rule: profile.reading_order_rule.clone(), page_count, @@ -3922,6 +4290,7 @@ mod tests { structural: None, derivation: DerivationClass::Extracted, findings: Vec::new(), + recognized: None, }) .collect(); @@ -4335,14 +4704,21 @@ mod tests { let profile = Profile::default(); let doc = Document::open_bytes(&two_pages(45), &profile).expect("opens"); - let e = extract_counted(&doc, &profile, &crate::budget::ContentBudget::default()) - .expect_err("page 1 refuses its rotation"); + let e = extract_counted( + &doc, + &profile, + &crate::budget::ContentBudget::default(), + None, + None, + ) + .expect_err("page 1 refuses its rotation"); assert!(e.to_string().contains("/Rotate"), "{e}"); let spent = crate::budget::ContentBudget::default(); let past = usize::try_from(crate::budget::MAX_DOCUMENT_OPERATIONS).expect("64-bit") + 1; assert!(spent.charge(0, past).is_err()); - let e = extract_counted(&doc, &profile, &spent).expect_err("the budget is spent"); + let e = + extract_counted(&doc, &profile, &spent, None, None).expect_err("the budget is spent"); assert!( matches!(&e, EngineError::ResourceLimit { limit, .. } if limit == "content operations in the document's pages"), @@ -4353,7 +4729,8 @@ mod tests { let doc = Document::open_bytes(&two_pages(0), &profile).expect("opens"); let full = crate::budget::ContentBudget::default(); full.charge(0, past - 1).expect("exactly at the ceiling"); - let e = extract_counted(&doc, &profile, &full).expect_err("the pages tip it over"); + let e = + extract_counted(&doc, &profile, &full, None, None).expect_err("the pages tip it over"); assert!( matches!(&e, EngineError::ResourceLimit { limit, .. } if limit == "content operations in the document's pages"), diff --git a/crates/ethos-parser-pdf/src/lib.rs b/crates/ethos-parser-pdf/src/lib.rs index c102bc8..6943299 100644 --- a/crates/ethos-parser-pdf/src/lib.rs +++ b/crates/ethos-parser-pdf/src/lib.rs @@ -92,6 +92,7 @@ pub(crate) mod pdfdoc; pub(crate) mod pies; pub(crate) mod reading_order; pub(crate) mod reasons; +pub(crate) mod recognize; pub(crate) mod represent; pub(crate) mod stroke_ruled; pub(crate) mod structure; @@ -116,11 +117,21 @@ pub use classify::{ CLASSIFICATION_SCHEMA_VERSION, }; pub use document::Document; -pub use extract::{extract, ExtractArtifact, EXTRACT_ARTIFACT_TYPE, EXTRACT_SCHEMA_VERSION}; +pub use extract::{ + extract, extract_image, extract_recognized, ExtractArtifact, EXTRACT_ARTIFACT_TYPE, + EXTRACT_SCHEMA_VERSION, +}; pub use magic::{aims_at_the_pdf_reader, check_pdf_magic}; -pub use nodes::{ImageRecord, PageExtract, PdfLocator, SynthesisReason, SynthesizedChar, TextRun}; +pub use nodes::{ + ImageRecord, PageExtract, PdfLocator, RecognizedSource, SynthesisReason, SynthesizedChar, + TextRun, +}; pub use overlay::{build_overlay, OVERLAY_ARTIFACT_TYPE}; pub use reasons::{LayoutComplexityReason, OcrNeedReason}; +pub use recognize::{ + image_media_type, Base, Colour, FlateParms, ImageSpec, PdfImageSpec, Recognition, + RecognitionRequest, Recognize, RecognizedLine, RecognizedWord, +}; pub use represent::{to_representation, PROCESSOR_NAME}; pub use tagging::{write_tags, TAGS_ARTIFACT_TYPE}; diff --git a/crates/ethos-parser-pdf/src/limitations.rs b/crates/ethos-parser-pdf/src/limitations.rs index 32eca12..a5d6364 100644 --- a/crates/ethos-parser-pdf/src/limitations.rs +++ b/crates/ethos-parser-pdf/src/limitations.rs @@ -858,6 +858,62 @@ pub fn form_xobjects_not_descended_on_page(page: u32, count: u32) -> Limitation ) } +/// A scanned page the OCR profile's recognizer did not read, and why (v3, `docs/35-OCR-SCOPE.md` +/// §7): never dropped in silence, because a page with no runs would otherwise read as blank. +pub fn scanned_page_not_recognized(page: u32, reason: &str) -> Limitation { + Limitation::page( + page, + ethos_parser_core::codes::SCANNED_PAGE_NOT_RECOGNIZED, + format!( + "This page showed no text and placed an image, and was NOT read by the recognizer: \ + `{reason}`. Its text, if it has any, is absent from this artifact. An empty run list \ + on this page is a declared gap, not a blank page." + ), + ) +} + +/// Recognized words not written because no viewer shows where they lie (v3). +pub fn recognized_words_outside_visible_box(words: u32) -> Limitation { + Limitation::document( + ethos_parser_core::codes::RECOGNIZED_WORDS_OUTSIDE_VISIBLE_BOX, + format!( + "{words} word(s) the recognizer read lie outside their page's visible box, where an \ + image runs past the crop, and were NOT written: text a viewer does not show is not \ + the page's." + ), + ) +} + +/// An image file's page is its pixel grid (v3, `docs/35-OCR-SCOPE.md` §4). +pub fn image_resolution_not_read() -> Limitation { + Limitation::document( + ethos_parser_core::codes::IMAGE_RESOLUTION_NOT_READ, + "This document is an image file, read as one page that is its own pixel grid, one pixel \ + to one point: every length on it is in pixels. The resolution the file may declare is \ + not read, so the page's size is not the size the image was scanned or printed at." + .to_string(), + ) +} + +/// What the OCR profile's recognized runs do not carry (v3, `docs/35-OCR-SCOPE.md` §7), declared +/// on every artifact made under it. +pub fn recognized_text_limitations() -> Vec { + vec![ + Limitation::profile( + ethos_parser_core::codes::RECOGNIZED_TEXT_ORIENTATION_NOT_CORRECTED, + "A scanned image is stood upright as its page shows it before it is read, but text \ + turned WITHIN an upright image — a sideways table header, an upside-down stamp — is \ + read as it stands, which is usually not at all.", + ), + Limitation::profile( + ethos_parser_core::codes::RECOGNIZED_TEXT_HAS_NO_FONT, + "A recognized run has no font: its `font_id` is empty and its `font_size` is its \ + line's height, as a text layer written by a recognizer states it. So no recognized \ + run is bold or italic, and a rule that reads a font reads none on a recognized page.", + ), + ] +} + /// The catalog declares no outline (`docs/29-OUTLINES-SCOPE.md`). /// /// On `untagged_structure_tree_absent`'s precedent, and it is the same distinction: **a statement diff --git a/crates/ethos-parser-pdf/src/nodes.rs b/crates/ethos-parser-pdf/src/nodes.rs index d1dc8cb..a4ec6c4 100644 --- a/crates/ethos-parser-pdf/src/nodes.rs +++ b/crates/ethos-parser-pdf/src/nodes.rs @@ -191,6 +191,26 @@ pub struct TextRun { /// observations happened to be made in. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub findings: Vec, + /// Where a recognizer read this run, on a run of derivation `Recognized` (v3, + /// `docs/35-OCR-SCOPE.md` §6): the image, and the word's corners in its pixels. Absent on every + /// run the page drew, which therefore serialize exactly as they did before a recognizer + /// existed. [`Self::locator`] then states the run's place as a text layer would — computed + /// from these corners, never drawn — and the representation's `recognized` address carries + /// both. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recognized: Option, +} + +/// The image a recognized run was read from, and the word's corners in its pixels as the +/// recognizer read them: the top-left, top-right, bottom-right and bottom-left of its text, in +/// reading orientation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecognizedSource { + /// The image XObject; `None` for an image file, whose page is the image. + pub image: Option, + /// The word's corners in the image's own pixel grid. + pub pixels: [[i32; 2]; 4], } impl TextRun { @@ -350,6 +370,7 @@ mod tests { geometry: GeometryPresence::Absent(GeometryAbsence::NotReportedByReader), mcid: None, findings: Vec::new(), + recognized: None, structural: None, derivation: DerivationClass::Extracted, } diff --git a/crates/ethos-parser-pdf/src/recognize.rs b/crates/ethos-parser-pdf/src/recognize.rs new file mode 100644 index 0000000..4417e1b --- /dev/null +++ b/crates/ethos-parser-pdf/src/recognize.rs @@ -0,0 +1,1201 @@ +// Copyright 2026 The ethos-parser maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! A scanned page, asked of a recognizer and answered as words on the page (v3, +//! `docs/35-OCR-SCOPE.md`). +//! +//! **This module decodes nothing.** It reads what the image's dictionary states — size, colour +//! space, filters and their parameters, `/Decode` — and hands the recognizer the stream exactly as +//! the file stores it; the recognizer decodes, and refuses by name what it cannot. What comes back +//! is mapped through the matrix the page painted the image with, which is the only arithmetic +//! here, and it is the same arithmetic `images.rs` applies to the unit square. +//! +//! The recognizer itself is behind [`Recognize`]: the engine never spawns a process, so this crate +//! stays testable with a recognizer that is a function, and the CLI owns the binary. + +use ethos_parser_core::{ + quantize, EngineError, PdfObjectNumber, QRect, RecognizerIdentity, QUANTUM_PER_POINT, +}; +use serde::{Deserialize, Serialize}; + +use crate::fonts::{resolve_array, resolve_object, resolve_stream}; + +/// The recognizer `--ocr` reads a scanned page with. Implemented by the CLI over +/// `ethos-parser-ocr serve`; by a function in tests. +pub trait Recognize { + /// The identity the recognizer reports. Extraction refuses to start unless it is the one the + /// profile pins. + fn identity(&self) -> &RecognizerIdentity; + /// Read one image. `Err` is a failure of the recognizer itself (it exited, it answered out of + /// protocol), which ends the extraction; a refusal to read an image is an answer + /// ([`Recognition::refused`]) and is counted against its page. + fn read(&mut self, request: &RecognitionRequest) -> Result; +} + +/// One request: the JSON header `ethos-parser-ocr serve` reads, and the image's bytes after it. +#[derive(Clone, Debug, Serialize)] +pub struct RecognitionRequest { + /// Protocol version, 1. + pub ocr_request: u32, + /// Echoed in the answer. + pub id: String, + /// What the bytes are. + pub image: ImageSpec, + /// Stands the stored raster upright as the page shows it: mirrored left to right first when 4 + /// or more, then turned `orientation % 4` quarter turns clockwise. + pub orientation: u8, + /// The length of [`Self::data`]. + pub bytes: u64, + /// The image's bytes: a PDF stream as stored, or an image file whole. + #[serde(skip)] + pub data: Vec, +} + +/// What a request's bytes are. +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ImageSpec { + /// An image file, PNG or JPEG. + File, + /// A PDF image XObject's stream, as its dictionary describes it. + Pdf(PdfImageSpec), +} + +/// What an image XObject's dictionary states, resolved to what the recognizer reads. +#[derive(Clone, Debug, Serialize)] +pub struct PdfImageSpec { + /// `/Width`, in samples. + pub width: u32, + /// `/Height`, in samples. + pub height: u32, + /// `/BitsPerComponent`; 1 for an image mask. + pub bits_per_component: u32, + /// `None` for an image mask, which has no colour space. + pub colour: Option, + /// `/Filter`, in order. + pub filters: Vec, + /// `/DecodeParms`, one per filter; `None` where the filter has none this protocol carries. + pub decode_parms: Vec>, + /// `/Decode`, where the dictionary states one. + pub decode: Option>, + /// `/ImageMask`. + pub image_mask: bool, +} + +/// A colour space, resolved to the components it holds. +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "space", rename_all = "snake_case")] +pub enum Colour { + /// One component: `DeviceGray`, `CalGray`, or ICC-based with one. + Gray, + /// Three: `DeviceRGB`, `CalRGB`, or ICC-based with three. + Rgb, + /// Four: `DeviceCMYK`, or ICC-based with four. + Cmyk, + /// `/Indexed`: a palette over a base space. + Indexed { + /// The base space. + base: Base, + /// The highest index. + hival: u32, + /// The palette, `hival + 1` colours in the base's components. + lookup: Vec, + }, +} + +/// An indexed space's base. +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Base { + /// One component. + Gray, + /// Three. + Rgb, + /// Four. + Cmyk, +} + +/// A Flate or LZW stream's `/DecodeParms` (32000-1 Table 8), defaults filled in. +#[derive(Clone, Debug, Serialize)] +pub struct FlateParms { + /// `/Predictor`, 1 when absent. + pub predictor: u32, + /// `/Colors`, 1 when absent. + pub colors: u32, + /// `/BitsPerComponent`, 8 when absent. + pub bits_per_component: u32, + /// `/Columns`, 1 when absent. + pub columns: u32, +} + +/// The recognizer's answer. +#[derive(Clone, Debug, Deserialize)] +pub struct Recognition { + /// Protocol version, 1. + pub ocr_response: u32, + /// The request's id. + pub id: String, + /// The identity the recognizer reports, on every answer. + pub identity: RecognizerIdentity, + /// The stored raster's width, as decoded. + pub width: u32, + /// The stored raster's height, as decoded. + pub height: u32, + /// What it read, in its reading order. + pub lines: Vec, + /// Why the image was not read, by name. + pub refused: Option, +} + +/// One line, its quad in the stored raster's pixels. +#[derive(Clone, Debug, Deserialize)] +pub struct RecognizedLine { + /// The line's text as read, spaces included. + pub text: String, + /// The text's top-left, top-right, bottom-right and bottom-left corners. + pub quad: [[i32; 2]; 4], + /// Its words, in order. + pub words: Vec, +} + +/// One word, its quad in the stored raster's pixels. +#[derive(Clone, Debug, Deserialize)] +pub struct RecognizedWord { + /// The word's characters. + pub text: String, + /// Its corners, as the line's are. + pub quad: [[i32; 2]; 4], +} + +/// The media type an image file's own signature states, for the two kinds `--ocr` reads whole +/// (`docs/35-OCR-SCOPE.md` §4): PNG and JPEG. Asked of the first bytes only, as every other +/// signature the engine reads is. +pub fn image_media_type(head: &[u8]) -> Option<&'static str> { + if head.starts_with(b"\x89PNG\r\n\x1a\n") { + Some("image/png") + } else if head.starts_with(&[0xFF, 0xD8, 0xFF]) { + Some("image/jpeg") + } else { + None + } +} + +/// A one-page PDF, `width` by `height` points, that draws nothing: the page an image file's words +/// are set on, one pixel to one point. Never written anywhere; the artifact's source is the file. +pub(crate) fn blank_page(width: u32, height: u32) -> Result, EngineError> { + use lopdf::{dictionary, Object}; + let mut doc = lopdf::Document::with_version("1.7"); + let pages = doc.new_object_id(); + let page = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages, + "MediaBox" => vec![0.into(), 0.into(), i64::from(width).into(), i64::from(height).into()], + "Resources" => dictionary! {}, + }); + doc.objects.insert( + pages, + Object::Dictionary( + dictionary! { "Type" => "Pages", "Kids" => vec![page.into()], "Count" => 1 }, + ), + ); + let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages }); + doc.trailer.set("Root", catalog); + let mut bytes = Vec::new(); + doc.save_to(&mut bytes).map_err(|e| EngineError::Io { + detail: e.to_string(), + })?; + Ok(bytes) +} + +/// The placement of an image file on its own page: pixel (x, y) at (x, y) points from the top +/// left, so a recognized word's place on the page is its place in the file. +pub(crate) fn whole_page(page_id: lopdf::ObjectId, width: u32, height: u32) -> Candidate { + let (w, h) = (f64::from(width), f64::from(height)); + Candidate { + page_id, + // No image object: the page is the image. + object: (0, 0), + corners: [(0.0, h), (w, h), (w, 0.0), (0.0, 0.0)], + visible: (0.0, 0.0, w, h), + } +} + +/// A page the trigger took: no text-showing operator, and exactly one image placed (§4). +#[derive(Clone, Debug)] +pub(crate) struct Candidate { + pub page_id: lopdf::ObjectId, + pub object: lopdf::ObjectId, + /// The unit square's corners (0,0), (1,0), (1,1), (0,1) on the page, in points, top-left + /// system, `/Rotate` applied: where the image's pixels land. + pub corners: [(f64, f64); 4], + /// The box a viewer shows, in the same system: (x0, y0, x1, y1). + pub visible: (f64, f64, f64, f64), +} + +/// A recognized word placed on its page, quantized, ready to become a run. +#[derive(Clone, Debug)] +pub(crate) struct PageWord { + /// The word, with the space the recognizer read after it where one follows on its line. + pub text: String, + pub origin_x: i64, + pub origin_y: i64, + /// The word box's width; `None` for a line that does not run left to right on the page, + /// whose travel along x is not its length. + pub advance: Option, + /// The height of the word's line, between its quad's top and bottom edges: the size a text + /// layer would set it at. + pub font_size: i64, + /// The word's left and right over its line's top and bottom. + pub rect: QRect, + pub image: Option, + pub pixels: [[i32; 2]; 4], +} + +/// The request for `candidate`'s image, or the name of what this protocol cannot carry. +pub(crate) fn request( + doc: &lopdf::Document, + candidate: &Candidate, + page_number: u32, +) -> Result { + let stream = doc + .get_object(candidate.object) + .and_then(lopdf::Object::as_stream) + .map_err(|_| "image-not-loaded".to_string())?; + let dict = &stream.dict; + let int = |key: &[u8]| -> Option { + resolve_object(doc, dict.get(key).ok()).and_then(|o| o.as_i64().ok()) + }; + let (width, height) = match (int(b"Width"), int(b"Height")) { + (Some(w), Some(h)) + if w > 0 && h > 0 && w <= i64::from(u32::MAX) && h <= i64::from(u32::MAX) => + { + (w as u32, h as u32) + } + _ => return Err("image-size-not-read".into()), + }; + // A mask decides what of the image a viewer sees, and the protocol carries the image alone: + // read without it, black text masked onto a black image reaches the recognizer all black, and + // what a stencil hides could be read as text no viewer shows (PR 53's review). + if dict.has(b"SMask") { + return Err("soft-mask-not-read".into()); + } + if dict.has(b"Mask") { + return Err("mask-not-read".into()); + } + let image_mask = resolve_object(doc, dict.get(b"ImageMask").ok()) + .and_then(|o| o.as_bool().ok()) + .unwrap_or(false); + let bits_per_component = match int(b"BitsPerComponent") { + Some(b) => u32::try_from(b).map_err(|_| "bits-per-component-not-read".to_string())?, + None if image_mask => 1, + None => 8, // a JPEG 2000 stream may omit it; the recognizer refuses that filter by name + }; + let colour = if image_mask { + None + } else { + Some(colour(doc, dict.get(b"ColorSpace").ok())?) + }; + let names = |o: Option| -> Vec { + match o { + Some(lopdf::Object::Name(n)) => vec![String::from_utf8_lossy(&n).into_owned()], + Some(lopdf::Object::Array(a)) => a + .iter() + .filter_map(|x| x.as_name().ok()) + .map(|n| String::from_utf8_lossy(n).into_owned()) + .collect(), + _ => Vec::new(), + } + }; + let filters = names(resolve_object(doc, dict.get(b"Filter").ok())); + let parms: Vec> = match resolve_object(doc, dict.get(b"DecodeParms").ok()) + { + Some(lopdf::Object::Array(a)) => a.iter().map(|o| resolve_object(doc, Some(o))).collect(), + Some(o) => vec![Some(o)], + None => Vec::new(), + }; + let decode_parms = filters + .iter() + .enumerate() + .map( + |(i, f)| match (f.as_str(), parms.get(i).cloned().flatten()) { + ( + "FlateDecode" | "Fl" | "LZWDecode" | "LZW", + Some(lopdf::Object::Dictionary(d)), + ) => { + let p = |key: &[u8], default: i64| { + resolve_object(doc, d.get(key).ok()) + .and_then(|o| o.as_i64().ok()) + .unwrap_or(default) + }; + Some(FlateParms { + predictor: p(b"Predictor", 1).clamp(0, 15) as u32, + colors: p(b"Colors", 1).clamp(1, 4) as u32, + bits_per_component: p(b"BitsPerComponent", 8).clamp(1, 16) as u32, + columns: p(b"Columns", 1).clamp(1, i64::from(u32::MAX)) as u32, + }) + } + _ => None, + }, + ) + .collect(); + let decode = resolve_array(doc, dict.get(b"Decode").ok()).map(|a| { + a.iter() + .filter_map(|o| match o { + lopdf::Object::Integer(i) => Some(*i as f64), + lopdf::Object::Real(r) => Some(f64::from(*r)), + _ => None, + }) + .collect() + }); + let orientation = + orientation(candidate.corners, width, height).ok_or("placement-not-axis-aligned")?; + Ok(RecognitionRequest { + ocr_request: 1, + id: format!("page {page_number}"), + image: ImageSpec::Pdf(PdfImageSpec { + width, + height, + bits_per_component, + colour, + filters, + decode_parms, + decode, + image_mask, + }), + orientation, + bytes: stream.content.len() as u64, + data: stream.content.clone(), + }) +} + +/// A colour space resolved to what the recognizer reads, or the name of one it does not. +fn colour(doc: &lopdf::Document, o: Option<&lopdf::Object>) -> Result { + let space = resolve_object(doc, o).ok_or("colour-space-not-read")?; + let named = |n: &[u8]| -> Result { + match n { + b"DeviceGray" | b"G" | b"CalGray" => Ok(Colour::Gray), + b"DeviceRGB" | b"RGB" | b"CalRGB" => Ok(Colour::Rgb), + b"DeviceCMYK" | b"CMYK" => Ok(Colour::Cmyk), + other => Err(format!( + "colour-space-not-read:{}", + String::from_utf8_lossy(other) + )), + } + }; + match &space { + lopdf::Object::Name(n) => named(n), + lopdf::Object::Array(a) => { + let family = a + .first() + .and_then(|f| f.as_name().ok()) + .ok_or("colour-space-not-read")?; + match family { + b"ICCBased" => { + let s = + resolve_stream(doc, a.get(1)).ok_or("colour-space-not-read:ICCBased")?; + match resolve_object(doc, s.dict.get(b"N").ok()).and_then(|n| n.as_i64().ok()) { + Some(1) => Ok(Colour::Gray), + Some(3) => Ok(Colour::Rgb), + Some(4) => Ok(Colour::Cmyk), + _ => Err("colour-space-not-read:ICCBased".into()), + } + } + b"CalGray" | b"CalRGB" | b"DeviceGray" | b"DeviceRGB" | b"DeviceCMYK" => { + named(family) + } + b"Indexed" | b"I" => { + let base = match colour(doc, a.get(1))? { + Colour::Gray => Base::Gray, + Colour::Rgb => Base::Rgb, + Colour::Cmyk => Base::Cmyk, + Colour::Indexed { .. } => { + return Err("colour-space-not-read:Indexed".into()) + } + }; + let hival = resolve_object(doc, a.get(2)) + .and_then(|h| h.as_i64().ok()) + .filter(|h| (0..=255).contains(h)) + .ok_or("colour-space-not-read:Indexed")?; + let lookup = match resolve_object(doc, a.get(3)) { + Some(lopdf::Object::String(bytes, _)) => bytes, + // A lookup table in a stream would have to be decoded here; refused by name. + _ => return Err("colour-space-not-read:Indexed".into()), + }; + Ok(Colour::Indexed { + base, + hival: hival as u32, + lookup, + }) + } + other => Err(format!( + "colour-space-not-read:{}", + String::from_utf8_lossy(other) + )), + } + } + _ => Err("colour-space-not-read".into()), + } +} + +/// The orientation that stands the stored raster upright as the page shows it: the one of the +/// eight axis-aligned ones whose turn (and mirror) points the raster's right and down where the +/// page draws them. `None` when the placement skews or turns by anything but a quarter. +pub(crate) fn orientation(corners: [(f64, f64); 4], width: u32, height: u32) -> Option { + let [c00, c10, _, c01] = corners; + // Where one stored pixel to the right, and one down, land on the page (top-left system). The + // stored raster's first row is the top of the unit square, v = 1. + let right = ( + (c10.0 - c00.0) / f64::from(width), + (c10.1 - c00.1) / f64::from(width), + ); + let down = ( + -(c01.0 - c00.0) / f64::from(height), + -(c01.1 - c00.1) / f64::from(height), + ); + let sign = |v: f64, scale: f64| -> i32 { + if v.abs() <= scale * 1e-9 { + 0 + } else if v > 0.0 { + 1 + } else { + -1 + } + }; + let scale = right + .0 + .abs() + .max(right.1.abs()) + .max(down.0.abs()) + .max(down.1.abs()); + let shown = [ + [sign(right.0, scale), sign(down.0, scale)], + [sign(right.1, scale), sign(down.1, scale)], + ]; + (0..8u8).find(|&o| turn_matrix(o) == shown) +} + +/// How orientation `o` moves the stored raster's right and down, as columns of a matrix over +/// (x right, y down): mirrored left to right first when `o >= 4`, then turned `o % 4` quarter turns +/// clockwise — a clockwise turn sends right to down and down to left. +fn turn_matrix(o: u8) -> [[i32; 2]; 2] { + let mut m = if o >= 4 { + [[-1, 0], [0, 1]] + } else { + [[1, 0], [0, 1]] + }; + for _ in 0..o % 4 { + // R = [[0, -1], [1, 0]]; m = R * m. + m = [[-m[1][0], -m[1][1]], [m[0][0], m[0][1]]]; + } + m +} + +/// One run of a recognized line: its text, its left and right on the page, the pixels it was read +/// from, and whether it advances along x. +type Piece = (String, f64, f64, [[i32; 2]; 4], bool); + +/// The words of `answer`, placed on `candidate`'s page: each word's left and right through the +/// placement, over its line's top and bottom; the space the recognizer read after a word kept on +/// it; the size the line's own height between its quad's top and bottom edges, not its box's, +/// which a skewed line makes taller. A line that does not run left to right on the page — turned +/// within the image, or upside down — is one run, its words joined by spaces, with no advance +/// along x: a word of it has no left and right a line's rules could read. Words whose box's +/// centre falls outside the visible box are dropped and counted, as text no viewer shows. +/// `image` is the image XObject read. +pub(crate) fn place( + candidate: &Candidate, + answer: &Recognition, + image: Option, +) -> Result<(Vec, u32), EngineError> { + let (w, h) = (f64::from(answer.width), f64::from(answer.height)); + let [c00, c10, _, c01] = candidate.corners; + // Pixel corner (px, py) to the page: u = px / w, v = 1 - py / h on the unit square. + let at = |p: [i32; 2]| -> (f64, f64) { + let (u, v) = (f64::from(p[0]) / w, 1.0 - f64::from(p[1]) / h); + ( + c00.0 + u * (c10.0 - c00.0) + v * (c01.0 - c00.0), + c00.1 + u * (c10.1 - c00.1) + v * (c01.1 - c00.1), + ) + }; + let span = |q: &[[i32; 2]; 4]| -> (f64, f64, f64, f64) { + let pts = q.map(at); + let xs = pts.map(|p| p.0); + let ys = pts.map(|p| p.1); + let lo = |v: [f64; 4]| v.iter().copied().fold(f64::INFINITY, f64::min); + let hi = |v: [f64; 4]| v.iter().copied().fold(f64::NEG_INFINITY, f64::max); + (lo(xs), lo(ys), hi(xs), hi(ys)) + }; + let q = |pt: f64| { + quantize(pt, QUANTUM_PER_POINT).map_err(|_| EngineError::Malformed { + what: "recognized word".into(), + detail: "a recognized word's place cannot be expressed in centipoints".into(), + }) + }; + let (vx0, vy0, vx1, vy1) = candidate.visible; + let (mut out, mut dropped) = (Vec::new(), 0u32); + for line in &answer.lines { + let p = line.quad.map(at); + let (dx, dy) = (p[1].0 - p[0].0, p[1].1 - p[0].1); + let (ux, uy) = (p[3].0 - p[0].0, p[3].1 - p[0].1); + let height = q((ux * ux + uy * uy).sqrt())?; + let (lx0, ly0, lx1, ly1) = span(&line.quad); + let along_x = dx > 0.0 && dy.abs() <= dx; + let pieces: Vec = if along_x { + let last = line.words.len().saturating_sub(1); + line.words + .iter() + .enumerate() + .map(|(k, word)| { + let (x0, _, x1, _) = span(&word.quad); + let text = if k < last { + format!("{} ", word.text) + } else { + word.text.clone() + }; + (text, x0, x1, word.quad, true) + }) + .collect() + } else { + let text = line + .words + .iter() + .map(|wd| wd.text.as_str()) + .collect::>() + .join(" "); + vec![(text, lx0, lx1, line.quad, false)] + }; + for (text, x0, x1, pixels, advances) in pieces { + if text.is_empty() { + continue; + } + let (cx, cy) = ((x0 + x1) / 2.0, (ly0 + ly1) / 2.0); + if !(vx0..=vx1).contains(&cx) || !(vy0..=vy1).contains(&cy) { + dropped = dropped.saturating_add(1); + continue; + } + let (qx0, qy0, qx1, qy1) = (q(x0)?, q(ly0)?, q(x1)?, q(ly1)?); + let rect = QRect::new(qx0, qy0, qx1.max(qx0), qy1.max(qy0)).map_err(|_| { + EngineError::Malformed { + what: "recognized word".into(), + detail: "a recognized word's box is not a rectangle".into(), + } + })?; + out.push(PageWord { + text, + origin_x: qx0, + origin_y: qy1, + advance: advances.then(|| qx1.max(qx0) - qx0), + font_size: height, + rect, + image, + pixels, + }); + } + } + Ok((out, dropped)) +} + +#[cfg(test)] +mod tests { + use super::*; + + use ethos_parser_core::{DerivationClass, NativeLocator, Profile, RecognizerPin}; + + use crate::document::Document; + + /// A one-page PDF, 200 x 100 points, drawing `images` copies of a 20 x 10 gray image with the + /// matrix `cm`, and `text` with Helvetica when given. + fn scanned(cm: &str, images: usize, text: Option<&str>) -> Vec { + use lopdf::{dictionary, Object, Stream}; + let mut doc = lopdf::Document::with_version("1.7"); + let pages = doc.new_object_id(); + let image = doc.add_object(Stream::new( + dictionary! { + "Type" => "XObject", "Subtype" => "Image", "Width" => 20, "Height" => 10, + "ColorSpace" => "DeviceGray", "BitsPerComponent" => 8, + }, + vec![255u8; 200], + )); + let font = doc.add_object(dictionary! { + "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica" + }); + let mut ops = String::new(); + for _ in 0..images { + ops.push_str(&format!("q {cm} cm /Im0 Do Q ")); + } + if let Some(t) = text { + ops.push_str(&format!("BT /F1 12 Tf 10 50 Td ({t}) Tj ET")); + } + let content = doc.add_object(Stream::new(dictionary! {}, ops.into_bytes())); + let page = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages, + "MediaBox" => vec![0.into(), 0.into(), 200.into(), 100.into()], + "Resources" => dictionary! { + "XObject" => dictionary! { "Im0" => image }, + "Font" => dictionary! { "F1" => font }, + }, + "Contents" => content, + }); + doc.objects.insert( + pages, + Object::Dictionary( + dictionary! { "Type" => "Pages", "Kids" => vec![page.into()], "Count" => 1 }, + ), + ); + let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages }); + doc.trailer.set("Root", catalog); + let mut bytes = Vec::new(); + doc.save_to(&mut bytes).expect("saves"); + bytes + } + + /// A recognizer that is a function: it answers every request with `answer` (or refuses with + /// `refused`) and keeps the requests. + struct Fake { + identity: RecognizerIdentity, + /// The identity its answers carry, where it is not the one it states. + answers_as: Option, + refused: Option, + requests: Vec, + /// The pixel "world" ends at; 18, inside the 20-pixel image, unless a test moves it. + reach: i32, + } + + impl Fake { + fn new() -> Self { + Fake { + identity: RecognizerIdentity::ethos_parser_ocr(), + answers_as: None, + refused: None, + requests: Vec::new(), + reach: 18, + } + } + } + + impl Recognize for Fake { + fn identity(&self) -> &RecognizerIdentity { + &self.identity + } + fn read(&mut self, request: &RecognitionRequest) -> Result { + self.requests.push(request.clone()); + let word = |t: &str, x0: i32, x1: i32| RecognizedWord { + text: t.into(), + quad: [[x0, 2], [x1, 2], [x1, 8], [x0, 8]], + }; + Ok(Recognition { + ocr_response: 1, + id: request.id.clone(), + identity: self + .answers_as + .clone() + .unwrap_or_else(|| self.identity.clone()), + width: 20, + height: 10, + lines: if self.refused.is_some() { + Vec::new() + } else { + vec![RecognizedLine { + text: "Hello world".into(), + quad: [[2, 2], [18, 2], [18, 8], [2, 8]], + words: vec![word("Hello", 2, 9), word("world", 11, self.reach)], + }] + }, + refused: self.refused.clone(), + }) + } + } + + fn codes(a: &crate::ExtractArtifact) -> Vec<&str> { + a.assurance + .limitations + .iter() + .map(|l| l.code.as_str()) + .collect() + } + + #[test] + fn a_scanned_page_reads_as_recognized_runs() { + let profile = Profile::ocr_v0(); + let doc = + Document::open_bytes(&scanned("200 0 0 100 0 0", 1, None), &profile).expect("opens"); + let mut fake = Fake::new(); + let artifact = crate::extract_recognized(&doc, &profile, &mut fake).expect("extracts"); + assert_eq!(fake.requests.len(), 1); + assert_eq!(fake.requests[0].orientation, 0); + assert_eq!(fake.requests[0].data.len(), 200, "the stream as stored"); + let runs = &artifact.pages[0].runs; + assert_eq!( + runs.iter().map(|r| r.text.as_str()).collect::>(), + ["Hello ", "world"] + ); + let r = &runs[0]; + assert_eq!(r.derivation, DerivationClass::Recognized); + // 2 of 20 pixels across 200 points, the line's foot 8 of 10 pixels down 100 points. + assert_eq!( + (r.locator.origin_x, r.locator.origin_y, r.locator.advance), + (2000, 8000, Some(7000)) + ); + assert_eq!(r.font_size, 6000); + assert_eq!( + r.recognized.as_ref().unwrap().pixels, + [[2, 2], [9, 2], [9, 8], [2, 8]] + ); + assert!(codes(&artifact).contains(ðos_parser_core::codes::RECOGNIZED_TEXT_HAS_NO_FONT)); + // The representation addresses it where the recognizer read it, and seals. + let rep = crate::to_representation(&artifact, &profile).expect("seals"); + let node = rep + .payload() + .nodes + .iter() + .find(|n| n.text == "Hello ") + .expect("the run is a node"); + assert_eq!(node.derivation, DerivationClass::Recognized); + let NativeLocator::Recognized(at) = &node.native_locator else { + panic!("a recognized run is addressed as one"); + }; + assert_eq!(at.run.origin_x, 2000); + assert!(at.image.is_some()); + } + + #[test] + fn a_flipped_image_is_asked_for_upright() { + let profile = Profile::ocr_v0(); + let doc = + Document::open_bytes(&scanned("200 0 0 -100 0 100", 1, None), &profile).expect("opens"); + let mut fake = Fake::new(); + let artifact = crate::extract_recognized(&doc, &profile, &mut fake).expect("extracts"); + assert_eq!(turn_matrix(fake.requests[0].orientation), [[1, 0], [0, -1]]); + // Flipped, the stored rows 2..8 of 10 land at 80..20 points down the page: the line's + // foot is the lower edge, 80 points, as it is upright, because the rows straddle the + // middle; what moved is the request, which stands the raster up before it is read. + assert_eq!(artifact.pages[0].runs[0].locator.origin_y, 8000); + } + + #[test] + fn a_page_that_shows_text_is_not_asked() { + let profile = Profile::ocr_v0(); + let doc = Document::open_bytes(&scanned("200 0 0 100 0 0", 1, Some("drawn")), &profile) + .expect("opens"); + let mut fake = Fake::new(); + let artifact = crate::extract_recognized(&doc, &profile, &mut fake).expect("extracts"); + assert!(fake.requests.is_empty()); + assert!(artifact.pages[0] + .runs + .iter() + .all(|r| r.derivation == DerivationClass::Extracted)); + assert!(!codes(&artifact).contains(ðos_parser_core::codes::SCANNED_PAGE_NOT_RECOGNIZED)); + } + + #[test] + fn two_images_are_refused_by_name() { + let profile = Profile::ocr_v0(); + let doc = + Document::open_bytes(&scanned("200 0 0 100 0 0", 2, None), &profile).expect("opens"); + let mut fake = Fake::new(); + let artifact = crate::extract_recognized(&doc, &profile, &mut fake).expect("extracts"); + assert!(fake.requests.is_empty()); + let l = artifact + .assurance + .limitations + .iter() + .find(|l| l.code == ethos_parser_core::codes::SCANNED_PAGE_NOT_RECOGNIZED) + .expect("counted on its page"); + assert!(l.detail.contains("several-images"), "{}", l.detail); + } + + #[test] + fn an_image_a_mask_shapes_is_refused_by_name() { + // PR 53's review: the protocol carries the image alone, so a page whose image a soft mask + // or a mask shapes is refused by name, never read as something no viewer sees. + use lopdf::{dictionary, Object, Stream}; + let profile = Profile::ocr_v0(); + for (key, reason) in [("SMask", "soft-mask-not-read"), ("Mask", "mask-not-read")] { + let mut pdf = + lopdf::Document::load_mem(&scanned("200 0 0 100 0 0", 1, None)).expect("loads"); + let mask = pdf.add_object(Stream::new( + dictionary! { + "Type" => "XObject", "Subtype" => "Image", "Width" => 20, "Height" => 10, + "ColorSpace" => "DeviceGray", "BitsPerComponent" => 8, + }, + vec![255u8; 200], + )); + let image = pdf + .objects + .iter_mut() + .find_map(|(id, o)| match o { + Object::Stream(s) if *id != mask && s.dict.has(b"Width") => Some(s), + _ => None, + }) + .expect("the page's image"); + image.dict.set(key, mask); + let mut bytes = Vec::new(); + pdf.save_to(&mut bytes).expect("saves"); + let doc = Document::open_bytes(&bytes, &profile).expect("opens"); + let mut fake = Fake::new(); + let artifact = crate::extract_recognized(&doc, &profile, &mut fake).expect("extracts"); + assert!(fake.requests.is_empty(), "{key}: nothing is sent"); + let l = artifact + .assurance + .limitations + .iter() + .find(|l| l.code == ethos_parser_core::codes::SCANNED_PAGE_NOT_RECOGNIZED) + .expect("counted on its page"); + assert!(l.detail.contains(reason), "{}", l.detail); + } + } + + #[test] + fn a_recognizer_refusal_is_counted_on_its_page() { + let profile = Profile::ocr_v0(); + let doc = + Document::open_bytes(&scanned("200 0 0 100 0 0", 1, None), &profile).expect("opens"); + let mut fake = Fake::new(); + fake.refused = Some("filter-not-decoded:JPXDecode".into()); + let artifact = crate::extract_recognized(&doc, &profile, &mut fake).expect("extracts"); + assert!(artifact.pages[0].runs.is_empty()); + assert!(artifact.assurance.limitations.iter().any(|l| l.code + == ethos_parser_core::codes::SCANNED_PAGE_NOT_RECOGNIZED + && l.detail.contains("filter-not-decoded:JPXDecode"))); + } + + #[test] + fn the_profile_and_the_recognizer_must_name_each_other() { + let bytes = scanned("200 0 0 100 0 0", 1, None); + // The OCR profile with no recognizer, and a recognizer under a profile pinning none. + let ocr = Profile::ocr_v0(); + let doc = Document::open_bytes(&bytes, &ocr).expect("opens"); + assert!(matches!( + crate::extract(&doc, &ocr), + Err(EngineError::Unsupported { .. }) + )); + let base = Profile::default(); + let doc = Document::open_bytes(&bytes, &base).expect("opens"); + assert!(matches!( + crate::extract_recognized(&doc, &base, &mut Fake::new()), + Err(EngineError::Unsupported { .. }) + )); + // A recognizer reporting another identity. + let doc = Document::open_bytes(&bytes, &ocr).expect("opens"); + let mut other = Fake::new(); + other.identity.version = "9.9.9".into(); + assert!(matches!( + crate::extract_recognized(&doc, &ocr, &mut other), + Err(EngineError::Unsupported { .. }) + )); + // Refused before anything is read: a document with no scanned page asks it nothing, and + // is still not made under a profile naming another recognizer. + let drawn = scanned("200 0 0 100 0 0", 1, Some("drawn")); + let doc = Document::open_bytes(&drawn, &ocr).expect("opens"); + assert!(matches!( + crate::extract_recognized(&doc, &ocr, &mut other), + Err(EngineError::Unsupported { .. }) + )); + assert!(other.requests.is_empty()); + // One that states the pinned identity and answers as another, a scan and an image file. + let mut two_faced = Fake::new(); + two_faced.answers_as = Some(other.identity.clone()); + let doc = Document::open_bytes(&bytes, &ocr).expect("opens"); + assert!(matches!( + crate::extract_recognized(&doc, &ocr, &mut two_faced), + Err(EngineError::Unsupported { .. }) + )); + assert!(matches!( + crate::extract_image(b"\x89PNG\r\n\x1a\n", &ocr, &mut two_faced), + Err(EngineError::Unsupported { .. }) + )); + // And without `--ocr` nothing about a scanned page changes: no runs, no new limitation. + let doc = Document::open_bytes(&bytes, &base).expect("opens"); + let plain = crate::extract(&doc, &base).expect("extracts"); + assert!(plain.pages[0].runs.is_empty()); + assert!(!codes(&plain) + .iter() + .any(|c| c.starts_with("scanned-page") || c.starts_with("recognized"))); + assert!(matches!(base.recognizer, RecognizerPin::NotPinned)); + } + + #[test] + fn an_upright_placement_needs_no_turn() { + // A 100 x 50 image painted upright at (10, 20)-(110, 70) on the page (top-left system): + // (0,0) of the unit square is the bottom-left corner, (0,1) the top-left. + let corners = [(10.0, 70.0), (110.0, 70.0), (110.0, 20.0), (10.0, 20.0)]; + assert_eq!(orientation(corners, 100, 50), Some(0)); + } + + #[test] + fn a_flipped_placement_is_stood_upright() { + // Painted with a negative vertical scale: the stored first row lands at the page's bottom. + let corners = [(10.0, 20.0), (110.0, 20.0), (110.0, 70.0), (10.0, 70.0)]; + let o = orientation(corners, 100, 50).unwrap(); + // Mirror then a half turn is a flip top to bottom. + assert_eq!(turn_matrix(o), [[1, 0], [0, -1]]); + } + + #[test] + fn a_quarter_turned_placement_turns_back() { + // The unit square's x axis runs up the page and its y axis to the left, so the stored + // raster's right points up and its down points right: the page shows it a quarter turn + // anticlockwise, which is three clockwise. + let corners = [(110.0, 70.0), (110.0, 20.0), (60.0, 20.0), (60.0, 70.0)]; + assert_eq!(orientation(corners, 50, 50), Some(3)); + assert_eq!(turn_matrix(3), [[0, 1], [-1, 0]]); + } + + #[test] + fn a_skewed_placement_is_refused() { + let corners = [(10.0, 70.0), (110.0, 60.0), (110.0, 10.0), (10.0, 20.0)]; + assert_eq!(orientation(corners, 100, 50), None); + } + + #[test] + fn words_land_through_the_placement_and_keep_their_spaces() { + // A 200 x 100 pixel image painted over (0,0)-(100,50) points: half a point per pixel. + let candidate = Candidate { + page_id: (1, 0), + object: (2, 0), + corners: [(0.0, 50.0), (100.0, 50.0), (100.0, 0.0), (0.0, 0.0)], + visible: (0.0, 0.0, 100.0, 50.0), + }; + let word = |t: &str, x0: i32, x1: i32| RecognizedWord { + text: t.into(), + quad: [[x0, 10], [x1, 10], [x1, 30], [x0, 30]], + }; + let answer = Recognition { + ocr_response: 1, + id: "page 1".into(), + identity: RecognizerIdentity::ethos_parser_ocr(), + width: 200, + height: 100, + lines: vec![ + RecognizedLine { + text: "May 2024".into(), + quad: [[10, 8], [90, 8], [90, 32], [10, 32]], + words: vec![word("May", 10, 40), word("2024", 50, 90)], + }, + // A line set up the image's right edge, read bottom to top: its top-left corner + // is its bottom-left on the page. + RecognizedLine { + text: "Side bar".into(), + quad: [[150, 90], [150, 10], [170, 10], [170, 90]], + words: vec![ + RecognizedWord { + text: "Side".into(), + quad: [[150, 90], [150, 10], [170, 10], [170, 90]], + }, + RecognizedWord { + text: "bar".into(), + quad: [[150, 90], [150, 10], [170, 10], [170, 90]], + }, + ], + }, + ], + refused: None, + }; + let (words, dropped) = place(&candidate, &answer, None).unwrap(); + assert_eq!(dropped, 0); + assert_eq!( + words.iter().map(|w| w.text.as_str()).collect::>(), + ["May ", "2024", "Side bar"] + ); + // x 10 px = 5 pt = 500 centipoints; the line runs 8..32 px = 4..16 pt. + assert_eq!( + (words[0].origin_x, words[0].origin_y, words[0].advance), + (500, 1600, Some(1500)) + ); + assert_eq!(words[0].font_size, 1200); + assert_eq!(words[1].origin_x, 2500); + // The sideways line is one run, sized by its own height (20 px = 10 pt), with no advance. + let side = &words[2]; + assert_eq!( + (side.text.as_str(), side.advance, side.font_size), + ("Side bar", None, 1000) + ); + assert_eq!(words.len(), 3); + } + + #[test] + fn a_word_past_its_pages_edge_keeps_its_text_and_not_its_box() { + // PR 53's review: a word whose centre is on the page and whose box runs past its edge + // reached the seal as a measured box outside the page, and the whole document was refused. + let profile = Profile::ocr_v0(); + let mut fake = Fake::new(); + fake.reach = 24; // "world", 11..24, runs past the 20-pixel image; its centre, 17.5, is on it + let png = b"\x89PNG\r\n\x1a\n".to_vec(); + let artifact = crate::extract_image(&png, &profile, &mut fake).expect("extracts"); + let world = &artifact.pages[0].runs[1]; + assert_eq!(world.text, "world"); + assert_eq!( + world.geometry, + ethos_parser_core::GeometryPresence::Absent( + ethos_parser_core::GeometryAbsence::MeasuredOffPage + ) + ); + crate::to_representation(&artifact, &profile).expect("seals"); + } + + #[test] + fn an_image_files_page_holds_a_word_where_the_file_does() { + // A word 5..25 pixels down a 100-pixel image lies 5..25 points down its page: off the + // middle, so a page set upside down would put it 75..95 points down. + let answer = Recognition { + ocr_response: 1, + id: "image".into(), + identity: RecognizerIdentity::ethos_parser_ocr(), + width: 200, + height: 100, + lines: vec![RecognizedLine { + text: "Top".into(), + quad: [[10, 5], [60, 5], [60, 25], [10, 25]], + words: vec![RecognizedWord { + text: "Top".into(), + quad: [[10, 5], [60, 5], [60, 25], [10, 25]], + }], + }], + refused: None, + }; + let (words, dropped) = place(&whole_page((1, 0), 200, 100), &answer, None).unwrap(); + assert_eq!(dropped, 0); + assert_eq!( + (words[0].origin_x, words[0].origin_y, words[0].advance), + (1000, 2500, Some(5000)) + ); + } + + #[test] + fn a_word_no_viewer_shows_is_dropped_and_counted() { + // A 20 x 10 pixel image painted over (0,0)-(200,100) points on a page whose visible box + // ends at x = 100: "Hello" (pixels 2..9) is shown, "world" (11..18) is not. + let candidate = Candidate { + page_id: (1, 0), + object: (2, 0), + corners: [(0.0, 100.0), (200.0, 100.0), (200.0, 0.0), (0.0, 0.0)], + visible: (0.0, 0.0, 100.0, 100.0), + }; + let mut fake = Fake::new(); + let answer = fake + .read(&RecognitionRequest { + ocr_request: 1, + id: "page 1".into(), + image: ImageSpec::File, + orientation: 0, + bytes: 0, + data: Vec::new(), + }) + .expect("answers"); + let (words, dropped) = place(&candidate, &answer, None).unwrap(); + assert_eq!( + words.iter().map(|w| w.text.as_str()).collect::>(), + ["Hello "] + ); + assert_eq!(dropped, 1); + } + + #[test] + fn an_image_file_is_one_page_of_its_own_pixels() { + let profile = Profile::ocr_v0(); + // The fake decodes nothing: a signature is all the engine reads of the file. + let png = b"\x89PNG\r\n\x1a\n and the rest of the file".to_vec(); + let mut fake = Fake::new(); + let artifact = crate::extract_image(&png, &profile, &mut fake).expect("extracts"); + assert_eq!(fake.requests.len(), 1); + assert!(matches!(fake.requests[0].image, ImageSpec::File)); + assert_eq!( + (fake.requests[0].orientation, &fake.requests[0].data), + (0, &png), + "the file whole, as stored" + ); + // The source is the file, not the page it was set on. + assert_eq!(artifact.source.media_type, "image/png"); + assert_eq!( + artifact.source.sha256.as_str(), + format!("sha256:{}", ethos_parser_core::c14n::sha256_hex_bytes(&png)) + ); + // 20 x 10 pixels is 20 x 10 points, and a word lies where the recognizer read it. + let page = &artifact.pages[0]; + assert_eq!((page.width, page.height), (2000, 1000)); + let r = &page.runs[0]; + assert_eq!( + (r.text.as_str(), r.locator.origin_x, r.locator.origin_y), + ("Hello ", 200, 800) + ); + assert_eq!((r.locator.advance, r.font_size), (Some(700), 600)); + let source = r.recognized.as_ref().expect("recognized"); + assert_eq!(source.image, None, "no image object: the page is the image"); + assert_eq!(source.pixels, [[2, 2], [9, 2], [9, 8], [2, 8]]); + assert!(codes(&artifact).contains(ðos_parser_core::codes::IMAGE_RESOLUTION_NOT_READ)); + let rep = crate::to_representation(&artifact, &profile).expect("seals"); + assert_eq!(rep.payload().source.media_type, "image/png"); + assert!( + rep.payload() + .nodes + .iter() + .all(|n| n.kind != ethos_parser_core::NodeKind::Image), + "the page is the image, so no image node is written" + ); + let node = rep + .payload() + .nodes + .iter() + .find(|n| n.text == "Hello ") + .expect("the run is a node"); + let NativeLocator::Recognized(at) = &node.native_locator else { + panic!("a recognized run is addressed as one"); + }; + assert_eq!(at.image, None); + } + + #[test] + fn an_image_file_the_recognizer_refuses_is_refused_by_name() { + let profile = Profile::ocr_v0(); + let mut fake = Fake::new(); + fake.refused = Some("not-decoded".into()); + let e = crate::extract_image(b"\xFF\xD8\xFF\xE0 a JPEG", &profile, &mut fake) + .expect_err("an image file has no other page to count it against"); + assert!( + e.to_string().contains("image/jpeg") && e.to_string().contains("not-decoded"), + "{e}" + ); + } + + #[test] + fn only_png_and_jpeg_are_image_files() { + assert_eq!(image_media_type(b"\x89PNG\r\n\x1a\n"), Some("image/png")); + assert_eq!(image_media_type(b"\xFF\xD8\xFF\xDB"), Some("image/jpeg")); + assert_eq!(image_media_type(b"%PDF-1.7"), None); + assert_eq!( + image_media_type(b"\x89PNG"), + None, + "a signature is all of it" + ); + assert_eq!( + image_media_type(b"\xFF\xD8\x00"), + None, + "a JPEG's first marker follows" + ); + let mut fake = Fake::new(); + let e = crate::extract_image(b"GIF89a", &Profile::ocr_v0(), &mut fake) + .expect_err("refused by name"); + assert!(e.to_string().contains("PNG or a JPEG"), "{e}"); + assert!( + fake.requests.is_empty(), + "refused before the recognizer is asked" + ); + // A profile that pins no recognizer is refused before the file is read, as a PDF is. + let e = crate::extract_image(b"\x89PNG\r\n\x1a\n", &Profile::default(), &mut fake) + .expect_err("no pin"); + assert!(e.to_string().contains("pins none"), "{e}"); + assert!(fake.requests.is_empty()); + } +} diff --git a/crates/ethos-parser-pdf/src/represent.rs b/crates/ethos-parser-pdf/src/represent.rs index 332c2c9..229fc6d 100644 --- a/crates/ethos-parser-pdf/src/represent.rs +++ b/crates/ethos-parser-pdf/src/represent.rs @@ -248,12 +248,26 @@ pub fn to_representation( parent: page_id.clone(), ordinal: (i + 1) as u32, text: run.text.clone(), - native_locator: NativeLocator::Pdf(PdfLocator { - page: run.locator.page, - origin_x: run.locator.origin_x, - origin_y: run.locator.origin_y, - advance: run.locator.advance, - }), + native_locator: { + let place = PdfLocator { + page: run.locator.page, + origin_x: run.locator.origin_x, + origin_y: run.locator.origin_y, + advance: run.locator.advance, + }; + // v3 (`docs/35-OCR-SCOPE.md` §6): a recognized run is addressed where the + // recognizer read it as well as where it lies, and never as a drawn run. + match &run.recognized { + Some(source) => { + NativeLocator::Recognized(ethos_parser_core::RecognizedLocator { + run: place, + image: source.image, + pixels: source.pixels, + }) + } + None => NativeLocator::Pdf(place), + } + }, // v1-S3. Resolved during extraction, against the document's own structure // tree: a role path where the tree cites this run, a bare marked-content id // where it does not, `pdf_artifact` where the page called this furniture, and diff --git a/crates/ethos-parser-pdf/src/tagging.rs b/crates/ethos-parser-pdf/src/tagging.rs index 5afae38..c89685e 100644 --- a/crates/ethos-parser-pdf/src/tagging.rs +++ b/crates/ethos-parser-pdf/src/tagging.rs @@ -5280,6 +5280,7 @@ mod tests { )), derivation: ethos_parser_core::DerivationClass::Extracted, findings: Vec::new(), + recognized: None, } } diff --git a/docs/00-NORTH-STAR.md b/docs/00-NORTH-STAR.md index 92fb236..0ae9d13 100644 --- a/docs/00-NORTH-STAR.md +++ b/docs/00-NORTH-STAR.md @@ -40,11 +40,11 @@ first that refuses a measured performance win on honesty grounds, and it names t | 8 | **Classify** uses reason codes on two independent axes (does it need OCR, is the layout hard), a boolean derived from those reasons, no confidence float, and three exit codes. | | 9 | **Open-source stance.** Ideas are borrowed, code is not: tables and tags from OpenDataLoader, the office IR and error taxonomy from Anydoc, rectangle and encoding handling from pdf-inspector, classify and forms ideas from LiteParse. None is a dependency for the grounded PDF core. | | 10 | **The v1 table bar, amended 2026-08-19: the chase is parked.** The original bar was a published 0.489 score. That number is somebody else's, on their corpus — and a **different unit**: 0.489 is TEDS, a tree edit distance over a DOM, where this engine's 70‰ is cell-slot F1. [`table-gate-v1.md`](table-gate-v1.md) §2 carries the identification; this row said *"same unit"* and was wrong from the day it was written. It gates no slice and is not a shipping precondition. What still binds: **fabrication stays 0**, the method stays in `table-gate-v1.md`. The chase resumes only if this repository acquires a labelled set it owns and chooses to resume it.

**Superseded on v1's status by row 18** (2026-08-30), which closed v1 on a capability statement and a band. This row said *"v1 is not complete"* and is left standing as the record of the amendment that parked the chase — but **row 18 is where v1's status is stated**, and a second copy of it here is what let five documents drift. | -| 11 | **OCR** is out until v4: a deterministic ONNX lane under its own profile, with any confidence value kept diagnostic and never used as a filter. Tesseract is never the default. | +| 11 | **OCR** is out until v4: a deterministic ONNX lane under its own profile, with any confidence value kept diagnostic and never used as a filter. Tesseract is never the default.

**Amended 2026-11-10 by row 120:** OCR is v3 and built now — PP-OCRv5 mobile on rten, a deterministic ONNX lane in its own binary under its own profile, which writes no confidence at all. Tesseract is not used. | | 12 | **Optional agents** may assist later, emitting `Proposed` only. They never overwrite `Extracted`, and never share a processor identity with the evidence path. | | 13 | **Converting office files to PDF is forbidden.** It invents pagination. | | 14 | **No AGPL.** PDFium only if caller-provided under an explicit ADR. The core is clean-room `lopdf` plus vendored specification data. | -| 15 | **Roadmap order, amended 2026-08-21:** after v2 comes **v2.2 (accessibility) → v3 (assist) → v4 (OCR)**. OCR moved last and was renumbered rather than resequenced, because `parser_version` sits inside `profile_sha256` and a later build carrying a lower number defeats the only job that field has. v2.1 is now a gap and nothing ever shipped under it.

**Amended 2026-09-17 by row 28:** v2.3 (distribution) sits between v2.2 and v3, so the order reads v2.2 → v2.3 → v3 → v4. Row 15's reason survives the insertion: v2.3's work ships after v2.2's and before v3's, so no later build carries a lower number. | +| 15 | **Roadmap order, amended 2026-08-21:** after v2 comes **v2.2 (accessibility) → v3 (assist) → v4 (OCR)**. OCR moved last and was renumbered rather than resequenced, because `parser_version` sits inside `profile_sha256` and a later build carrying a lower number defeats the only job that field has. v2.1 is now a gap and nothing ever shipped under it.

**Amended 2026-09-17 by row 28:** v2.3 (distribution) sits between v2.2 and v3, so the order reads v2.2 → v2.3 → v3 → v4. Row 15's reason survives the insertion: v2.3's work ships after v2.2's and before v3's, so no later build carries a lower number.

**Amended 2026-11-10 by row 120:** OCR is built ahead of assist, so the two swap numbers and the order reads v2.2 → v2.3 → v3 (OCR) → v4 (assist). Row 15's reason is why they swap rather than resequence: OCR's builds ship first, so they carry the lower number. | | 16 | **What v2's gate means, settled 2026-08-21: *ground* means *bind*.** A DOCX quote and an XLSX cell each resolve to an address the file itself states. It does not mean emitting the paginated `ethos.grounding.v1`, which a DOCX cannot satisfy — and the gate's own second clause, *no synthesised pages*, forbids what the literal reading would require. Widening the schema stayed blocked on an Ethos-side revision rather than refused; that revision landed as 1.1.0, and 0.39.0 took it. Office representations now project the page-less shape, with no synthesised page anywhere. | | 17 | **"Embedded assets", settled 2026-08-21: counting them satisfies v2.** Every reader declares how many entries it passed over that hold a picture, a clip or an embedded object. *Reading* one is explicitly not v2 — an office image has no page and no coordinate system, so a node for it is a contract change rather than a reader change. | | 18 | **DECIDED 2026-08-30. v1 closes on a capability statement and a band, not on a macro.** What v1 publishes is four clauses: the engine **reads the tables a document declares** in its structure tree, **detects ruled tables** where the producer drew the rules, **emits nothing** where neither holds, and **fabricates nothing**.

**The numbers that go with them**, on twelve tagged, public, redistributable documents — 2,068 pages, 172 tagged tables, 15,755 tagged cell slots. `tagged-tables-v1` recovers **157 of the 172** gold tables and takes combined cell-slot recall to **502‰**. The geometric detectors alone score **70‰** macro cell-slot F1, and the band is published with it and never without it: **0‰–590‰, median 0‰, ten of the twelve exactly zero**, two documents supplying the whole average. Micro recall for the geometric rules alone is **4‰**. Detection precision after v2-S20 is **941‰**, with cross-check disagreements at 0. **Fabrication is 0 on all twelve.**

**Why a band and not an average.** A single macro over mostly zeros reads as *"the engine gets 7% of table cells right"* when the truth is *"most of two documents, nothing on ten"*. The band says the real shape — bimodal — which is the finding twelve documents established that four could not.

**What stays parked.** The geometric chase. v2-S20 spent the one concrete lead this row offered and it moved precision without moving the macro; reopening means a genuinely new geometric idea, not more effort on the current one. The 0.489 comparator is somebody else's score on their own corpus and is not chased. **What stays binding:** fabrication stays 0, the method stays in `table-gate-v1.md`, and table numbers are quoted from there or not at all. | @@ -149,6 +149,8 @@ first that refuses a measured performance win on honesty grounds, and it names t | 117 | **DECIDED 2026-11-08. A ruled row holding rows the page did not rule refuses the page's grid, and a figure under a figure opens a row.** Told "Fresh table census", the census of tables whose rows or columns we cut differently from ParseBench's ground truth (77 tables, 0.73 overall points on `6be3031`) found ruled cells holding rows the page did not rule and rows of figures read as the wrap of the row above; of the fixes measured the owner chose this one with #118 and #119. The page's lattice, which no clause of `standing` asked, is now refused where a ruled row holds, in every cell holding text, three lines or more on baselines its neighbours share — its first row spared where a row under it holds text — and the page is read group by group; where no group's grid stands the page records `rows the page did not rule`, and a refused grid takes no table id. And a line of two cells or more that sets a number, not a year, under a number of the row above opens a row. ParseBench alone on `6be3031`: tables 0.6224 -> 0.6326, 24 pages up and 5 down (`SERFF_CA_random_pages 1` p46 0.01 -> 1.00; down, `Apple 10-k` p45 0.74 -> 0.66, its rows split and its table read as two); visual grounding 0.5915 -> 0.5987, frames round prose no longer tables, `llama` p6 and p7 down; overall 51.49 -> 51.84. opendataloader-bench 0.7999 -> 0.8004, one document up; the fabrication check holds (0 of 1,544 cells); the gate documents' 556 tables unchanged. Not built: a ruled row split at two shared baselines, a `$` drawn in a cell of its own joined to its amount. Folded into `ruled-rects-v14` and `whitespace-tracks-v16`, which no release carries, so no rule id and no `profile_sha256` moves. | | 118 | **DECIDED 2026-11-08. A header set above a section's label, or naming few columns over its last line, heads the table.** Told "Fresh table census", the census of tables whose header we left above them (88 tables, 1.05 overall points on `6be3031`) found two shapes; of the fixes measured the owner chose this one with #117 and #119. A band naming half the columns or fewer joins the header where a line of it holds two cells or more and the first row reads as the header's last line; and where the first row reads as no header's line, one-cell lines just above it — a section's label on the first column, a units note on none — are passed over to a band naming more than half the columns, holding no number but a year, which heads the table, the labels a row under it and a units note in no cell. This reverses `-v11`'s reading of a long cell alone on its line under a header, which ended the band. ParseBench alone on `6be3031`: tables 0.6224 -> 0.6313, 25 pages up and 3 down by under 0.004 each (`METLIFE-10Q-20240205 unrestricted` p16 0.45 -> 0.90, `NEE.2011` p47 0.44 -> 0.88); charts, content, formatting and visual grounding unchanged on every page; overall 51.49 -> 51.67. opendataloader-bench's 200 Markdown files byte-identical; the fabrication check holds; the gate documents' 556 tables unchanged. Not built: header words set less than an em apart; heads over pairs of columns, which this stacks into one column of each pair. Folded into `whitespace-tracks-v16`, which no release carries, so no rule id and no `profile_sha256` moves. | | 119 | **DECIDED 2026-11-08. A footnote mark set against its label joins its line, an amount that opens with its sign opens its cell, and small numbers beside amounts key a table.** Told "Fresh table census", the census of tables we read as text — found by no rule (25 tables, 0.53 overall points on `6be3031`) or cut off after their first rows (25, 0.48) — found three causes; of the fixes measured the owner chose this one with #117 and #118. A line set smaller than the line below joins it raised up to three fifths of an em where each of its runs starts within a quarter em of another run's end (a chart's labels stand against no text and stay apart); in a line that sets its signs first a run that opens with a sign after a figure opens the next cell, as a sign alone does since #116; and two columns whose first is list labels are a table where the second is all numbers. ParseBench alone on `6be3031`: tables 0.6224 -> 0.6258, 8 pages up and none down (`SERFF_TX_random_pages 1` p1124 0 -> 0.49, `BXP.2008` p112 0 -> 0.40); charts, content, formatting and visual grounding unchanged on every page; overall 51.49 -> 51.56. opendataloader-bench's 200 Markdown files byte-identical; the fabrication check holds; the gate documents' 556 tables, trees stripped, unchanged and one new, `irs-fw9` p5's, read from an item's last line. Not built: a table in one column of a two-column page; a run holding two columns' text. Folded into `whitespace-tracks-v16`, which no release carries, so no rule id and no `profile_sha256` moves. | +| 120 | **DECIDED 2026-11-10. OCR is built now, as v3: a page that shows no text and places one image, and a PNG or JPEG file, are read by a recognizer, `ethos-parser-ocr`, whose words are text runs of derivation `Recognized` under a profile of their own.** Told "Go with all your recommendations, start Phase 1" after Phase 0 measured the census (165 of 3,817 pages need OCR), six recognizers and their determinism (`docs/35-OCR-SCOPE.md` §3), the owner took all six: OCR ahead of assist (amending #15's order, so OCR is v3 and assist v4, and #11); PP-OCRv5 mobile; rten, pure Rust; output byte-identical on every CPU — rten's sums made order-fixed in a vendored `rten-vecmath`, identical on Linux x86-64 and arm64, Windows x86-64 with AVX-512 and macOS arm64 and x86-64, an x86-64 CPU without AVX2 and FMA refused by name; a separate binary on Rust 1.98.0 while the engine stays on 1.88; scans first, photos later. Then: contract §6 kept as it is, so a page with a stray text layer is not read; image files in; the models committed, 12.7 MB, Apache-2.0; the toolchain installed; OCR kept open source. The trigger is a reason code — no text-showing operator, exactly one image placed, no inline image — and every other page that needs OCR is counted under `scanned-page-not-recognized` with its reason. A recognized word is a `text_run`, no new kind, addressed by a new locator variant, `recognized`; the seal refuses a page holding both `Extracted` and `Recognized` text; the profile pins the recognizer's identity, so the OCR profile's hash differs, and the default profile's moved for the new field (`0903edb3` -> `8c2f8672`). An image file's page is its pixel grid, declared under `image-resolution-not-read`; `ground` refuses it by name, as #16 refuses any source its schema does not admit. ParseBench with `--ocr`: 52.10 -> 53.94, 75 pages up and none down — short of the scope's 55.0, which Phase 0's rendering reached; without the flag the eight gate documents and opendataloader-bench's 200 extract byte-identical but the profile hash, and with it they hold the same nodes. Not built: composite pages, pages that need a renderer, CCITT, JBIG2 and JPEG 2000, photos, grounding an image file. | +| 121 | **DECIDED 2026-11-10. ParseBench scores are reused for byte-identical output, and a held-out split is not built.** Told "Do 1 and 2 after Phase 1 ... Come back if they really add value", the two lessons from articles on extraction pipelines were measured on the stored runs of the last six decisions before either was built, and the owner chose to build the second and drop the first. The adapter's opt-in `ETHOS_BENCH_LEDGER` stores each result under the SHA-256 of what ParseBench's scorer receives, less the run's clock and the four fields every engine version changes, and reuses it where they repeat: 93-97% of examples repeat between decisions, scoring is half to two thirds of a run, and none of 13,225 repeated outputs ever scored differently. Checked against fresh scoring, 0 of 2,553 apart; an `--ocr` run scored in 38 s, not 298. A fixed 30% held out never disagreed in sign, could not resolve a typical +0.1 decision, and read lower than the rest on all six for what it holds, not tuning; pages up and down stay the check. `docs/measurements/parsebench/README.md`. | ## 3. Trust ladder — who owns what @@ -197,14 +199,14 @@ Never edit the Ethos repo from this project. Read it for contracts, fixtures and | **v2** | A DOCX quote and an XLSX cell both bind to an address the file states, with no synthesised pages | Office formats through one shared record | | **v2.2** | **Reopened by row 23.** Clause one — a region is emitted wherever the cut divided a page and nowhere else — **met** at 0.42.0. Clause two — a tag this engine writes is one it can read back and ground against — was closed by row 21 and is **open again**, on an attribute object carrying the derivation. **Not built**

**Built 2026-09-17 on the branch for the next version:** [`23-AUTO-TAGGING-SCOPE.md`](23-AUTO-TAGGING-SCOPE.md) (scope), [`24-AUTO-TAGGING-MILESTONES.md`](24-AUTO-TAGGING-MILESTONES.md) (slices); clause two's tests and measurements are in the slices' S3–S4 and [`measurements/auto-tagging/`](measurements/auto-tagging/); the owner's rows #25–#27 are proposed in the scope's §12 and not yet recorded

**Met, recorded 2026-09-17 by row 27**: the roadmap's two clauses are the whole gate, and clause two is met on the fixtures. Row 25 records the shape. It ships in the version after 0.58.0 | Layout and accessibility — geometric block structure, and a tag this engine writes and reads back as computed | | **v2.3** | A user installs a prebuilt binary on any of three platforms and gets a byte-identical artifact on all three. **Added by row 28**; not met | Distribution — prebuilt binaries and packages, with `locate` and the knobs settled beside them | -| **v3** | Assist on and assist off produce identical grounded artifacts | Propose-only VLM assist | -| **v4** | An OCR'd document's fingerprint is provably incomparable with a born-digital parse | OCR under its own profile | +| **v3** | An OCR'd document's fingerprint is provably incomparable with a born-digital parse. **Met by row 120**, as contract §6 rule 4 defines it: an `--ocr` run's profile pins its recognizer, so everything that run writes, a born-digital document's artifact included, hashes apart from any parse without `--ocr` | OCR under its own profile (v4 until row 120) | +| **v4** | Assist on and assist off produce identical grounded artifacts | Propose-only VLM assist (v3 until row 120) | Five versions have full scope and milestone documents: v0, v1, v1.1, v1.2 and v2, and **v2.2's first half has a scope document** — [`16-D4-SCOPE.md`](16-D4-SCOPE.md), by decision #19. **Its second half has none yet.** Row 21 said it never would, because refused work is not work anybody may build; row 23 reversed that, so the sentence changed from *never* to *not yet* and a scope document is now the -next thing owed rather than a thing forbidden. v3 and v4 are still one line each, deliberately. The +next thing owed rather than a thing forbidden. v4 is still one line, deliberately; v3 has its scope, [`35-OCR-SCOPE.md`](35-OCR-SCOPE.md), by row 120. The next thing owed was written on 2026-09-16 — [`23-AUTO-TAGGING-SCOPE.md`](23-AUTO-TAGGING-SCOPE.md), with [`24-AUTO-TAGGING-MILESTONES.md`](24-AUTO-TAGGING-MILESTONES.md) beside it — so *none yet* above diff --git a/docs/01-CONTRACT.md b/docs/01-CONTRACT.md index e22606e..7d78437 100644 --- a/docs/01-CONTRACT.md +++ b/docs/01-CONTRACT.md @@ -230,8 +230,8 @@ without laundering into born-digital certainty. | --- | --- | --- | --- | | **`Extracted`** | Read from the source's own encoding | Text, origins, font identity, `mcid` | The only class v0 produces | | **`Computed`** | Derived deterministically from `Extracted` values by a versioned rule | Reading order, line grouping, boxes from font metrics (§5.3), a structure tag this engine wrote and read back ([`23-AUTO-TAGGING-SCOPE.md`](23-AUTO-TAGGING-SCOPE.md)) | The rule's version is part of the profile | -| **`Recognized`** | Produced by a recognition engine over pixels | OCR text and geometry | v4, own profile. **May author only on canvases where the deterministic reader found no text layer at all** | -| **`Proposed`** | Suggested by a model | Nothing citable, ever | v3. Never evidence | +| **`Recognized`** | Produced by a recognition engine over pixels | OCR text and geometry | v3 (decision #120, `--ocr`), own profile. **May author only on canvases where the deterministic reader found no text layer at all** | +| **`Proposed`** | Suggested by a model | Nothing citable, ever | v4. Never evidence | Four hard rules: @@ -247,7 +247,7 @@ Four hard rules: 4. **Different classes mean different profiles.** An OCR run has a different `profile_sha256`, so its output is non-comparable with a born-digital parse by contract, with no new machinery. -Rules 1 and 2 are the whole of the overwrite rule. The v4 constraint in the `Recognized` row governs +Rules 1 and 2 are the whole of the overwrite rule. The v3 constraint in the `Recognized` row governs *where* a node may be placed, not which classes may replace which, so it is deliberately not encoded as a class-pair rule. @@ -415,10 +415,10 @@ processor-owned and never verifier-owned; it must never appear in a verification deterministic result, or blend into a combined score; and **absence is not confidence** — a processor reporting no uncertainty emits an absent field, never an implied `1.0`. -**This engine has no uncertainty to report.** Every node is `Extracted` by a deterministic reader, -with no recognition step, so the field is absent and §9's prohibition stands unqualified. When the -OCR lane lands at v4 it may populate span-level uncertainty as a **diagnostic** — recorded, and never -filtered on. The bug not to inherit: a surveyed parser drops text below 0.3 and again below 0.1, +**This engine has no uncertainty to report.** Without `--ocr` every node is read by a deterministic +reader with no recognition step; with it (v3, decision #120) the recognizer's confidence is not +written at all, so the field is absent and §9's prohibition stands unqualified. A later OCR slice +may populate span-level uncertainty as a **diagnostic** — recorded, and never filtered on. The bug not to inherit: a surveyed parser drops text below 0.3 and again below 0.1, silently, in two different places. --- diff --git a/docs/02-ROADMAP.md b/docs/02-ROADMAP.md index 1fd2d3a..c7f7b4f 100644 --- a/docs/02-ROADMAP.md +++ b/docs/02-ROADMAP.md @@ -10,8 +10,9 @@ scope document for its first half** (decision #19) **and, since 2026-09-16, a sc document for its second half** ([`23-AUTO-TAGGING-SCOPE.md`](23-AUTO-TAGGING-SCOPE.md), [`24-AUTO-TAGGING-MILESTONES.md`](24-AUTO-TAGGING-MILESTONES.md)). **v2.3** (decision #28) has neither yet; whether its independent items get them as one version or one by one is the owner's -([`OPEN-WORK.md`](OPEN-WORK.md) §4). **v3 and v4 stay one line each**, which is what "one page" -protects. +([`OPEN-WORK.md`](OPEN-WORK.md) §4). **v3, the OCR lane, has a scope document**, +[`35-OCR-SCOPE.md`](35-OCR-SCOPE.md) (decision #120), and **v4 stays one line**, which is what "one +page" protects. --- @@ -27,8 +28,8 @@ protects. | **v2** | Office formats | DOCX, XLSX, PPTX, ODT, ODS, ODP, RTF, EPUB · CSV argued and refused · one shared record and one serializer · embedded assets counted | A DOCX quote and an XLSX cell both **bind** to an address the file states, with **no synthesised pages**. **Met** | | **v2.2** | Layout and accessibility | **Geometric block structure** — the reading-order cut's own column bands, emitted per node as `Computed` layout · **the block cut, shipped 0.55.0** as `TextRunAttributes.block` under `gutter-columns-v3` — the unnamed `Computed` index [`19-BLOCK-SUBDIVISION-SCOPE.md`](19-BLOCK-SUBDIVISION-SCOPE.md) §6 permits, and shipped **without the scope document this page requires before code**; its limits ride on every PDF artifact as `block-subdivision-leading-gap-only` · **auto-tagging, shipping in 0.59.0**: the reader binds an engine-written `/Div` as `Computed` under `struct-tree-v2`, and the writer, the `tag` subcommand, fills absence only — scope [`23-AUTO-TAGGING-SCOPE.md`](23-AUTO-TAGGING-SCOPE.md), slices [`24-AUTO-TAGGING-MILESTONES.md`](24-AUTO-TAGGING-MILESTONES.md) | **Met**, on decision #27 (2026-09-17). Clause one — a region is emitted wherever the cut divided a page and nowhere else — **met** at 0.42.0. Clause two — a tag this engine writes is one it can read back and ground against — was closed by #21, because a `/P` written from a `Computed` cut reads back as `Extracted` and is uncorrectable, and reopened by #23 with an attribute object under `/A` carrying the derivation. **It is met on the fixtures**: the reader binds an engine-written `/Div` as `Computed` (`struct-tree-v2`), the writer `tag` fills absence only, and the block cut is its consumer; S3's round-trip tests pass, and S4 published the round trip over 293 documents — 129 tagged, each read back as computed, projected and grounded as its untagged original — in [`measurements/auto-tagging/`](measurements/auto-tagging/) §2. Decision #25 records the shape and #26 keeps the writer off MCP and the SDKs. It ships in 0.59.0 | | **v2.3** | Distribution | Cross-OS byte identity, compared in CI on Linux, macOS and Windows · GitHub Releases for three platforms, each binary run before it is labelled `verified` · the memory ceiling · **`locate(representation, quote)`**, on the engine's side of the verify boundary (decision #30), scoped in [`26-LOCATE-SCOPE.md`](26-LOCATE-SCOPE.md) and **built**: the core query and `ethos.parser.locations.v0` (S1, with the cap measured at 127.6 MiB — [`measurements/locate/README.md`](measurements/locate/README.md)), then the eleventh subcommand, a fourth MCP tool and `locate()` in both SDKs (S2), slices in [`27-LOCATE-MILESTONES.md`](27-LOCATE-MILESTONES.md) · the knobs, settled (decision #31) · PyPI, npm and crates.io packages, the owner's to publish, with D8 deferred | A user installs a prebuilt binary on any of three platforms and gets a byte-identical artifact on all three. **Added by decision #28** (2026-09-17), which dropped the local plan's second condition, *a quote grounded to its own glyphs*, on [`22-WORD-BOXES-SCOPE.md`](22-WORD-BOXES-SCOPE.md)'s refusal of word boxes. **Met on measurement 2026-09-20, and the distribution step remains.** The `v0.59.0` and `v0.60.0` tag pushes each built, executed and verified four targets — Linux, Windows and both macOS architectures — with one fingerprint across all four runners, which is this row's condition measured on the binaries a user would install. What remains is the install itself: the owner attaches each bundle to its release. Before those runs: *Both workflows exist and have never run* stood until 2026-09-18, when `main` was first pushed with them: **`cross-os byte identity` ran and passed** (run 35318173062, on `791f0fe`) — CI builds on Linux, macOS and Windows produce byte-identical artifacts over the fixture corpus, the operating-system axis of this row's condition. The other half — prebuilt release binaries, each executed where it was built and verified identical across runners — waits for **0.59.0, the first release `release-artifacts.yml` can build**: the same push showed that file was invalid (fixed that day), and the 0.58.0 commit predates it (`RELEASING.md` §8) | -| **v3** | Assist | Propose-only VLM · dual-read and review · formula and chart enrichment as `Recognized` or `Proposed` | Assist on and assist off produce byte-identical grounded artifacts | -| **v4** | OCR lane | Deterministic ONNX OCR in-process · an HTTP OCR contract · its own profile · per-page routing · never overwrites `Extracted` · confidence never filtered on | An OCR fingerprint is provably incomparable with a born-digital parse | +| **v3** | OCR lane | Deterministic ONNX OCR, PP-OCRv5 mobile on rten, in its own binary `ethos-parser-ocr` · its own profile · per-page routing · PNG and JPEG files as one page · never overwrites `Extracted` · no confidence written. Renumbered from v4 by decision #120, which built it first | An OCR fingerprint is provably incomparable with a born-digital parse | +| **v4** | Assist | Propose-only VLM · dual-read and review · formula and chart enrichment as `Recognized` or `Proposed` | Assist on and assist off produce byte-identical grounded artifacts | ### The two gates worth memorising @@ -107,5 +108,6 @@ second half followed the rule**: its scope, [`23-AUTO-TAGGING-SCOPE.md`](23-AUTO and its slices, [`24-AUTO-TAGGING-MILESTONES.md`](24-AUTO-TAGGING-MILESTONES.md), were both written on 2026-09-16, and the code came after both — so the milestones document that waited until the second half was scoped exists now. **v2.3's** documents are pending with the owner -([`OPEN-WORK.md`](OPEN-WORK.md) §4). **v3 and v4 still should not get one** — a scope document for -work in progress is a plan, and one for work not yet started is a guess. +([`OPEN-WORK.md`](OPEN-WORK.md) §4). **v3 got one when it was decided** (decision #120, +[`35-OCR-SCOPE.md`](35-OCR-SCOPE.md)); **v4 still should not** — a scope document for work not yet +started is a guess. diff --git a/docs/04-ARCHITECTURE.md b/docs/04-ARCHITECTURE.md index 78663c4..cbf8ce0 100644 --- a/docs/04-ARCHITECTURE.md +++ b/docs/04-ARCHITECTURE.md @@ -17,14 +17,17 @@ ethos-parser/ │ ├── ethos-parser-office/ # eight office formats │ ├── ethos-parser-grounding/ # representation → ethos.grounding.v1, plus the validator │ └── ethos-parser-cli/ # eleven subcommands; tests/oracle.rs lives here +├── ocr/ # ethos-parser-ocr, the recognizer `--ocr` runs: its own workspace and toolchain ├── vendor/README.md # what is carried, and what deliberately is not ├── fixtures/ # a manifest referencing four corpus roots └── docs/ ``` **A crate exists when a boundary needs enforcing** — not when a module gets large. That is why -`ethos-parser-office` was added rather than a DOCX module going into `ethos-parser-pdf`, and why -`engine-ocr` does not exist yet. +`ethos-parser-office` was added rather than a DOCX module going into `ethos-parser-pdf`, and why the +recognizer is no crate here at all: `ethos-parser-ocr` is its own workspace under `ocr/`, on the +toolchain its runtime needs, behind a process boundary the engine's pin never crosses (decision +#120, [`35-OCR-SCOPE.md`](35-OCR-SCOPE.md) §5). **The oracle test lives with the CLI, not at the workspace root.** Cargo only builds integration tests for packages, so a root `tests/` directory in a virtual workspace would be silently ignored — @@ -195,8 +198,8 @@ values. | Lane | Plugs in as | Impact on what exists | | --- | --- | --- | -| **OCR (v4)** | A node source authoring `Recognized` nodes only where the deterministic reader found no text layer, under its own profile | None. The derivation class exists and the profile hash already isolates | -| **Assist (v3)** | Authors `Proposed` nodes only. Never overwrites, never citable | None | +| **OCR (v3)** | A node source authoring `Recognized` nodes only where the deterministic reader found no text layer, under its own profile | None. The derivation class exists and the profile hash already isolates | +| **Assist (v4)** | Authors `Proposed` nodes only. Never overwrites, never citable | None | | **A new format (v2)** | A new locator variant, adapter profile, fixtures and inspection behaviour | None downstream — provided `ethos-parser-grounding` never learned about pages | | **A second backend** | A trait seam with backend identity in the profile | Design the seam early; implement one side | @@ -206,6 +209,14 @@ every node's parent is a declared page**, which had to be split by locator famil assumption was never in grounding; it was in the contract types, where a page-less document could not become a representation at all. +**What the OCR row cost when it was paid** (decision #120). No node kind, as it said: a recognized +word is a `text_run`. Four line items it did not name — a locator variant, `recognized`, since the +place a word was read is neither a drawn origin nor an object; the profile's recognizer pin, which +the profile hash needed something to hold; the seal's placement check, which makes the `Recognized` +row's canvas rule mechanical; and one accessor, `NativeLocator::run_locator`, through which the +projections read a run's place whichever variant carries it. Grounding was untouched for PDFs; an +image file is a source its schema does not admit, refused by name. + **The processor-identity rule, designed now and implemented when assist exists:** every lane declares its processor identity in the processing run, and a run whose drafting model and representation processor share an identity is **rejected mechanically** rather than caught in review. That is diff --git a/docs/07-VERIFY-BOUNDARY.md b/docs/07-VERIFY-BOUNDARY.md index 180f97d..a93a4aa 100644 --- a/docs/07-VERIFY-BOUNDARY.md +++ b/docs/07-VERIFY-BOUNDARY.md @@ -171,16 +171,17 @@ supported-format claim. ## 7. Optional lanes — OCR, agents, VLM -All of these are out of scope today. The boundary is written now because the type system that makes -them safe is being built now, and because a lane added later without these rules contaminates -everything upstream of it. +OCR is built (v3, decision #120, [`35-OCR-SCOPE.md`](35-OCR-SCOPE.md)): `--ocr` reads a page that +shows no text and places one image, and a PNG or JPEG file, and the rules below are what it was +built to. Agents and VLMs are still out of scope. The boundary was written before any of them +because a lane added without these rules contaminates everything upstream of it. | Lane | Derivation | Profile | May author | Never | | --- | --- | --- | --- | --- | | Deterministic reader | `Extracted` | base | Text, origins, font identity, marked-content ids | — | | Deterministic rules | `Computed` | base, rule version pinned | Reading order, line grouping, ink boxes | Overwrite `Extracted` | -| **OCR (v4)** | `Recognized` | **its own** | Nodes **only on canvases where the deterministic reader found no text layer at all** | Overwrite `Extracted`. Merge into the native stream. Filter on confidence | -| **Assist (v3)** | **`Proposed`** | its own | Suggestions | Be citable. Be evidence. Overwrite anything | +| **OCR (v3)** | `Recognized` | **its own** | Nodes **only on canvases where the deterministic reader found no text layer at all** | Overwrite `Extracted`. Merge into the native stream. Filter on confidence | +| **Assist (v4)** | **`Proposed`** | its own | Suggestions | Be citable. Be evidence. Overwrite anything | **Four rules that make this safe:** diff --git a/docs/35-OCR-SCOPE.md b/docs/35-OCR-SCOPE.md new file mode 100644 index 0000000..f18d7cd --- /dev/null +++ b/docs/35-OCR-SCOPE.md @@ -0,0 +1,197 @@ +# 35 — OCR scope: a scanned page is read, and says so + +**Status: accepted, built (S1-S4).** Written 2026-11-10 against `main` at `0e4eade`, workspace +0.66.0-dev.1. The owner took every recommendation of the Phase 0 measurements ("Go with all your +recommendations, start Phase 1") and the follow-ups in §9, recorded as North Star decision #120. +Where the build departs from the text below, the paragraph will say **Amended** and why. + +Source: the Phase 0 and Phase 1 measurements of 2026-11-08 to 2026-11-10 (census, determinism, +accuracy, speed), summarised in §3. The private corpus named there is used as a bug-finder only and +is not redistributable; every other number is from ParseBench (`run-llama/ParseBench`, Apache-2.0) +or from this repository's own fixtures. + +--- + +## 1. The one sentence + +**With `--ocr`, a page on which the deterministic reader found no text layer at all, and on which +exactly one image was painted, is read by a pinned recognizer whose output is the same bytes on +every supported CPU**; each word it reads is a run of derivation `Recognized`, under the OCR +profile, on that page and nowhere else. + +## 2. Why now + +North Star #11 kept OCR out until a deterministic ONNX lane existed; #45 put a Tesseract pass +outside the engine, in the benchmark adapter. Phase 0 measured what an engine lane would buy and +whether it could be deterministic; Phase 1 built the part that was in doubt. Both answers are yes, +and the owner moved OCR ahead of assist (#120, amending #15's order; by #15's own rule the lane is +renumbered **v3** and assist **v4**). + +## 3. The measurement + +**Census** (3,817 pages: ParseBench, opendataloader-bench, this repository's fixtures). 165 pages +need OCR: ParseBench 161, opendataloader-bench 1, fixtures 3. 115 are one decodable image (46 JPEG, +1 Flate then JPEG, 21 Flate, 5 CCITT, 42 image files); 19 are composites (12 mixed raster content, +6 strips, 1 soft mask); 12 need a renderer; 1 is JPEG 2000; 18 are other. No document mixes scanned +and digital pages. + +**Accuracy** (Phase 0, reference Python pipeline). ParseBench, OCR on the 146 documents the engine +reads no text from or refuses: overall 52.10 → 55.88 with PP-OCRv5 mobile, no page down (Tesseract +55.08, Apple Vision 56.10). Six clean 200-dpi pages, word F1: PP-OCRv5 mobile .947, Tesseract .951. +A private corpus of 290 English pages: .917 (Tesseract .911). Synthetic phone photos, hard +condition: .890 (Tesseract .304). + +**Determinism** (Phase 1). rten 0.26 adds sums per vector lane, so its bits depend on the CPU's +vector width (4 NEON, 8 AVX2, 16 AVX-512). With `rten-vecmath`'s sums made order-fixed (16 partial +sums whatever the width, one fixed tree), PP-OCRv5 mobile gives bit-identical outputs on Apple NEON +and Intel AVX2 at one, four and the default thread count on 222 pages (detection) and 3,062 line +crops (recognition). Against stock rten no detected box and no recognized text changed. The whole +Rust pipeline gives identical output on Linux x86-64 (AVX2), Linux arm64, Windows x86-64 with +AVX-512, and macOS arm64 (22 fixture pages, 1,405 lines). x86-64 without FMA still differs and is +refused. ONNX Runtime was not used: its bits change with the thread count. + +**The Rust pipeline matches the Python one.** Clean pages .947 (Python .947); private corpus .917 +(Python .917); photos equal or better in all six conditions. + +**Cost.** About 4 s per 200-dpi page at four threads on an Apple M4 Pro, against 56 ms for a +digital page; up to 1.85 GB peak. 12.7 MB of models (Apache-2.0). **Phase 1**, with a page's lines +read in parallel: 1.9 s per page and 1.35 GB peak over the 22 fixture pages. + +## 4. What is read + +- **The trigger is a reason code, not a score** (`docs/07-VERIFY-BOUNDARY.md` §7). A PDF page is + read when it carries `classify`'s `scanned` (no text-showing operator at all, and an image) and + exactly one image placement. The coverage threshold the census used to sort pages is not a + condition: one image on a page with no text layer is read whatever its size. +- **The image is decoded by the recognizer**, not the engine: DCT and Flate streams, Flate with + PNG and TIFF predictors; DeviceGray, DeviceRGB, DeviceCMYK, ICC-based by component count, + Indexed, CalGray and CalRGB; 1, 2, 4, 8 and 16 bits per component; `/Decode` and `/ImageMask`. + **Amended:** CCITT was listed here and is not decoded — no decoder is in the dependency tree, and + one is a new dependency or about 500 lines. A CCITT page (5 of ParseBench's) is refused by name, + `filter-not-decoded:CCITTFaxDecode`, as JBIG2 and JPEG 2000 are. +- **Its placement is any of the eight axis-aligned orientations** of the unit square, after the + page's `/Rotate`. The raster is turned upright before it is read and every box is mapped back. +- **An image file is a one-page document.** PNG and JPEG, known by their signatures at byte 0, are + read whole; the page is the file's own pixel grid, one pixel to one point, which the document + limitation `image-resolution-not-read` states. The source is the file — `image/png` or + `image/jpeg` and its SHA-256 — and no image node is written, because the page is the image. + Without `--ocr` an image file is refused by name, with the flag that reads it. **Grounding is not + extended:** `ethos.grounding.v1` admits PDF sources and the page-less office ones only (decision + #16), so `ground` refuses an image file by name. + +## 5. How, in this code + +- **`ocr/` is its own workspace** with its own `rust-toolchain.toml` (1.98.0; rten needs 1.94). + The engine's pin (1.88) does not move, so nothing the engine writes without `--ocr` moves. + `ocr/vendor/rten-vecmath` is rten-vecmath 0.26.0 with its sums order-fixed. `ocr/models` holds the + three model files, built into the binary. +- **`ethos-parser-ocr` speaks over stdin and stdout**: one JSON header line and the image's raw + bytes per request; one JSON line per answer, carrying the recognizer's identity (its version, the + models' SHA-256, the runtime and the pipeline rule id) and the lines with their words, or a named + refusal. It refuses an x86-64 CPU without AVX2 and FMA. It reads a page's lines in parallel: + rten's bits do not depend on the thread count. +- **The engine finds it beside its own executable**, or at `ETHOS_PARSER_OCR`. With `--ocr` and no + recognizer, `extract` fails with a named error; it never falls back to an artifact without OCR + (`docs/07-VERIFY-BOUNDARY.md` §8.6). +- **The engine pins the identity it expects.** An answer from any other recognizer is refused, so + one profile can never hold two recognizers' output. + +## 6. Where it goes on the wire, and what moves + +- **Each recognized word is a `text_run` node** with derivation `Recognized`. No new kind: a run + of text at a place on a page is the fact `text_run` carries, and the North Star's standing rule + is not to add a kind for a fact an existing node carries. What differs is provenance, which the + derivation and the locator state. So the reading-order, line, block, heading and table rules read + recognized words as they read drawn ones — which is how Phase 0's 55.88 was measured: its OCR + words were written into a text layer that the engine then read. +- **A new locator variant, `recognized`**: the run's place on its page as a text layer states it — + the foot of its line under the word's left edge, and the word box's width as the advance — + computed through the placement; the image it came from (its object and generation for a PDF, + none for an image file); and the word's quadrilateral in the image's own pixel grid, as the + recognizer reported it. The run's box is its node's geometry. Its size is the height of the + word's line, as a text layer written by Tesseract states it. A line that does not run left to + right on the page — turned within the image, or upside down — is one run with no advance. +- **Derivation `Recognized`**, always, and never beside `Extracted`: the seal refuses a page that + holds both, so contract §6's placement rule is enforced mechanically, not by convention. +- **The OCR profile** adds the recognizer's identity, so its `profile_sha256` differs from the base + profile's (contract §6 rule 4). Without `--ocr` every artifact is unchanged but the default + profile's hash, which the new field moved (§10.2). +- **Projections** need no rule of their own: Markdown, HTML, grounding and `locate` read recognized + runs as runs, and each span's box is its word's page-space box. Grounding an image file is + refused (§4). +- **No recognizer score is written.** Confidence is a diagnostic at most and never a filter; this + slice has no use for it. + +## 7. Refusals and counted absences + +A page that needs OCR and is not read is counted under a named reason, never dropped in silence: +several images on the page; an inline image; a stream whose filter the recognizer does not decode +(CCITT, JBIG2, JPEG 2000); a placement that is not axis-aligned; a colour space or bit depth it does +not read; an image a soft mask or a mask shapes, since the protocol carries the image alone. A recognizer that refuses the CPU fails `--ocr` by name for the whole document, as a +missing one does. A word whose box's centre falls outside the page's visible box is not written and +is counted; one whose centre is on the page and whose box runs past its edge is written without its +box (`measured_off_page`), as a drawn run is. Two declared limitations ride on every OCR artifact: text in a rotated or upside-down +orientation within an upright image is not turned; a recognized word has no font, so emphasis +(bold, italic) and a font's own size are not read on recognized pages. + +## 8. What it does to `tag`, `overlay`, `ground` and the table rules + +`tag` writes no structure for recognized text and says so. `overlay` draws recognized runs like any +others, and their derivation says what they are. The table, heading and reading-order rules read +recognized runs as runs; a rule that reads a font reads none. + +## 9. What the owner decided + +- 2026-11-10, "Go with all your recommendations, start Phase 1": build OCR now, ahead of v3; + PP-OCRv5 mobile; rten; byte-identical on every CPU; a separate binary on its own toolchain; scans + first, photos later. +- Contract §6: "Keep §6 as is" — pages with a scanner's stray text layer are not read. +- Numbering: "Renumber: OCR v3". +- CI: "Yes, push a CI branch now" (a throwaway branch; its results are in §3). +- Image files: "Yes, include image files". +- Models: "Commit them". +- Toolchain: "Install it". + +## 10. The acceptance bar, set before the code + +1. **Same bytes everywhere.** For the same request, `ethos-parser-ocr` answers identically on Linux + x86-64 with and without AVX-512, Linux arm64, Windows x86-64, macOS arm64 and macOS x86-64, at + one thread and at the default. An x86-64 CPU without FMA is refused by name. **Met** (§3: five + runners, 22 pages, 1,405 lines). +2. **Nothing moves without `--ocr`.** Every gate artifact is byte-identical to `0e4eade`'s. + **Amended:** but the profile hash. The default profile gained `recognizer: not_pinned`, so its + `profile_sha256` moved (`0903edb3…` -> `8c2f8672…`) and every artifact's with it, as every + release's does through `parser_version`. With that hash masked, the eight gate documents and + opendataloader-bench's 200 extract byte-identical to `0e4eade`'s. +3. **With `--ocr`, a document with no scanned page gains no node**; only the profile moves. + **Met:** the same 208 documents under `--ocr` hold the same nodes, and differ by the profile and + its two declared OCR limitations alone. +4. **ParseBench with `--ocr` scores at least 55.0 overall** (52.10 without), and no page scores + lower than without it. **Amended: missed — 53.94, 75 pages up and none down.** The bar was set + from Phase 0's 55.88, which read every textless page by rendering it; this scope reads one + image a page decodes. What the build does not reach is composite pages and pages that need a + renderer (about 1.2 points, Phase 0's figures) and the image files' layout boxes, which need + `ground` to admit an image source (about 0.75): two decisions for the owner, neither taken here. +5. **The placement rule is enforced**: a test that puts a `Recognized` node beside `Extracted` text + fails the seal. **Met**, and the test fails when the seal's check is removed. +6. **Two runs are byte-identical**, as for every artifact, and so are runs at one thread and at the + default. **Met** on six ParseBench scans and three of its image files, `extract --ocr`. +7. **Speed and memory are reported** per page against the 4 s and 1.85 GB of §3, with a target of + at most 4 s median and 2 GB peak for a 200-dpi page on the M4 Pro. **Met:** 1.9 s per page and + 1.35 GB peak (§3). + +## 11. Slices + +| Slice | What | Done when | +| --- | --- | --- | +| S1 | `ocr/` workspace: decoders, pipeline, protocol, identity, models, tests | §10.1 in CI — **done** | +| S2 | Engine: `--ocr`, trigger, request, recognized `text_run` nodes, profile, refusals, projections | §10.2, .3, .5, .6 — **done** (§10.2 amended) | +| S3 | Image files as one-page documents | PNG and JPEG fixtures read — **done** | +| S4 | ParseBench adapter on `--ocr`; docs renumbered (OCR v3, assist v4) | §10.4, .7 — **done**, §10.4 missed | + +## 12. What this scope does not decide + +Photos (perspective, lighting, orientation); composite pages (mixed raster content, strips, soft +masks); CCITT, JPEG 2000 and JBIG2; grounding an image file; pages with a stray text layer (contract §6 as it stands); tables, +headings and emphasis on recognized pages; languages other than English; how the recognizer is +released and installed. diff --git a/docs/CAPABILITY.md b/docs/CAPABILITY.md index d13ed59..aa477c6 100644 --- a/docs/CAPABILITY.md +++ b/docs/CAPABILITY.md @@ -41,6 +41,7 @@ Everything below runs locally, with no network and no renderer. | **Bold and italic** | A run carries `bold` or `italic` where the font that drew it declares the style — a `/FontWeight` of 600 or more, the ForceBold or Italic flag, or `Bold`, `Black`, `Heavy`, `Semibold`, `Demibold`, `Italic` or `Oblique` in its `/BaseFont` (since decision #93 any weight word of 600 or more in its style, `-Demi`) — and since `page-observations-v4` where the TrueType or OpenType program it embeds states one (`OS/2` weight class 600 or more, or `head`'s macStyle where it has no `OS/2`; the italic or oblique bit), or, under a name whose style names neither weight nor slant, a bare CFF or Type 1 program names one (its `Weight`, decision #93), and a run painted fill-then-stroke (rendering mode 2 or 6) is bold; the Markdown and HTML projections write `**`/`*` and ``/``, never inside a heading or a table cell (decision #38). Nothing is read from stem widths or outlines. PDF only; the office readers do not set it yet. On ParseBench semantic formatting moved 0.097 -> 0.349 | `extract`, `markdown`, `html` | | **Superscripts and subscripts** | A run carries `script` — `superscript` or `subscript` — where it is set short (six characters or fewer), at most nine tenths the em of a run beside it on another baseline, and raised or lowered off that baseline by a tenth to half that run's em: a footnote mark, an ordinal's letters, a formula's index. The Markdown and HTML projections write `` and ``, inside any emphasis and never inside a heading or a table cell (decision #40). Read from positions and rendered ems alone; the text is the document's. PDF only. On ParseBench semantic formatting moved 0.3623 -> 0.4060; content faithfulness 0.6633 -> 0.6613 and opendataloader-bench NID 0.8857 -> 0.8835, because both score footnote marks as plain text — a cost the owner accepted | `extract`, `markdown`, `html` | | **Form XObjects** | A `Do` naming a `/Form` runs the form's content stream where it stands — under its `/Matrix`, with its own resources, each time it is drawn, nested to a depth of eight and never into itself — so its text runs and images are the page's, and `classify` counts its text. A run drawn inside one names its font by the resource path that reaches it (`Xf1/F1`). A form drawing itself, one nested past eight and one whose `/Matrix` is not six numbers are not entered and are counted as `form-xobjects-not-descended`; an `/MCID` a form opens binds nothing and is counted as `form-xobject-mcids-not-bound`; `tag` refuses a document showing text through one. **Measured** (decision #37, [`30-FORM-XOBJECTS-SCOPE.md`](30-FORM-XOBJECTS-SCOPE.md) §13): over 2,308 documents no run lost and no new refusal, 566 documents gain 82,285 runs; ParseBench content faithfulness 0.6478 -> 0.6612 | `extract`, `classify` | +| **OCR (`--ocr`)** | A page that shows no text and places one image, and a PNG or JPEG file read as one page, are read by `ethos-parser-ocr` — PP-OCRv5 mobile on rten, the same bytes on every supported CPU — into text runs of derivation `recognized`, under the OCR profile, whose `profile_sha256` differs. Nothing recognized is merged with drawn text, no confidence is written, and a page that needs OCR and is not read is counted under `scanned-page-not-recognized` with its reason (decision #120, [`35-OCR-SCOPE.md`](35-OCR-SCOPE.md)) | `extract --ocr`, `markdown --source --ocr` | | **Determinism** | Two runs over one document produce identical bytes, in every format | CI | --- @@ -52,7 +53,7 @@ Some of these are *not yet*. Others are refusals no version reverses. The middle | Claim | Kind | Why | | --- | --- | --- | | **"v1 is complete"** | done | Closed on decision #18: v1 states the table capability and the band rather than a single macro. Fabrication is 0, the method is pinned in `table-gate-v1.md`, and the geometric chase stays parked | -| **OCR, or a scan read as extracted text** | not yet — v4 | Recognition is a different class of derivation and needs its own trust ladder. An OCR fingerprint must be provably incomparable with a born-digital parse before that lane can exist **Until then, a scan can be read through an OCR pass outside the engine** (decision #45): rasterize its pages (`gs -sDEVICE=png16m -r300`), read each with Tesseract into a PDF (`tesseract page.png page pdf`), and extract that PDF — the text arrives as invisible text, declared under `invisible-render-mode-text`, and the artifact is the OCR'd file's, never the scan's. `docs/measurements/parsebench/ethos_bench.py` runs exactly this under `ETHOS_BENCH_OCR=1`. | +| **OCR of a page `--ocr` does not read** | not yet | `--ocr` reads a page that places one image it can decode (DCT or Flate), and a PNG or JPEG file. Composite pages — mixed raster content, strips, masks — pages that need a renderer, and CCITT, JBIG2 and JPEG 2000 images are refused by name under `scanned-page-not-recognized`; text turned within an upright image is read as it stands; photos are not straightened. `ground` refuses an image file by name: `ethos.grounding.v1` admits PDF and the office sources only (decision #16). **For those pages, a scan can still be read through an OCR pass outside the engine** (decision #45): rasterize its pages (`gs -sDEVICE=png16m -r300`), read each with Tesseract into a PDF (`tesseract page.png page pdf`), and extract that PDF — the text arrives as invisible text, declared under `invisible-render-mode-text`, and the artifact is the OCR'd file's, never the scan's. `docs/measurements/parsebench/ethos_bench.py` runs exactly this under `ETHOS_BENCH_OCR=1`. | | **CSV** | refusal, with named reopening conditions | A CSV parse would not fabricate content — every character would be real. Exactly one field would be false: the record would claim the file *is* a CSV when nobody measured that, and there is nowhere in the record to mark a field as asserted rather than measured. Reopens if either the record gains that distinction, or someone builds a format predicate with a measured false-positive rate | | **Converting office files to PDF to get pages** | refusal | It invents pagination. A DOCX has no pages, and a page on a Word citation measures the printer rather than the document. Not as a fallback, not behind a flag | | **An EPUB's `page-list` as pagination** | refusal | It names pages of a *print* edition. A page record is a page with a width and a height; a publisher's label about someone else's paper has no geometry to validate against | diff --git a/docs/PUBLIC-API.md b/docs/PUBLIC-API.md index 14dc776..db54813 100644 --- a/docs/PUBLIC-API.md +++ b/docs/PUBLIC-API.md @@ -46,10 +46,10 @@ the items re-exported at the crate root. | `html` | `to_html`, `HtmlArtifact`, `HTML_ARTIFACT_TYPE`, `HTML_SCHEMA_VERSION`, `HTML_RULE_BLOCKS_V13` | | `markdown` | `to_markdown`, `AnchorMap`, `Segment`, `SegmentKind`, `Coverage`, `DroppedBucket`, `StructuralErasure`, `MarkdownArtifact`, `MARKDOWN_ARTIFACT_TYPE`, `MARKDOWN_SCHEMA_VERSION`, `MARKDOWN_RULE_BLOCKS_V13`, `GFM_SPAN_SLOTS_UNREPRESENTABLE`, `GFM_ROW_ZERO_SEPARATOR`, `GFM_CELL_RUN_CLAIMED_TWICE`, `GFM_CELL_NOT_PLACED`, `GFM_TABLE_NOT_PROJECTED`, `GFM_LIST_ITEM_RUN_JOINS`, `MCID_RUN_JOINS`, `BASELINE_RUN_JOINS_ABUTTED`, `BASELINE_RUN_JOINS_SPACED`, `LAYOUT_UNIT_LINE_JOINS`, `HEADING_NUMBER_JOINS` (**new with decision #65**), `LAYOUT_UNIT_GAP_JOINS` (**new with decision #68**), `BLOCK_ELEMENT_RUN_JOINS` (**new with decision #84**), `HEADING_LEVEL_UNRESOLVED`, `HEADING_LEVEL_UNREPRESENTABLE` | | `locate` | `locate`, `Locations`, `Occurrence`, `OccurrencePart`, `OccurrencesWithheld`, `Searched`, `LOCATIONS_ARTIFACT_TYPE`, `LOCATIONS_SCHEMA_VERSION`, `LOCATE_RULE_V1`, `LOCATE_MAX_QUOTE_BYTES`, `LOCATE_MAX_OCCURRENCES` — **new at v2.3**. Where a string lies in a representation, and nothing else: no verdict, no score, and a string occurring nowhere is an empty answer rather than an error. The two ceilings are `ethos.grounding.v1`'s own, re-declared because this crate may not depend on the grounding crate and asserted equal by a test that can see both | -| `profile` | `Profile`, `profile_sha256`, `BackendIdentity`, `Capabilities`, `PageBudget`, `XrefRepair`, `VerifierPin`, `TableDetection`, `CMAP_DATA_VERSION`, `READING_ORDER_RULE_V0`, `READING_ORDER_RULE_V1`, `READING_ORDER_RULE_V2`, `READING_ORDER_RULE_V3`, `READING_ORDER_RULE_V4` (**new with decision #32**), `OBSERVATION_RULE_V3`, `OBSERVATION_RULE_V4` (**new with page-observations-v4**), `TEXT_CODE_RULE_V2`, `TEXT_CODE_RULE_V3` (**new with decision #43**), `TEXT_CODE_RULE_V4` (**new with decision #44**), `RasterDpi`, `NOT_RUN`, `DOCX_READING_ORDER_RULE_V2`, `DOCX_TEXT_CODE_RULE_V3`, `XLSX_READING_ORDER_RULE_V1`, `XLSX_TEXT_CODE_RULE_V3`, `PPTX_READING_ORDER_RULE_V1`, `PPTX_TEXT_CODE_RULE_V3`, `ODT_READING_ORDER_RULE_V1`, `ODT_TEXT_CODE_RULE_V2`, `ODS_READING_ORDER_RULE_V1`, `ODS_TEXT_CODE_RULE_V2`, `ODP_READING_ORDER_RULE_V1`, `ODP_TEXT_CODE_RULE_V2`, `RTF_READING_ORDER_RULE_V1`, `RTF_TEXT_CODE_RULE_V2`, `EPUB_READING_ORDER_RULE_V1`, `EPUB_TEXT_CODE_RULE_V2`, `TABLE_DETECTION_V1`, `TABLE_DETECTION_V2`, `TABLE_DETECTION_V3`, `TABLE_DETECTION_V4`, `TABLE_DETECTION_V5`, `TABLE_DETECTION_V6`, `TABLE_DETECTION_V7` (**new with ruled-rects-v7**), `TABLE_DETECTION_V8` (**new with ruled-rects-v8**), `TABLE_DETECTION_V9` (**new with ruled-rects-v9**), `TABLE_DETECTION_V10` (**new with ruled-rects-v10**), `TABLE_DETECTION_V11` (**new with ruled-rects-v11**), `TABLE_DETECTION_V12` (**new with ruled-rects-v12**), `TABLE_DETECTION_V13` (**new with decision #70**), `TABLE_DETECTION_V14` (**new with decision #113**), `TABLE_DETECTION_UNRULED_V1`, `TABLE_DETECTION_STROKE_V1`, `TABLE_DETECTION_STROKE_V2` (**new with decision #72**), `TABLE_DETECTION_TAGGED_V1`, `TABLE_DETECTION_TRACKS_V3`, `TABLE_DETECTION_TRACKS_V4` (**new with whitespace-tracks-v4**), `TABLE_DETECTION_TRACKS_V5` (**new with whitespace-tracks-v5**), `TABLE_DETECTION_TRACKS_V6` (**new with whitespace-tracks-v6**), `TABLE_DETECTION_TRACKS_V7` (**new with decision #49**), `TABLE_DETECTION_TRACKS_V8` (**new with decision #51**), `TABLE_DETECTION_TRACKS_V9` (**new with decision #52**), `TABLE_DETECTION_TRACKS_V10` (**new with decision #53**), `TABLE_DETECTION_TRACKS_V11` (**new with decision #55**), `TABLE_DETECTION_TRACKS_V12` (**new with decision #56**), `TABLE_DETECTION_TRACKS_V13` (**new with decision #58**), `TABLE_DETECTION_TRACKS_V14` (**new with decision #60**), `TABLE_DETECTION_TRACKS_V15` (**new with decision #69**), `TABLE_DETECTION_TRACKS_V16` (**new with decision #112**), `TABLE_DETECTION_CHARTS_V1` (**new with decision #42**), `TABLE_DETECTION_CHARTS_V2` (**new with decision #57**), `TABLE_DETECTION_CHARTS_V3` (**new with decision #66**), `TABLE_DETECTION_LINES_V1`, `TABLE_DETECTION_PIES_V1` (**new with decision #66**), `STRUCT_TREE_RULE_V1`, `STRUCT_TREE_RULE_V2`, `STRUCT_TREE_RULE_V3` (**new with decision #84**), `HEADING_INFERENCE_RULE_V4` (**new with decision #29**), `HEADING_INFERENCE_RULE_V5` (**new with decision #47**), `HEADING_INFERENCE_RULE_V6` (**new with decision #50**), `HEADING_INFERENCE_RULE_V7` (**new with decision #64**), `LAYOUT_UNIT_RULE_V1` (**new with decision #38**), `LAYOUT_UNIT_RULE_V2` (**new with line-units-v2**), `LAYOUT_UNIT_RULE_V3` (**new with line-units-v3**), `LAYOUT_UNIT_RULE_V4` (**new with decision #63**), `LAYOUT_UNIT_RULE_V5`, `LAYOUT_UNIT_RULE_V6` (**new with decision #67**), `FURNITURE_RULE_V1` (**new with decision #41**), `FIGURE_RULE_V1` (**new with decision #46**), `RIGHT_TO_LEFT_RULE_V1` (**new with decision #54**), `FORM_ANNOTATION_RULE_V2` | +| `profile` | `Profile`, `profile_sha256`, `BackendIdentity`, `Capabilities`, `PageBudget`, `XrefRepair`, `VerifierPin`, `RecognizerPin`, `RecognizerIdentity` (**new with decision #120**: which recognizer an OCR profile's runs come from, `Profile::ocr_v0` pinning it), `TableDetection`, `CMAP_DATA_VERSION`, `READING_ORDER_RULE_V0`, `READING_ORDER_RULE_V1`, `READING_ORDER_RULE_V2`, `READING_ORDER_RULE_V3`, `READING_ORDER_RULE_V4` (**new with decision #32**), `OBSERVATION_RULE_V3`, `OBSERVATION_RULE_V4` (**new with page-observations-v4**), `TEXT_CODE_RULE_V2`, `TEXT_CODE_RULE_V3` (**new with decision #43**), `TEXT_CODE_RULE_V4` (**new with decision #44**), `RasterDpi`, `NOT_RUN`, `DOCX_READING_ORDER_RULE_V2`, `DOCX_TEXT_CODE_RULE_V3`, `XLSX_READING_ORDER_RULE_V1`, `XLSX_TEXT_CODE_RULE_V3`, `PPTX_READING_ORDER_RULE_V1`, `PPTX_TEXT_CODE_RULE_V3`, `ODT_READING_ORDER_RULE_V1`, `ODT_TEXT_CODE_RULE_V2`, `ODS_READING_ORDER_RULE_V1`, `ODS_TEXT_CODE_RULE_V2`, `ODP_READING_ORDER_RULE_V1`, `ODP_TEXT_CODE_RULE_V2`, `RTF_READING_ORDER_RULE_V1`, `RTF_TEXT_CODE_RULE_V2`, `EPUB_READING_ORDER_RULE_V1`, `EPUB_TEXT_CODE_RULE_V2`, `TABLE_DETECTION_V1`, `TABLE_DETECTION_V2`, `TABLE_DETECTION_V3`, `TABLE_DETECTION_V4`, `TABLE_DETECTION_V5`, `TABLE_DETECTION_V6`, `TABLE_DETECTION_V7` (**new with ruled-rects-v7**), `TABLE_DETECTION_V8` (**new with ruled-rects-v8**), `TABLE_DETECTION_V9` (**new with ruled-rects-v9**), `TABLE_DETECTION_V10` (**new with ruled-rects-v10**), `TABLE_DETECTION_V11` (**new with ruled-rects-v11**), `TABLE_DETECTION_V12` (**new with ruled-rects-v12**), `TABLE_DETECTION_V13` (**new with decision #70**), `TABLE_DETECTION_V14` (**new with decision #113**), `TABLE_DETECTION_UNRULED_V1`, `TABLE_DETECTION_STROKE_V1`, `TABLE_DETECTION_STROKE_V2` (**new with decision #72**), `TABLE_DETECTION_TAGGED_V1`, `TABLE_DETECTION_TRACKS_V3`, `TABLE_DETECTION_TRACKS_V4` (**new with whitespace-tracks-v4**), `TABLE_DETECTION_TRACKS_V5` (**new with whitespace-tracks-v5**), `TABLE_DETECTION_TRACKS_V6` (**new with whitespace-tracks-v6**), `TABLE_DETECTION_TRACKS_V7` (**new with decision #49**), `TABLE_DETECTION_TRACKS_V8` (**new with decision #51**), `TABLE_DETECTION_TRACKS_V9` (**new with decision #52**), `TABLE_DETECTION_TRACKS_V10` (**new with decision #53**), `TABLE_DETECTION_TRACKS_V11` (**new with decision #55**), `TABLE_DETECTION_TRACKS_V12` (**new with decision #56**), `TABLE_DETECTION_TRACKS_V13` (**new with decision #58**), `TABLE_DETECTION_TRACKS_V14` (**new with decision #60**), `TABLE_DETECTION_TRACKS_V15` (**new with decision #69**), `TABLE_DETECTION_TRACKS_V16` (**new with decision #112**), `TABLE_DETECTION_CHARTS_V1` (**new with decision #42**), `TABLE_DETECTION_CHARTS_V2` (**new with decision #57**), `TABLE_DETECTION_CHARTS_V3` (**new with decision #66**), `TABLE_DETECTION_LINES_V1`, `TABLE_DETECTION_PIES_V1` (**new with decision #66**), `STRUCT_TREE_RULE_V1`, `STRUCT_TREE_RULE_V2`, `STRUCT_TREE_RULE_V3` (**new with decision #84**), `HEADING_INFERENCE_RULE_V4` (**new with decision #29**), `HEADING_INFERENCE_RULE_V5` (**new with decision #47**), `HEADING_INFERENCE_RULE_V6` (**new with decision #50**), `HEADING_INFERENCE_RULE_V7` (**new with decision #64**), `LAYOUT_UNIT_RULE_V1` (**new with decision #38**), `LAYOUT_UNIT_RULE_V2` (**new with line-units-v2**), `LAYOUT_UNIT_RULE_V3` (**new with line-units-v3**), `LAYOUT_UNIT_RULE_V4` (**new with decision #63**), `LAYOUT_UNIT_RULE_V5`, `LAYOUT_UNIT_RULE_V6` (**new with decision #67**), `FURNITURE_RULE_V1` (**new with decision #41**), `FIGURE_RULE_V1` (**new with decision #46**), `RIGHT_TO_LEFT_RULE_V1` (**new with decision #54**), `FORM_ANNOTATION_RULE_V2` | | `derivation` | `DerivationClass`, `GeometryPresence`, `GeometryAbsence` | | `assurance` | `Assurance`, `Limitation`, `LimitationScope`, `PageState`, `PageStateEntry`, `CoverageSummary`, `ProcessingGaps`, `ProcessingTerminalState`, `RefusalCode`, `PageBindingResult`, `page_binding_status`, `codes` | -| `representation` | `DocumentRepresentation`, `RepresentationPayload`, `Node`, `NodeKind`, `NodeGeometry`, `PageRecord`, `NativeLocator`, `PdfLocator`, `DocxLocator`, `XlsxLocator`, `PptxLocator`, `OdtLocator`, `OdsLocator`, `OdpLocator`, `RtfLocator`, `EpubLocator`, `PdfObjectLocator`, `PdfObjectNumber` (**new with decision #84**), `StructuralLocator`, `PdfTaggedLocator`, `PdfArtifactLocator`, `AnnotationRect`, `NodeAttributes`, `OfficeRunAttributes`, `OfficeCellAttributes`, `OfficeSlideRunAttributes`, `OfficeParagraphAttributes`, `OdfBlockKind`, `OfficeOdfCellAttributes`, `OfficeOdfShapeAttributes`, `RtfParagraphAttributes`, `RtfParagraphBreak`, `EpubBlockAttributes`, `OdfValueType`, `OdfCellTextSource`, `CellValueType`, `CellTextSource`, `FormFieldAttributes`, `AnnotationAttributes`, `FieldValue`, `SourceIdentity`, `ProcessingRun`, `ProcessorIdentity`, `SynthesizedAt`, `TextRunAttributes`, `Script` (**new with decision #40**), `Furniture` (**new with decision #41**), `Reading` (**new with decision #54**: a run's text as read, beside the order the page drew; `Node::reading_text` returns it where a run carries one and the text elsewhere), `TextFinding`, `PdfImageLocator`, `PaintedRect`, `ImageAttributes`, `ImageMediaType`, `REPRESENTATION_ARTIFACT_TYPE`, `REPRESENTATION_SCHEMA_VERSION` | +| `representation` | `DocumentRepresentation`, `RepresentationPayload`, `Node`, `NodeKind`, `NodeGeometry`, `PageRecord`, `NativeLocator`, `PdfLocator`, `RecognizedLocator` (**new with decision #120**: a recognized run's place on its page and its pixels in the image read; `NativeLocator::run_locator` reads a run's place whether drawn or recognized), `DocxLocator`, `XlsxLocator`, `PptxLocator`, `OdtLocator`, `OdsLocator`, `OdpLocator`, `RtfLocator`, `EpubLocator`, `PdfObjectLocator`, `PdfObjectNumber` (**new with decision #84**), `StructuralLocator`, `PdfTaggedLocator`, `PdfArtifactLocator`, `AnnotationRect`, `NodeAttributes`, `OfficeRunAttributes`, `OfficeCellAttributes`, `OfficeSlideRunAttributes`, `OfficeParagraphAttributes`, `OdfBlockKind`, `OfficeOdfCellAttributes`, `OfficeOdfShapeAttributes`, `RtfParagraphAttributes`, `RtfParagraphBreak`, `EpubBlockAttributes`, `OdfValueType`, `OdfCellTextSource`, `CellValueType`, `CellTextSource`, `FormFieldAttributes`, `AnnotationAttributes`, `FieldValue`, `SourceIdentity`, `ProcessingRun`, `ProcessorIdentity`, `SynthesizedAt`, `TextRunAttributes`, `Script` (**new with decision #40**), `Furniture` (**new with decision #41**), `Reading` (**new with decision #54**: a run's text as read, beside the order the page drew; `Node::reading_text` returns it where a run carries one and the text elsewhere), `TextFinding`, `PdfImageLocator`, `PaintedRect`, `ImageAttributes`, `ImageMediaType`, `REPRESENTATION_ARTIFACT_TYPE`, `REPRESENTATION_SCHEMA_VERSION` | | `ids` | `NodeId`, `IdAllocator`, `IdKind`, `sort_ids` | | `error` | `EngineError` — the six-variant taxonomy | | `diagnostics` | `Diagnostics`, `DiagnosticsRun`, `HostInfo`, `Stage`, `DIAGNOSTICS_VERSION` — **new at M7** | @@ -144,8 +144,9 @@ Do not persist it as a record or feed it to a fingerprint — `docs/01-CONTRACT. | Kind | Supported items | | --- | --- | | Handle | `Document` — `open`, `open_bytes`, `source_sha256`, `byte_len`, `page_count` | -| Stages | `classify`, `extract`, `to_representation`, `build_overlay`, `write_tags` — the last writes a copy of an untagged PDF carrying this engine's own `/Document`/`/Div` structure tree, marked computed on every element, fills absence only, and reads its output back before returning it (auto-tagging S2, `docs/23-AUTO-TAGGING-SCOPE.md` §3) | -| Stage artifacts | `Classification`, `PageClassification`, `SourceRef`, `ExtractArtifact`, `PageExtract`, `TextRun`, `SynthesizedChar`, `SynthesisReason`, `PdfLocator`, `ImageRecord` | +| Stages | `classify`, `extract`, `extract_recognized` and `extract_image` (**new with decision #120**: `--ocr`, a scanned page read by a recognizer, and a PNG or JPEG file read as one page, which `image_media_type` recognizes by its signature, `docs/35-OCR-SCOPE.md`), `to_representation`, `build_overlay`, `write_tags` — the last writes a copy of an untagged PDF carrying this engine's own `/Document`/`/Div` structure tree, marked computed on every element, fills absence only, and reads its output back before returning it (auto-tagging S2, `docs/23-AUTO-TAGGING-SCOPE.md` §3) | +| Stage artifacts | `Classification`, `PageClassification`, `SourceRef`, `ExtractArtifact`, `PageExtract`, `TextRun`, `SynthesizedChar`, `SynthesisReason`, `PdfLocator`, `ImageRecord`, `RecognizedSource` (**new with decision #120**) | +| Recognizer protocol | `Recognize` — the trait a recognizer implements (the CLI's over `ethos-parser-ocr serve`, a test's as a function) · `RecognitionRequest`, `ImageSpec`, `PdfImageSpec`, `Colour`, `Base`, `FlateParms`, `Recognition`, `RecognizedLine`, `RecognizedWord` — the protocol's request and answer (**new with decision #120**) | | Reason vocabulary | `OcrNeedReason`, `LayoutComplexityReason` | | Format detection | `check_pdf_magic` · `aims_at_the_pdf_reader` | | Modules | `exit` (`SIMPLE`, `NEEDS_ATTENTION`, `COULD_NOT_READ`, `exit_code`) · `limitations` (limitation-code constants and builders) | diff --git a/docs/draft-schemas/document-representation.draft.json b/docs/draft-schemas/document-representation.draft.json index d9a1750..0d7b364 100644 --- a/docs/draft-schemas/document-representation.draft.json +++ b/docs/draft-schemas/document-representation.draft.json @@ -52,9 +52,11 @@ "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.oasis.opendocument.presentation", "application/rtf", - "application/epub+zip" + "application/epub+zip", + "image/png", + "image/jpeg" ], - "$comment": "`application/pdf` at v0, and one of nine since v2-S2: ethos-parser-office writes the other eight. NOT ethos.grounding.v1's source.media_type, which that schema pins to `application/pdf` — decision #16: a DOCX can bind to its source through this field and still cannot be grounded." + "$comment": "`application/pdf` at v0, and one of nine since v2-S2: ethos-parser-office writes the other eight. `image/png` and `image/jpeg` since v3 (decision #120): an image file read with `--ocr` as one page. NOT ethos.grounding.v1's source.media_type, which that schema pins to `application/pdf` — decision #16: a DOCX can bind to its source through this field and still cannot be grounded." }, "sha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } } @@ -160,7 +162,7 @@ "native_locator": { "title": "NativeLocator — required on every node (§5.1)", - "$comment": "A discriminated union: adding a format adds a variant, plus an adapter profile, fixtures and inspection behaviour. It does not change the source, run, artifact or verification models. Externally tagged so a new variant is additive and an unrecognised one fails closed: exactly one key, naming the variant. Lives in ethos-parser-core because the union is part of the CONTRACT — see docs/04-ARCHITECTURE.md §1 on machinery versus vocabulary. ethos-parser-grounding never reads one. The three PDF variants address a page; the eight office ones address a part and positions in it, and none carries a page, a box or a coordinate (docs/history/14-V2-SCOPE.md §3).", + "$comment": "A discriminated union: adding a format adds a variant, plus an adapter profile, fixtures and inspection behaviour. It does not change the source, run, artifact or verification models. Externally tagged so a new variant is additive and an unrecognised one fails closed: exactly one key, naming the variant. Lives in ethos-parser-core because the union is part of the CONTRACT — see docs/04-ARCHITECTURE.md §1 on machinery versus vocabulary. ethos-parser-grounding never reads one. The three PDF variants, and the recognized one (v3), address a page; the eight office ones address a part and positions in it, and none carries a page, a box or a coordinate (docs/history/14-V2-SCOPE.md §3).", "type": "object", "minProperties": 1, "maxProperties": 1, @@ -169,6 +171,7 @@ "pdf": { "$ref": "#/$defs/pdf_locator" }, "pdf_object": { "$ref": "#/$defs/pdf_object_locator" }, "pdf_image": { "$ref": "#/$defs/pdf_image_locator" }, + "recognized": { "$ref": "#/$defs/recognized_locator" }, "docx": { "type": "object", "additionalProperties": false, @@ -268,6 +271,38 @@ } }, + "recognized_locator": { + "title": "A recognized run's address: its place on its page, and where in an image a recognizer read it (v3, decision #120, docs/35-OCR-SCOPE.md §6)", + "$comment": "A variant of its own, never `pdf`: nothing here was drawn. `run` states the place a text layer written by a recognizer states — the foot of the word's line under its left edge, the word's width as the advance — COMPUTED from `pixels` through the matrix the page painted the image with. A node carrying this address is a text_run of derivation `recognized`, and no page holds both such a node and an `extracted` text run: the seal refuses it.", + "type": "object", + "additionalProperties": false, + "required": ["run", "image", "pixels"], + "properties": { + "run": { "$ref": "#/$defs/pdf_locator" }, + "image": { + "description": "The image XObject read; null for an image file, whose page is the image.", + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "additionalProperties": false, + "required": ["object", "generation"], + "properties": { + "object": { "type": "integer", "minimum": 0 }, + "generation": { "type": "integer", "minimum": 0 } + } + } + ] + }, + "pixels": { + "description": "The word's corners in the image's own pixel grid as the recognizer read them: the top-left, top-right, bottom-right and bottom-left of its text, in reading orientation.", + "type": "array", + "minItems": 4, + "maxItems": 4, + "items": { "type": "array", "minItems": 2, "maxItems": 2, "items": { "type": "integer" } } + } + } + }, "pdf_object_locator": { "title": "A PDF object's address: page, object id, and the rectangle it declares (v1-S4)", "$comment": "An annotation or a form field has no baseline, no advance and no character origin, so the glyph-run variant is the wrong shape for it. An object id is both halves.", diff --git a/docs/draft-schemas/extract.draft.json b/docs/draft-schemas/extract.draft.json index 25d2cab..5622ba1 100644 --- a/docs/draft-schemas/extract.draft.json +++ b/docs/draft-schemas/extract.draft.json @@ -28,7 +28,12 @@ ], "properties": { "media_type": { - "const": "application/pdf" + "enum": [ + "application/pdf", + "image/png", + "image/jpeg" + ], + "$comment": "`application/pdf`, or since v3 (decision #120) an image file read with `--ocr` as one page." }, "sha256": { "type": "string", @@ -294,6 +299,20 @@ }, "derivation": { "$ref": "urn:ethos-parser:draft-schema:derivation-class:0" + }, + "recognized": { + "$comment": "Where a recognizer read this run, on a run of derivation `recognized` (v3, decision #120): the image XObject and the word's corners in its pixels — the representation's `recognized` address less the run's place, which `locator` carries. Omitted on every run the page drew.", + "type": "object", + "additionalProperties": false, + "required": ["image", "pixels"], + "properties": { + "image": { + "$ref": "urn:ethos-parser:draft-schema:document-representation:0#/$defs/recognized_locator/properties/image" + }, + "pixels": { + "$ref": "urn:ethos-parser:draft-schema:document-representation:0#/$defs/recognized_locator/properties/pixels" + } + } } } }, diff --git a/docs/draft-schemas/profile.draft.json b/docs/draft-schemas/profile.draft.json index b53c8a1..c94545a 100644 --- a/docs/draft-schemas/profile.draft.json +++ b/docs/draft-schemas/profile.draft.json @@ -35,7 +35,8 @@ "form_annotation_rule", "observation_rule", "raster_dpi", - "text_code_rule" + "text_code_rule", + "recognizer" ], "properties": { "parser_version": { @@ -339,6 +340,61 @@ "text_box_rule": { "type": "string", "$comment": "docs/01-CONTRACT.md §5.3 and §6. The rule that built the text-run boxes. `advance-over-font-envelope-v2` on the PDF profile: along the baseline the pen's advance, across it the font's ascent-to-descent envelope scaled to the rendered em — which is NOT glyph ink, and is what this id exists to say. `-v2` (decision #80) gives a run in a font under `/Identity-V`, which writes down the page, no box, where `-v1` laid one across it. `not-run` on the eight office profiles, which emit no coordinate at all." + }, + "recognizer": { + "type": "object", + "description": "Which recognizer this run's `Recognized` runs come from (v3, docs/35-OCR-SCOPE.md). `not_pinned` is a declared state, not a missing field: no recognizer read any pixels for this run. Pinned only by the OCR profile, by the identity the recognizer reports — not by its binary's digest, which differs between platforms that give the same bytes.", + "required": [ + "mode" + ], + "additionalProperties": false, + "properties": { + "mode": { + "enum": [ + "not_pinned", + "pinned" + ] + }, + "identity": { + "type": "object", + "required": [ + "recognizer", + "version", + "rule", + "det_sha256", + "rec_sha256", + "dict_sha256", + "runtime" + ], + "additionalProperties": false, + "properties": { + "recognizer": { + "type": "string" + }, + "version": { + "type": "string" + }, + "rule": { + "type": "string" + }, + "det_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "rec_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "dict_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "runtime": { + "type": "string" + } + } + } + } } }, "$defs": { @@ -517,8 +573,8 @@ "form_annotation_rule": "form-annotations-v2", "furniture_rule": "margin-bands-v1", "heading_inference_rule": "type-size-v7", - "layout_unit_rule": "line-units-v6", "html_rule": "html-blocks-v13", + "layout_unit_rule": "line-units-v6", "locate_rule": "locate-scalar-exact-v1", "markdown_rule": "markdown-blocks-v13", "observation_rule": "page-observations-v4", @@ -532,6 +588,9 @@ "mode": "not_emitted" }, "reading_order_rule": "gutter-columns-v4", + "recognizer": { + "mode": "not_pinned" + }, "right_to_left_rule": "right-to-left-lines-v1", "struct_tree_rule": "struct-tree-v3", "table_detection": { diff --git a/docs/measurements/parsebench/README.md b/docs/measurements/parsebench/README.md index f056c6f..8d65cc1 100644 --- a/docs/measurements/parsebench/README.md +++ b/docs/measurements/parsebench/README.md @@ -15,14 +15,16 @@ here. ```bash git clone https://github.com/run-llama/ParseBench ~/ethos-external-benchmarks/ParseBench -cd ~/ethos-external-benchmarks/ParseBench && git checkout afb36bd && uv sync +cd ~/ethos-external-benchmarks/ParseBench && git checkout 1f5b37a && uv sync cp /docs/measurements/parsebench/ethos_bench.py . uv run parse-bench download ETHOS_PARSER_BIN=/target/release/ethos-parser uv run python ethos_bench.py run ethos_markdown ``` `--group text_content` (or `table`, `chart`, `layout`, `text_formatting`) runs one dimension; -`--test` runs fifteen pages. A full run takes about three minutes. +`--test` runs fifteen pages. A full run takes about eight minutes, two thirds of it scoring. With +`ETHOS_BENCH_LEDGER=`, an example whose output is byte-identical to one scored before takes the +stored score, and a run that changes a few pages scores in under a minute (decision #121, below). ## What the adapter does @@ -1412,6 +1414,68 @@ them: 137 pages rise and none falls. Overall 42.55 engine-only, 45.47 with the pass. +### The engine's own OCR, `--ocr` (decision #120, 2026-11-10) + +Under `ETHOS_BENCH_OCR=engine` the adapter runs `extract --ocr` on every document, and the engine +reads each page that shows no text and places one image, and each PNG or JPEG file, through +`ethos-parser-ocr` (`docs/35-OCR-SCOPE.md`). On `0e4eade` plus Phase 1, against the same tree +without the flag: + +| Dimension | Without `--ocr` | With `--ocr` | Pages up / down | +| --- | ---: | ---: | ---: | +| Content faithfulness | 0.6750 | 0.7524 | 57 / 0 | +| Semantic formatting | 0.5131 | 0.5261 | 13 / 0 | +| Tables | 0.6453 | 0.6454 | 1 / 0 | +| Charts | 0.1726 | 0.1726 | 0 / 0 | +| Layout | 0.5987 | 0.6004 | 4 / 0 | + +Overall 52.10 -> 53.94; it read 53.91 until PR 53's review found a recognized word whose box ran +past its page's edge refusing `text_ocr__cramer` whole, and the fix let that page be read. **The layout figure counts the image files as zero, and ParseBench's own +aggregate does not:** `ground` refuses an image file by name (decision #16), so the adapter sends +those pages no layout items, the evaluator reports "No layout data for page 0" and leaves them out +of its average, which then reads 0.6415. A page that failed is a page that scored nothing, so the +table divides by all 500. Phase 0's 55.88 read every textless page by rendering it; the +difference is composite pages, pages that need a renderer, and the image files' layout boxes. + +## Scores reused, and a held-out split measured (decision #121, 2026-11-10) + +Two lessons from articles on extraction pipelines, measured on the stored runs of the last six +decisions before either was built (2,553 examples a run). + +**Scores are reused (`ETHOS_BENCH_LEDGER`).** Between one decision and the next, 93-97% of examples +hand the scorer byte-identical output (42% across a Markdown rule's new version), and scoring is +half to two thirds of a run. ParseBench's scorer is deterministic: of 13,225 examples whose output +repeated, none scored differently. So the adapter stores each result under the SHA-256 of what the +scorer receives (the test case and the output, ParseBench's commit and uncommitted diff, its +arguments) less the run's clock and the four fields every engine version changes, and reuses it +where they repeat. Checked against fresh scoring: + +| Run | Extract | Score and report | Scored differently from a fresh run | +| --- | ---: | ---: | --- | +| `0e4eade`, no ledger | 164 s | 307 s | — | +| Same, filling an empty ledger | 168 s | 304 s | 0 of 2,553 | +| `--ocr` after PR 53's fix, reusing it | 327 s | 38 s | `text_ocr__cramer` alone, which the fix reads | + +A ledger holds only what its scorer gave: moving ParseBench changes every key, so the old entries +are never read again. + +**A held-out split was measured and not built.** A fixed 30% of documents, hash-picked and kept +from every census, with its change reported beside each decision's: + +| Decisions | All pages | The 70% | The 30% held out | Any 30%, p5 to p95 | +| --- | ---: | ---: | ---: | ---: | +| #109-#111 | +0.12 | +0.14 | +0.07 | 0.05 to 0.21 | +| #112-#114 | +0.53 | +0.55 | +0.51 | 0.30 to 0.83 | +| #115 | +0.10 | +0.12 | +0.06 | 0.00 to 0.21 | +| #116 | +0.07 | +0.09 | +0.03 | 0.02 to 0.13 | +| #117-#119 | +0.60 | +0.64 | +0.53 | 0.37 to 0.86 | +| #120, scanned pages | +1.82 | +2.03 | +1.32 | 1.21 to 2.45 | + +It never disagreed in sign. A typical decision, +0.07 to +0.12, lies inside what any 30% of the +pages shows by chance, so the split could neither confirm nor refute it; and the fixed 30% read +7-67% lower than the rest on all six though every census read its pages too, which is what it +holds, not tuning. The count of pages up and down beside every decision stays the stronger check. + ## What it found that is not fixed - **Content on readable pages** (decision #61's census, 2026-10-11). Of the content lost on text diff --git a/docs/measurements/parsebench/ethos_bench.py b/docs/measurements/parsebench/ethos_bench.py index 8539a8b..82a96bb 100644 --- a/docs/measurements/parsebench/ethos_bench.py +++ b/docs/measurements/parsebench/ethos_bench.py @@ -14,12 +14,21 @@ role path, an `inferred_heading` run, a detected table, a drawn image, a figure region (decision #46) — and `Text` otherwise. Nothing here infers a role or a unit the engine did not. +`ETHOS_BENCH_OCR=engine` (decision #120) runs `extract --ocr`: the engine's own OCR lane reads every page +that shows no text and places one image (`docs/35-OCR-SCOPE.md`), and needs `ethos-parser-ocr` beside the +binary or named by `ETHOS_PARSER_OCR`. + `ETHOS_BENCH_OCR=1` (decision #45) adds an OCR pass outside the engine, which is never the default: where ethos reads no text from a PDF, or refuses it because no font states its characters, the pages are rasterized (Ghostscript, 300 dpi) and read by Tesseract (`ETHOS_BENCH_OCR_LANG`, default `eng`) into a PDF whose invisible text layer ethos then extracts; an image file ethos does not take goes to Tesseract directly. Results under it are "ethos + OCR pass", reported beside the engine-only ones. Needs `gs`, `tesseract` and `qpdf` on PATH. + +`ETHOS_BENCH_LEDGER=` (decision #121) reuses scores: an example whose output and test case are +byte-identical to one scored before takes the stored result instead of being scored again. The +scores are the same; a run that changes a few pages scores in under a minute instead of five. The +ledger block at the end says what the key holds. """ import json @@ -205,6 +214,12 @@ def run_inference(self, pipeline: PipelineSpec, request: InferenceRequest) -> Ra extract_path.write_text(_run(binary, "extract", _ocr(source, ocr_dir, image))) else: extract_path.write_text(_run(binary, "extract", source)) + elif os.environ.get("ETHOS_BENCH_OCR") == "engine": + # Decision #120: the engine's own OCR lane. Every document is read with `--ocr`, and + # the engine decides page by page: a page that shows no text and places one image is + # read by `ethos-parser-ocr` (beside the binary, or `ETHOS_PARSER_OCR`); every other + # page reads exactly as it does without the flag. + extract_path.write_text(_run(binary, "extract", "--ocr", source)) else: extract_path.write_text(_run(binary, "extract", source)) artifact = json.loads(_run(binary, "markdown", str(extract_path))) @@ -296,5 +311,84 @@ def normalize(self, raw_result: RawInferenceResult) -> InferenceResult: register_layout_adapter("ethos", priority=90)(LiteParseLayoutAdapter) register_layout_label_mapper("ethos", priority=90)(LiteParseLabelMapper) +# The score ledger (decision #121), opt in with `ETHOS_BENCH_LEDGER=`. ParseBench's scorer is a +# function of what it reads, so a result is stored under the SHA-256 of that, and an example whose +# inputs are byte-identical to one scored before takes the stored result instead of being scored +# again. The key is everything the scorer receives — the inference result, the test case, the +# scorer's own arguments, the command line but its output directory, and ParseBench's commit with +# its uncommitted diff — less the run's clock and the four fields every engine version changes, +# which ParseBench never reads. Only a success is stored; a stored result's stats are rebuilt from +# this run's inference, so its latency is this run's. An entry that will not read is scored again, +# and a write that fails stores nothing and keeps the score. Both of ParseBench's paths go through it: +# the worker its process pool runs, and the multi-task path it runs in its own process. +_LEDGER = os.environ.get("ETHOS_BENCH_LEDGER") +if _LEDGER: + import hashlib + + from parse_bench.evaluation import runner as _runner + from parse_bench.evaluation.stats import build_operational_stats + from parse_bench.schemas.evaluation import EvaluationResult + + def _git(*args: str) -> str: + root = Path(_runner.__file__).resolve().parents[3] + return subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, check=True).stdout + + _ARGV = sys.argv[1:] + if "--output_dir" in _ARGV: + _ARGV = _ARGV[: _ARGV.index("--output_dir")] + _ARGV[_ARGV.index("--output_dir") + 2 :] + _SCORER = [_git("rev-parse", "HEAD").strip(), hashlib.sha256(_git("diff", "HEAD").encode()).hexdigest(), *_ARGV] + _RUN_CLOCK = ("started_at", "completed_at", "latency_in_ms") + _VERSION_FIELDS = ("parser_version", "profile_sha256", "representation_sha256", "markdown_rule") + + def _ledgered(inference: dict, rest: list, score) -> dict: + """The stored result for these inputs, or `score()`'s, stored when it succeeded.""" + keyed = {k: v for k, v in inference.items() if k not in _RUN_CLOCK} + if isinstance(keyed.get("raw_output"), dict): + keyed["raw_output"] = {k: v for k, v in keyed["raw_output"].items() if k not in _VERSION_FIELDS} + key = hashlib.sha256(json.dumps([_SCORER, keyed, rest], sort_keys=True, default=str).encode()).hexdigest() + path = Path(_LEDGER) / key[:2] / f"{key}.json" + if path.exists(): + try: + result = json.loads(path.read_text()) + except (OSError, ValueError): # an entry that will not read is scored again + result = None + if result is not None: + stats = build_operational_stats(InferenceResult.model_validate(inference)) + result["stats"] = [s.model_dump(mode="json") for s in stats] + return result + result = EvaluationResult.model_validate(score()).model_dump(mode="json") + if result["success"]: + try: + path.parent.mkdir(parents=True, exist_ok=True) + part = path.with_name(f"{key}.{os.getpid()}.part") + part.write_text(json.dumps(result)) + os.replace(part, path) + except OSError as error: # a full or read-only ledger stores nothing; the score stands + print(f"ethos_bench: score ledger not written: {error}", file=sys.stderr) + return result + + _worker = _runner._evaluate_single_worker + + def _ledgered_worker(inference_result_dict: dict, test_case_dict: dict, *rest): + def score(): + return _worker(inference_result_dict, test_case_dict, *rest) + + return _ledgered(inference_result_dict, [test_case_dict, *rest], score) + + _single = _runner.EvaluationRunner._evaluate_single + + def _ledgered_single(self, inference_result, test_case, evaluator, eval_mode): + if eval_mode != "multi_task": + return _single(self, inference_result, test_case, evaluator, eval_mode) + + def score(): + return _single(self, inference_result, test_case, evaluator, eval_mode).model_dump(mode="json") + + inference = inference_result.model_dump(mode="json") + return EvaluationResult.model_validate(_ledgered(inference, [test_case.model_dump(mode="json"), eval_mode], score)) + + _runner._evaluate_single_worker = _ledgered_worker + _runner.EvaluationRunner._evaluate_single = _ledgered_single + if __name__ == "__main__": sys.exit(main()) diff --git a/ocr/.gitignore b/ocr/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/ocr/.gitignore @@ -0,0 +1 @@ +/target diff --git a/ocr/Cargo.lock b/ocr/Cargo.lock new file mode 100644 index 0000000..cdbfa5e --- /dev/null +++ b/ocr/Cargo.lock @@ -0,0 +1,642 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "ethos-parser-ocr" +version = "0.1.0" +dependencies = [ + "flate2", + "image", + "rayon", + "rten", + "rten-imageproc", + "rten-tensor", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rten" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7741649977427aa13009537ac9a8fe1a503b0aa58784069ce8f117e47eb6ab" +dependencies = [ + "flatbuffers", + "num_cpus", + "rayon", + "rten-base", + "rten-gemm", + "rten-model-file", + "rten-onnx", + "rten-parallel", + "rten-shape-inference", + "rten-simd", + "rten-tensor", + "rten-vecmath", + "rustc-hash", + "smallvec", + "typeid", + "wasm-bindgen", +] + +[[package]] +name = "rten-base" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413dc975c7a3d0e6da0b146cd0387e3d389cceab0fc48328bfa716c7f2ae1b1e" + +[[package]] +name = "rten-gemm" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8efc07c654d4b3f8d81e2efd8b13aea7475568395289e5bb35393411aa6a451e" +dependencies = [ + "rayon", + "rten-base", + "rten-parallel", + "rten-simd", + "rten-tensor", +] + +[[package]] +name = "rten-imageproc" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18726eae9614e23a400560eb3a5e3fdb06526914c31aadaf767f14afde7680dd" +dependencies = [ + "rten-tensor", +] + +[[package]] +name = "rten-model-file" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04ea9e630b377f9249ff3c290466fef282319f86d13d560964209104cd3af542" +dependencies = [ + "flatbuffers", + "rten-base", +] + +[[package]] +name = "rten-onnx" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96944374568990b8d1bbc5bae5957585866f9dd8256130460405730948bfd42e" + +[[package]] +name = "rten-parallel" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba9075b130f81df7be82496ea3a9bbf20fe9e75f106a22b0ece8700850f8f664" +dependencies = [ + "rayon", + "rten-base", +] + +[[package]] +name = "rten-shape-inference" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89289c26b6a5248fccf4d67627596848a03ed31888e6ce4ba6f666c689258122" +dependencies = [ + "rten-tensor", + "smallvec", +] + +[[package]] +name = "rten-simd" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0c01294eb782adca256aa130b48a04006d03763073a68f45a9c33b982ea556" + +[[package]] +name = "rten-tensor" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf72556ba231d7af50238d26dd264856a2149c6369694eb79b22644c8280c81b" +dependencies = [ + "rayon", + "rten-base", + "rten-parallel", + "smallvec", + "typeid", +] + +[[package]] +name = "rten-vecmath" +version = "0.26.0" +dependencies = [ + "rten-base", + "rten-simd", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "smallvec" +version = "1.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/ocr/Cargo.toml b/ocr/Cargo.toml new file mode 100644 index 0000000..ad45a4b --- /dev/null +++ b/ocr/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "ethos-parser-ocr" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false +description = "The recognizer behind ethos-parser --ocr: PP-OCRv5 mobile on rten, the same bytes on every supported CPU" + +# Exact versions: the same-bytes claim was measured on these (docs/35-OCR-SCOPE.md §3). +[dependencies] +rten = "=0.26.0" +rten-imageproc = "=0.26.0" +rten-tensor = "=0.26.0" +image = { version = "=0.25.10", default-features = false, features = ["png", "jpeg"] } +flate2 = "1" +rayon = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +sha2 = "0.11" + +[profile.release] +debug = false + +# Its own workspace and its own toolchain (rust-toolchain.toml): not a member of the engine's. +[workspace] + +# rten-vecmath 0.26.0 with its sums in one fixed order: vendor/rten-vecmath/ETHOS-PATCH.md. +[patch.crates-io] +rten-vecmath = { path = "vendor/rten-vecmath" } diff --git a/ocr/models/MODELS.md b/ocr/models/MODELS.md new file mode 100644 index 0000000..b4de95e --- /dev/null +++ b/ocr/models/MODELS.md @@ -0,0 +1,11 @@ +# Models + +PP-OCRv5 mobile text detection and English recognition, from PaddleOCR (Apache-2.0), in the ONNX +conversion RapidOCR publishes (v3.10.0 on ModelScope, `RapidAI/RapidOCR`). Built into +`ethos-parser-ocr`; their SHA-256 is part of its identity. + +| File | Source path at v3.10.0 | SHA-256 | +| --- | --- | --- | +| `ch_PP-OCRv5_det_mobile.onnx` | `onnx/PP-OCRv5/det/ch_PP-OCRv5_det_mobile.onnx` | `4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae` | +| `en_PP-OCRv5_rec_mobile.onnx` | `onnx/PP-OCRv5/rec/en_PP-OCRv5_rec_mobile.onnx` | `c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8` | +| `ppocrv5_en_dict.txt` | `paddle/PP-OCRv5/rec/en_PP-OCRv5_rec_mobile/ppocrv5_en_dict.txt` | `e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6` | diff --git a/ocr/models/ch_PP-OCRv5_det_mobile.onnx b/ocr/models/ch_PP-OCRv5_det_mobile.onnx new file mode 100644 index 0000000..96c2b33 Binary files /dev/null and b/ocr/models/ch_PP-OCRv5_det_mobile.onnx differ diff --git a/ocr/models/en_PP-OCRv5_rec_mobile.onnx b/ocr/models/en_PP-OCRv5_rec_mobile.onnx new file mode 100644 index 0000000..3ccc12a Binary files /dev/null and b/ocr/models/en_PP-OCRv5_rec_mobile.onnx differ diff --git a/ocr/models/ppocrv5_en_dict.txt b/ocr/models/ppocrv5_en_dict.txt new file mode 100644 index 0000000..7bf3f5f --- /dev/null +++ b/ocr/models/ppocrv5_en_dict.txt @@ -0,0 +1,436 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +: +; +< += +> +? +@ +[ +\ +] +_ +` +{ +| +} +^ +~ +© +® +℉ +№ +Ω +℮ +™ +∆ +✓ +✔ +✗ +✘ +✕ +☑ +☒ +● +▪ +▫ +◼ +▶ +◀ +⬆ +¤ +¦ +§ +¨ +ª +« +¬ +¯ +° +² +³ +´ +µ +¶ +¸ +¹ +º +» +¼ +½ +¾ +¿ +× +‐ +‑ +‒ +— +― +‖ +‗ +‘ +’ +‚ +‛ +“ +” +„ +‟ +† +‡ +‣ +․ +… +‧ +‰ +‴ +‵ +‶ +‷ +‸ +‹ +› +※ +‼ +‽ +‾ +− +₤ +₡ +₹ +₽ +₴ +₿ +¢ +€ +£ +¥ +Ⅰ +Ⅱ +Ⅲ +Ⅳ +Ⅴ +Ⅵ +Ⅶ +Ⅷ +Ⅸ +Ⅹ +Ⅺ +Ⅻ +ⅰ +ⅱ +ⅲ +ⅳ +ⅴ +ⅵ +ⅶ +ⅷ +ⅸ +ⅹ +ⅺ +ⅻ +➀ +➁ +➂ +➃ +➄ +➅ +➆ +➇ +➈ +➉ +➊ +➋ +➌ +➍ +➎ +➏ +➐ +➑ +➒ +➓ +❶ +❷ +❸ +❹ +❺ +❻ +❼ +❽ +❾ +❿ +① +② +③ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +↑ +→ +↓ +↕ +← +↔ +⇒ +⇐ +⇔ +∀ +∃ +∄ +∴ +∵ +∝ +∞ +∩ +∪ +∂ +∫ +∬ +∭ +∮ +∯ +∰ +∑ +∏ +√ +∛ +∜ +∱ +∲ +∳ +∶ +∷ +∼ +∖ +∗ +≈ +≠ +≡ +≤ +≥ +⊂ +⊃ +⊥ +⊾ +⊿ +□ +∥ +∋ +ƒ +′ +″ +À +Á +Â +Ã +Ä +Å +Æ +Ç +È +É +Ê +Ë +Ì +Í +Î +Ï +Ð +Ñ +Ò +Ó +Ô +Õ +Ö +Ø +Ù +Ú +Û +Ü +Ý +Þ +à +á +â +ã +ä +å +æ +ç +è +é +ê +ë +ì +í +î +ï +ð +ñ +ò +ó +ô +õ +ö +ø +ù +ú +û +ü +ý +þ +ÿ +Α +Β +Γ +Δ +Ε +Ζ +Η +Θ +Ι +Κ +Λ +Μ +Ν +Ξ +Ο +Π +Ρ +Σ +Τ +Υ +Φ +Χ +Ψ +Ω +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +σ +ς +τ +υ +φ +χ +ψ +ω +Å +ℏ +⌀ +⍺ +⍵ +𝑢 +𝜓 +० +‥ +︽ +﹥ +• +÷ +∕ +∙ +⋅ +· +± +∓ +∟ +∠ +∡ +∢ +℧ +☺ diff --git a/ocr/rust-toolchain.toml b/ocr/rust-toolchain.toml new file mode 100644 index 0000000..b8e2c79 --- /dev/null +++ b/ocr/rust-toolchain.toml @@ -0,0 +1,8 @@ +# The recognizer's own compiler: rten 0.26 needs Rust 1.94. The engine's pin (1.88, at the +# repository root) does not move, so nothing the engine writes without `--ocr` moves +# (docs/35-OCR-SCOPE.md §5). Bumping this is output-affecting for `--ocr`: it belongs in the +# recognizer's identity, and the determinism tests are re-run on the bump. +[toolchain] +channel = "1.98.0" +components = ["rustfmt", "clippy"] +profile = "minimal" diff --git a/ocr/src/decode.rs b/ocr/src/decode.rs new file mode 100644 index 0000000..65b9f4e --- /dev/null +++ b/ocr/src/decode.rs @@ -0,0 +1,454 @@ +// Copyright 2026 The ethos-parser maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! An image's bytes to an 8-bit BGR raster: a PDF image XObject's stream as stored, or an image +//! file (`docs/35-OCR-SCOPE.md` §4). What is not read is refused by name, never guessed. + +use std::io::Read; + +use serde::Deserialize; + +use crate::pipeline::Bgr; + +/// What a PDF image XObject's dictionary says, as the engine resolved it. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PdfImage { + pub width: u32, + pub height: u32, + pub bits_per_component: u32, + /// Absent for an image mask. + pub colour: Option, + /// `/Filter`, in order. + pub filters: Vec, + /// `/DecodeParms`, one per filter. + pub decode_parms: Vec>, + pub decode: Option>, + pub image_mask: bool, +} + +/// A colour space, resolved by the engine to the components it holds. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(tag = "space", rename_all = "snake_case", deny_unknown_fields)] +pub enum Colour { + Gray, + Rgb, + Cmyk, + Indexed { + base: Base, + hival: u32, + lookup: Vec, + }, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum Base { + Gray, + Rgb, + Cmyk, +} + +impl Base { + fn comps(self) -> usize { + match self { + Base::Gray => 1, + Base::Rgb => 3, + Base::Cmyk => 4, + } + } +} + +/// `/DecodeParms` of a Flate stream (32000-1 Table 8). +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FlateParms { + pub predictor: u32, + pub colors: u32, + pub bits_per_component: u32, + pub columns: u32, +} + +/// An image file: PNG or JPEG, told by its first bytes. +pub fn file(data: &[u8]) -> Result { + let format = match data { + [0x89, b'P', b'N', b'G', ..] => image::ImageFormat::Png, + [0xFF, 0xD8, 0xFF, ..] => image::ImageFormat::Jpeg, + _ => return Err("image-format-not-read".into()), + }; + let img = image::load_from_memory_with_format(data, format) + .map_err(|_| "image-not-decoded".to_string())?; + if !img.color().has_alpha() { + return Ok(from_rgb(img.to_rgb8())); + } + // A viewer shows a transparent pixel over white, so it is read over white: dropped, the alpha + // would turn black text on a transparent black ground all black. Integers, so every CPU agrees. + let rgba = img.to_rgba8(); + let over_white = |c: u8, a: u8| { + let (c, a) = (u32::from(c), u32::from(a)); + ((c * a + 255 * (255 - a) + 127) / 255) as u8 + }; + Ok(from_rgb(image::RgbImage::from_fn( + rgba.width(), + rgba.height(), + |x, y| { + let [r, g, b, a] = rgba.get_pixel(x, y).0; + image::Rgb([over_white(r, a), over_white(g, a), over_white(b, a)]) + }, + ))) +} + +fn from_rgb(rgb: image::RgbImage) -> Bgr { + let (w, h) = (rgb.width() as usize, rgb.height() as usize); + let px = rgb + .as_raw() + .as_chunks::<3>() + .0 + .iter() + .flat_map(|&[r, g, b]| [b, g, r]) + .collect(); + Bgr { w, h, px } +} + +/// A PDF image XObject's stream, as stored, under `spec`. +pub fn pdf(spec: &PdfImage, data: &[u8]) -> Result { + let (w, h) = (spec.width as usize, spec.height as usize); + if w == 0 || h == 0 { + return Err("image-empty".into()); + } + let mut bytes = data.to_vec(); + for (i, f) in spec.filters.iter().enumerate() { + match f.as_str() { + "FlateDecode" | "Fl" => { + let mut out = Vec::new(); + flate2::read::ZlibDecoder::new(&bytes[..]) + .read_to_end(&mut out) + .map_err(|_| "stream-not-decoded".to_string())?; + bytes = match spec.decode_parms.get(i).cloned().flatten() { + Some(p) if p.predictor > 1 => unpredict(&out, &p)?, + _ => out, + }; + } + "DCTDecode" | "DCT" if i + 1 == spec.filters.len() => return dct(spec, &bytes), + other => return Err(format!("filter-not-decoded:{other}")), + } + } + samples(spec, &bytes) +} + +/// A JPEG stream: gray or RGB, inverted where `/Decode` says so. +fn dct(spec: &PdfImage, data: &[u8]) -> Result { + let comps = match &spec.colour { + Some(Colour::Gray) => 1, + Some(Colour::Rgb) => 3, + _ => return Err("colour-space-not-read:jpeg".into()), + }; + let img = image::load_from_memory_with_format(data, image::ImageFormat::Jpeg) + .map_err(|_| "stream-not-decoded".to_string())?; + let mut out = from_rgb(img.to_rgb8()); + if (out.w, out.h) != (spec.width as usize, spec.height as usize) { + return Err("image-size-mismatch".into()); + } + match spec.decode.as_deref() { + None => {} + Some(d) if d.chunks(2).all(|p| p == [0.0, 1.0]) => {} + Some(d) if d.len() == 2 * comps && d.chunks(2).all(|p| p == [1.0, 0.0]) => { + out.px.iter_mut().for_each(|v| *v = 255 - *v); + } + Some(_) => return Err("decode-array-not-read".into()), + } + Ok(out) +} + +/// Undo a PNG (10-15) or TIFF (2, 8 bits) predictor. +fn unpredict(data: &[u8], p: &FlateParms) -> Result, String> { + // A row's bits past u32, or a row of no bytes, is the page refused by name, not a panic that + // stops `serve` and with it the whole document. + let size = || "predictor-not-read:size".to_string(); + let bits = p + .colors + .checked_mul(p.bits_per_component) + .ok_or_else(size)?; + let bpp = (bits as usize).div_ceil(8).max(1); + let row = (bits.checked_mul(p.columns).ok_or_else(size)? as usize).div_ceil(8); + if row == 0 { + return Err(size()); + } + if p.predictor == 2 { + if p.bits_per_component != 8 { + return Err("predictor-not-read:2".into()); + } + let mut out = data.to_vec(); + for r in out.chunks_mut(row) { + for i in bpp..r.len() { + r[i] = r[i].wrapping_add(r[i - bpp]); + } + } + return Ok(out); + } + if !(10..=15).contains(&p.predictor) { + return Err(format!("predictor-not-read:{}", p.predictor)); + } + let mut out = Vec::with_capacity(data.len()); + let mut prev = vec![0u8; row]; + for chunk in data.chunks(row + 1) { + if chunk.len() < row + 1 { + break; + } + let (kind, src) = (chunk[0], &chunk[1..]); + let mut cur = vec![0u8; row]; + for i in 0..row { + let a = if i >= bpp { cur[i - bpp] as i16 } else { 0 }; + let b = prev[i] as i16; + let c = if i >= bpp { prev[i - bpp] as i16 } else { 0 }; + let pred = match kind { + 0 => 0, + 1 => a, + 2 => b, + 3 => (a + b) / 2, + 4 => { + let pp = a + b - c; + let (pa, pb, pc) = ((pp - a).abs(), (pp - b).abs(), (pp - c).abs()); + if pa <= pb && pa <= pc { + a + } else if pb <= pc { + b + } else { + c + } + } + _ => return Err("predictor-not-read:png-row".into()), + }; + cur[i] = src[i].wrapping_add(pred as u8); + } + out.extend_from_slice(&cur); + prev = cur; + } + Ok(out) +} + +/// Unpacked samples to BGR, through the colour space and `/Decode`. +fn samples(spec: &PdfImage, data: &[u8]) -> Result { + let (w, h, bpc) = ( + spec.width as usize, + spec.height as usize, + spec.bits_per_component, + ); + if !matches!(bpc, 1 | 2 | 4 | 8 | 16) { + return Err(format!("bits-per-component-not-read:{bpc}")); + } + let comps = match (&spec.colour, spec.image_mask) { + (_, true) => 1, + (Some(Colour::Gray), _) => 1, + (Some(Colour::Rgb), _) => 3, + (Some(Colour::Cmyk), _) => 4, + (Some(Colour::Indexed { .. }), _) => 1, + (None, false) => return Err("colour-space-not-read".into()), + }; + let stride = (w * comps * bpc as usize).div_ceil(8); + if data.len() < stride * h { + return Err("stream-short".into()); + } + let max = ((1u32 << bpc.min(16)) - 1) as f64; + let sample = |row: &[u8], i: usize| -> u32 { + match bpc { + 8 => row[i] as u32, + 16 => u16::from_be_bytes([row[2 * i], row[2 * i + 1]]) as u32, + _ => { + let bit = i * bpc as usize; + let byte = row[bit / 8] as u32; + (byte >> (8 - bpc as usize - bit % 8)) & ((1 << bpc) - 1) + } + } + }; + // Decode: component value v of `max` maps to dmin + v * (dmax - dmin) / max. An index's + // default is [0 max], the identity (32000-1 Table 90). + let default: Vec = match &spec.colour { + Some(Colour::Indexed { .. }) if !spec.image_mask => vec![0.0, max], + _ => [0.0, 1.0].repeat(comps), + }; + let dec = spec.decode.clone().unwrap_or(default); + if dec.len() != 2 * comps { + return Err("decode-array-not-read".into()); + } + let to8 = |v: u32, k: usize| -> u8 { + let d = dec[2 * k] + v as f64 * (dec[2 * k + 1] - dec[2 * k]) / max; + (d * 255.0).round_ties_even().clamp(0.0, 255.0) as u8 + }; + let mut px = vec![0u8; w * h * 3]; + for y in 0..h { + let row = &data[y * stride..(y + 1) * stride]; + for x in 0..w { + let bgr: [u8; 3] = if spec.image_mask { + // Sample 0 paints under the default /Decode [0 1]; [1 0] makes 1 paint. + let paint = (sample(row, x) == 0) == (dec[0] == 0.0); + if paint { + [0, 0, 0] + } else { + [255, 255, 255] + } + } else { + match spec.colour.as_ref().ok_or("colour-space-not-read")? { + Colour::Gray => { + let g = to8(sample(row, x), 0); + [g, g, g] + } + Colour::Rgb => { + let (r, g, b) = ( + to8(sample(row, 3 * x), 0), + to8(sample(row, 3 * x + 1), 1), + to8(sample(row, 3 * x + 2), 2), + ); + [b, g, r] + } + Colour::Cmyk => { + let c: [u8; 4] = std::array::from_fn(|k| to8(sample(row, 4 * x + k), k)); + cmyk(c) + } + Colour::Indexed { + base, + hival, + lookup, + } => { + let d = dec[0] + sample(row, x) as f64 * (dec[1] - dec[0]) / max; + let i = (d.round_ties_even().clamp(0.0, *hival as f64)) as usize; + let n = base.comps(); + let e = lookup + .get(i * n..(i + 1) * n) + .ok_or("indexed-lookup-short")?; + match base { + Base::Gray => [e[0], e[0], e[0]], + Base::Rgb => [e[2], e[1], e[0]], + Base::Cmyk => cmyk([e[0], e[1], e[2], e[3]]), + } + } + } + }; + px[(y * w + x) * 3..(y * w + x) * 3 + 3].copy_from_slice(&bgr); + } + } + Ok(Bgr { w, h, px }) +} + +/// CMYK to BGR, the naive way (32000-1 §10.4.2 with no undercolour removal). +fn cmyk([c, m, y, k]: [u8; 4]) -> [u8; 3] { + let inv = |v: u8| 255 - (v as u16 + k as u16).min(255) as u8; + [inv(y), inv(m), inv(c)] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn spec(w: u32, h: u32, bpc: u32, colour: Option) -> PdfImage { + PdfImage { + width: w, + height: h, + bits_per_component: bpc, + colour, + filters: vec![], + decode_parms: vec![], + decode: None, + image_mask: false, + } + } + + #[test] + fn one_bit_gray_reads_zero_as_black() { + let img = pdf(&spec(8, 1, 1, Some(Colour::Gray)), &[0b1010_0000]).unwrap(); + let first: Vec = img.px.chunks(3).map(|p| p[0]).collect(); + assert_eq!(first, [255, 0, 255, 0, 0, 0, 0, 0]); + } + + #[test] + fn an_image_mask_paints_its_zeros_unless_decode_inverts() { + let mut s = spec(4, 1, 1, None); + s.image_mask = true; + let img = pdf(&s, &[0b0101_0000]).unwrap(); + assert_eq!( + img.px.chunks(3).map(|p| p[0]).collect::>(), + [0, 255, 0, 255] + ); + s.decode = Some(vec![1.0, 0.0]); + let img = pdf(&s, &[0b0101_0000]).unwrap(); + assert_eq!( + img.px.chunks(3).map(|p| p[0]).collect::>(), + [255, 0, 255, 0] + ); + } + + #[test] + fn png_predictors_round_trip() { + // Two rows of three gray bytes, filtered Sub then Up. + let p = FlateParms { + predictor: 15, + colors: 1, + bits_per_component: 8, + columns: 3, + }; + let data = [1, 10, 5, 5, 2, 1, 1, 1]; + assert_eq!(unpredict(&data, &p).unwrap(), [10, 15, 20, 11, 16, 21]); + } + + #[test] + fn a_transparent_pixel_is_read_over_white() { + // PR 53's review: with the alpha dropped, black text on a transparent black ground was all + // black. A transparent black pixel then an opaque one: white, then black. + let mut png = Vec::new(); + image::RgbaImage::from_raw(2, 1, vec![0, 0, 0, 0, 0, 0, 0, 255]) + .unwrap() + .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) + .unwrap(); + assert_eq!(file(&png).unwrap().px, [255, 255, 255, 0, 0, 0]); + } + + #[test] + fn a_predictor_row_past_u32_or_of_no_bytes_is_refused_by_name() { + // PR 53's review: 2^29 columns of one 8-bit colour is 2^32 bits a row, which wrapped to a + // row of no bytes, and `chunks_mut(0)` panicked; `/Columns 0` reached the same panic. + let mut p = FlateParms { + predictor: 2, + colors: 1, + bits_per_component: 8, + columns: 1 << 29, + }; + assert_eq!( + unpredict(&[0; 8], &p).unwrap_err(), + "predictor-not-read:size" + ); + p.columns = 0; + assert_eq!( + unpredict(&[0; 8], &p).unwrap_err(), + "predictor-not-read:size" + ); + } + + #[test] + fn indexed_rgb_looks_up_its_palette() { + let c = Colour::Indexed { + base: Base::Rgb, + hival: 1, + lookup: vec![255, 0, 0, 0, 0, 255], + }; + let img = pdf(&spec(2, 1, 8, Some(c)), &[1, 0]).unwrap(); + assert_eq!(img.px, [255, 0, 0, 0, 0, 255]); + } + + #[test] + fn an_unread_filter_is_refused_by_name() { + let mut s = spec(1, 1, 8, Some(Colour::Gray)); + s.filters = vec!["JPXDecode".into()]; + assert_eq!(pdf(&s, &[0]).unwrap_err(), "filter-not-decoded:JPXDecode"); + } +} diff --git a/ocr/src/main.rs b/ocr/src/main.rs new file mode 100644 index 0000000..7ba650d --- /dev/null +++ b/ocr/src/main.rs @@ -0,0 +1,333 @@ +// Copyright 2026 The ethos-parser maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! `ethos-parser-ocr`: the recognizer behind `ethos-parser --ocr` (`docs/35-OCR-SCOPE.md`). +//! +//! ```text +//! ethos-parser-ocr serve requests on stdin, answers on stdout, until stdin closes +//! ethos-parser-ocr identity this recognizer's identity, one JSON line +//! ethos-parser-ocr image FILE... read PNG or JPEG files, one answer per file +//! ``` +//! +//! **A request** is one JSON line, then exactly `bytes` raw bytes: +//! `{"ocr_request":1,"id":"..","image":{"kind":"file"}|{"kind":"pdf",..},"orientation":0,"bytes":N}`. +//! `orientation` stands the stored raster upright: mirrored left to right first when 4 or more, +//! then turned `orientation % 4` quarter turns clockwise. +//! +//! **An answer** is one JSON line: the request's `id`, this recognizer's identity, the stored +//! raster's size, and its lines — text, and the quad of the text's top-left, top-right, +//! bottom-right and bottom-left corners as read, in the stored raster's pixels — or a named +//! refusal. No score is written: what a recognizer reports about its own certainty is never a +//! filter (`docs/07-VERIFY-BOUNDARY.md` §7). +//! +//! The same request gets the same bytes on every CPU this runs on; an x86-64 CPU without AVX2 and +//! FMA computes differently and is refused before anything is read. + +mod decode; +mod pipeline; + +use std::io::{BufRead, Read, Write}; +use std::process::ExitCode; + +use serde::{Deserialize, Serialize}; + +use pipeline::{Bgr, Line, Quad, Recognizer}; + +/// The pipeline's rule id: moves whenever any step around the models changes what is written. +pub const RULE: &str = "ppocrv5-mobile-v1"; +pub const DET_SHA256: &str = "4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae"; +pub const REC_SHA256: &str = "c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8"; +pub const DICT_SHA256: &str = "e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6"; +pub const RUNTIME: &str = "rten 0.26.0; rten-vecmath 0.26.0 with fixed-order sums"; + +#[derive(Serialize, Clone)] +struct Identity { + recognizer: &'static str, + version: &'static str, + rule: &'static str, + det_sha256: &'static str, + rec_sha256: &'static str, + dict_sha256: &'static str, + runtime: &'static str, +} + +fn identity() -> Identity { + Identity { + recognizer: "ethos-parser-ocr", + version: env!("CARGO_PKG_VERSION"), + rule: RULE, + det_sha256: DET_SHA256, + rec_sha256: REC_SHA256, + dict_sha256: DICT_SHA256, + runtime: RUNTIME, + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Request { + ocr_request: u32, + id: String, + image: ImageSpec, + orientation: u8, + bytes: u64, +} + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum ImageSpec { + File, + Pdf(decode::PdfImage), +} + +#[derive(Serialize)] +struct LineOut { + text: String, + quad: Quad, + words: Vec, +} + +#[derive(Serialize)] +struct WordOut { + text: String, + quad: Quad, +} + +#[derive(Serialize)] +struct Answer { + ocr_response: u32, + id: String, + identity: Identity, + width: usize, + height: usize, + lines: Vec, + refused: Option, +} + +/// The CPU check: rten's kernels compute differently without AVX2 and FMA. +fn refusal_for_cpu() -> Option<&'static str> { + #[cfg(target_arch = "x86_64")] + if !(is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma")) { + return Some("cpu-lacks-avx2-fma"); + } + None +} + +/// Stand `img` upright as `orientation` says. +fn upright(img: &Bgr, orientation: u8) -> Bgr { + let m = if orientation >= 4 { + pipeline::mirror(img) + } else { + img.clone() + }; + pipeline::turn(&m, orientation % 4) +} + +/// A point of the upright raster, back in the stored raster's pixels. `w` and `h` are the stored +/// raster's size; points are pixel-corner coordinates. +fn stored_point(p: [i32; 2], orientation: u8, w: usize, h: usize) -> [i32; 2] { + let turns = orientation % 4; + // The size after `turns` quarter turns, undone one at a time: a clockwise turn maps (x, y) of a + // raster of width cw to (cw' - y, x), so its inverse maps (x, y) to (y, cw' - x). + let (mut x, mut y) = (p[0], p[1]); + let (mut cw, mut chh) = if turns % 2 == 1 { + (h as i32, w as i32) + } else { + (w as i32, h as i32) + }; + for _ in 0..turns { + (x, y) = (y, cw - x); + (cw, chh) = (chh, cw); + } + if orientation >= 4 { + x = w as i32 - x; + } + [x, y] +} + +fn answer(rec: &Recognizer, id: String, stored: Result, orientation: u8) -> Answer { + let (img, refused) = match stored { + Ok(img) => (Some(img), None), + Err(why) => (None, Some(why)), + }; + let (w, h) = img.as_ref().map_or((0, 0), |i| (i.w, i.h)); + let mut lines = Vec::new(); + let mut refused = refused; + if let Some(img) = img { + match rec.read(&upright(&img, orientation)) { + Ok(read) => { + let back = |q: Quad| q.map(|p| stored_point(p, orientation, w, h)); + for Line { text, quad, words } in read { + let words = words + .into_iter() + .map(|wd| WordOut { + text: wd.text, + quad: back(wd.quad), + }) + .collect(); + lines.push(LineOut { + text, + quad: back(quad), + words, + }); + } + } + Err(e) => refused = Some(format!("recognizer-failed:{e}")), + } + } + Answer { + ocr_response: 1, + id, + identity: identity(), + width: w, + height: h, + lines, + refused, + } +} + +fn serve(rec: &Recognizer) -> Result<(), String> { + let stdin = std::io::stdin(); + let mut input = stdin.lock(); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + loop { + let mut header = String::new(); + if input.read_line(&mut header).map_err(|e| e.to_string())? == 0 { + return Ok(()); + } + let req: Request = serde_json::from_str(header.trim_end()) + .map_err(|e| format!("request-not-read: {e}"))?; + if req.ocr_request != 1 { + return Err(format!("request-version-not-read: {}", req.ocr_request)); + } + if req.orientation > 7 { + return Err(format!("orientation-not-read: {}", req.orientation)); + } + let mut data = vec![0u8; req.bytes as usize]; + input + .read_exact(&mut data) + .map_err(|e| format!("request-bytes-short: {e}"))?; + let stored = match &req.image { + ImageSpec::File => decode::file(&data), + ImageSpec::Pdf(spec) => decode::pdf(spec, &data), + }; + let a = answer(rec, req.id, stored, req.orientation); + serde_json::to_writer(&mut out, &a).map_err(|e| e.to_string())?; + out.write_all(b"\n") + .and_then(|_| out.flush()) + .map_err(|e| e.to_string())?; + } +} + +fn main() -> ExitCode { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str); + if cmd == Some("identity") { + println!( + "{}", + serde_json::to_string(&identity()).expect("identity serializes") + ); + return ExitCode::SUCCESS; + } + if let Some(why) = refusal_for_cpu() { + eprintln!( + "ethos-parser-ocr: refused: {why}: OCR output on this CPU would differ from other CPUs" + ); + return ExitCode::from(3); + } + let rec = match Recognizer::new() { + Ok(r) => r, + Err(e) => { + eprintln!("ethos-parser-ocr: {e}"); + return ExitCode::from(1); + } + }; + let result = match cmd { + Some("serve") if args.len() == 2 => serve(&rec), + Some("image") if args.len() > 2 => { + let mut out = std::io::stdout().lock(); + args[2..].iter().try_for_each(|path| { + let stored = std::fs::read(path) + .map_err(|e| format!("{path}: {e}")) + .and_then(|d| decode::file(&d)); + let a = answer(&rec, path.clone(), stored, 0); + serde_json::to_writer(&mut out, &a).map_err(|e| e.to_string())?; + out.write_all(b"\n").map_err(|e| e.to_string()) + }) + } + _ => Err("usage: ethos-parser-ocr serve | identity | image FILE...".into()), + }; + match result { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("ethos-parser-ocr: {e}"); + ExitCode::from(2) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_identity_names_the_models_built_in() { + use sha2::{Digest, Sha256}; + let hex = |b: &[u8]| { + Sha256::digest(b) + .iter() + .map(|x| format!("{x:02x}")) + .collect::() + }; + assert_eq!(hex(pipeline::DET), DET_SHA256); + assert_eq!(hex(pipeline::REC), REC_SHA256); + assert_eq!(hex(pipeline::DICT.as_bytes()), DICT_SHA256); + } + + #[test] + fn points_return_to_the_stored_grid() { + // A 4 x 2 raster: each orientation's upright image maps its corners back to the stored ones. + let (w, h) = (4usize, 2usize); + for o in 0..8u8 { + let img = Bgr { + w, + h, + px: vec![0; w * h * 3], + }; + let up = upright(&img, o); + let corners = [ + [0, 0], + [up.w as i32, 0], + [up.w as i32, up.h as i32], + [0, up.h as i32], + ]; + let mut back: Vec<[i32; 2]> = + corners.iter().map(|&p| stored_point(p, o, w, h)).collect(); + back.sort(); + assert_eq!( + back, + vec![[0, 0], [0, 2], [4, 0], [4, 2]], + "orientation {o}" + ); + } + } + + #[test] + fn a_clockwise_turn_maps_the_upright_top_left_to_the_stored_bottom_left() { + assert_eq!(stored_point([0, 0], 1, 4, 2), [0, 2]); + assert_eq!(stored_point([0, 0], 0, 4, 2), [0, 0]); + assert_eq!(stored_point([0, 0], 4, 4, 2), [4, 0]); + } +} diff --git a/ocr/src/pipeline.rs b/ocr/src/pipeline.rs new file mode 100644 index 0000000..d147e9c --- /dev/null +++ b/ocr/src/pipeline.rs @@ -0,0 +1,704 @@ +// Copyright 2026 The ethos-parser maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PP-OCRv5 mobile, detection then recognition (`docs/35-OCR-SCOPE.md`). +//! +//! The steps around the two models follow PaddleOCR's reference pipeline, measured in Phase 0, +//! except where a step would call a platform maths library or depend on the thread count: the +//! line crop is an affine bilinear warp (PaddleOCR: a perspective bicubic one), each line is read +//! on its own (PaddleOCR: batches of six, which couples a line's result to its neighbours'), and +//! unclip grows the box by `area * 1.5 / perimeter` on every side, whose minimum-area box is +//! pyclipper's. Everything here is integer or IEEE arithmetic in a fixed order, so the output +//! depends only on the input bytes; the models run on rten with fixed-order sums +//! (`vendor/rten-vecmath/ETHOS-PATCH.md`). + +use rayon::prelude::*; +use rten::Model; +use rten_imageproc::{find_contours, RetrievalMode}; +use rten_tensor::prelude::*; +use rten_tensor::{NdTensor, Tensor}; + +/// PaddleOCR's normalisation, applied in the image's BGR channel order. +const MEAN: [f32; 3] = [0.485, 0.456, 0.406]; +const STD: [f32; 3] = [0.229, 0.224, 0.225]; +const THRESH: f32 = 0.3; +const BOX_THRESH: f64 = 0.6; +const UNCLIP: f64 = 1.5; +const MAX_CANDIDATES: usize = 1000; +const MIN_SIZE: f64 = 3.0; +const REC_H: usize = 48; +const REC_W: usize = 320; + +/// An 8-bit image, rows of BGR pixels. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Bgr { + pub w: usize, + pub h: usize, + pub px: Vec, +} + +impl Bgr { + fn at(&self, x: usize, y: usize, c: usize) -> u8 { + self.px[(y * self.w + x) * 3 + c] + } +} + +/// A line's four corners in pixels, clockwise from top-left. +pub type Quad = [[i32; 2]; 4]; + +/// One line read: its text as recognized, its quad, and its words. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Line { + pub text: String, + pub quad: Quad, + pub words: Vec, +} + +/// A word: the line's characters between spaces, and the part of the line's quad they were read +/// from — PaddleOCR's `return_word_box` arithmetic: a word runs from its first character's +/// recognition step to its last character's. A line the crop turned upright keeps the line's +/// quad for each of its words. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Word { + pub text: String, + pub quad: Quad, +} + +/// The two models and the recognizer's alphabet. +pub struct Recognizer { + det: Model, + rec: Model, + chars: Vec, +} + +pub const DET: &[u8] = include_bytes!("../models/ch_PP-OCRv5_det_mobile.onnx"); +pub const REC: &[u8] = include_bytes!("../models/en_PP-OCRv5_rec_mobile.onnx"); +pub const DICT: &str = include_str!("../models/ppocrv5_en_dict.txt"); + +impl Recognizer { + pub fn new() -> Result { + let det = Model::load_static_slice(DET).map_err(|e| format!("detection model: {e}"))?; + let rec = Model::load_static_slice(REC).map_err(|e| format!("recognition model: {e}"))?; + let mut chars = vec![String::from("blank")]; + chars.extend(DICT.lines().map(String::from)); + chars.push(" ".into()); + Ok(Recognizer { det, rec, chars }) + } + + /// Every line in `img`, in PaddleOCR's reading order (top to bottom, then left to right + /// within 10 pixels), its quad in `img`'s pixels. + pub fn read(&self, img: &Bgr) -> Result, String> { + let prob = run(&self.det, det_input(img))?; + let (ph, pw) = (prob.shape()[2], prob.shape()[3]); + let pv: Vec = prob.iter().copied().collect(); + let quads = boxes(&pv, ph, pw, img.h, img.w); + // Lines are independent, so they are read in parallel; rten's bits do not depend on the + // thread count, and the results are collected in order. + let reads: Vec, String>> = quads + .par_iter() + .map(|q| { + let Some((c, turned)) = crop(img, q) else { + return Ok(None); + }; + let (x, img_w, rw) = rec_input(&c); + let (text, chars, steps) = ctc(&run(&self.rec, x)?, &self.chars); + let words = words(q, &chars, steps, img_w, rw, c.w, turned); + Ok(Some(Line { + text, + quad: if turned { reading(q) } else { *q }, + words, + })) + }) + .collect(); + let mut out = Vec::new(); + for r in reads { + if let Some(line) = r? { + if !line.text.is_empty() { + out.push(line); + } + } + } + Ok(out) + } +} + +fn run(model: &Model, x: Tensor) -> Result, String> { + let y = model + .run_one(x.view().into(), None) + .map_err(|e| e.to_string())?; + y.try_into() + .map_err(|_| String::from("model output is not f32")) +} + +/// One axis of cv2's INTER_LINEAR on 8-bit data: per destination index, the two source indices +/// and their 11-bit fixed-point weights. +fn taps(src: usize, dst: usize) -> Vec<(usize, usize, i64, i64)> { + let scale = src as f64 / dst as f64; + (0..dst) + .map(|d| { + let f = ((d as f64 + 0.5) * scale - 0.5) as f32; + let (mut s, mut frac) = (f.floor(), f - f.floor()); + if s < 0.0 { + (s, frac) = (0.0, 0.0); + } + if s >= (src - 1) as f32 { + (s, frac) = ((src - 1) as f32, 0.0); + } + let s = s as usize; + let w0 = ((1.0 - frac) * 2048.0).round_ties_even() as i64; + let w1 = (frac * 2048.0).round_ties_even() as i64; + (s, (s + 1).min(src - 1), w0, w1) + }) + .collect() +} + +/// cv2.resize INTER_LINEAR: integer weights, one rounding at the end. +pub fn resize(img: &Bgr, dw: usize, dh: usize) -> Bgr { + let (tx, ty) = (taps(img.w, dw), taps(img.h, dh)); + let mut px = vec![0u8; dw * dh * 3]; + for (y, &(y0, y1, b0, b1)) in ty.iter().enumerate() { + for (x, &(x0, x1, a0, a1)) in tx.iter().enumerate() { + for c in 0..3 { + let row = |yy| img.at(x0, yy, c) as i64 * a0 + img.at(x1, yy, c) as i64 * a1; + let v = (row(y0) * b0 + row(y1) * b1 + (1 << 21)) >> 22; + px[(y * dw + x) * 3 + c] = v.clamp(0, 255) as u8; + } + } + } + Bgr { w: dw, h: dh, px } +} + +/// PaddleOCR's detection size: no downscale, each side to a multiple of 32, at most 4000. +fn det_size(h: usize, w: usize) -> (usize, usize) { + let (mut rh, mut rw) = (h as f64, w as f64); + let m = h.min(w) as f64; + if m < 64.0 { + (rh, rw) = ((h as f64 * 64.0 / m).trunc(), (w as f64 * 64.0 / m).trunc()); + } + if rh.max(rw) > 4000.0 { + let r = 4000.0 / rh.max(rw); + (rh, rw) = ((rh * r).trunc(), (rw * r).trunc()); + } + let r32 = |v: f64| (((v / 32.0).round_ties_even() * 32.0) as usize).max(32); + (r32(rh), r32(rw)) +} + +fn det_input(img: &Bgr) -> Tensor { + let (rh, rw) = det_size(img.h, img.w); + let r = resize(img, rw, rh); + let n = rh * rw; + let mut data = vec![0f32; 3 * n]; + for (c, plane) in data.chunks_exact_mut(n).enumerate() { + for (i, v) in plane.iter_mut().enumerate() { + *v = (r.px[i * 3 + c] as f32 / 255.0 - MEAN[c]) / STD[c]; + } + } + Tensor::from_data(&[1, 3, rh, rw], data) +} + +/// The four corners ordered as PaddleOCR's `get_mini_boxes`: top-left, top-right, bottom-right, +/// bottom-left, from the two leftmost and the two rightmost. +fn mini_box(c: [[f32; 2]; 4]) -> [[f32; 2]; 4] { + let mut p = c.to_vec(); + p.sort_by(|a, b| a[0].total_cmp(&b[0])); + let (i1, i4) = if p[1][1] > p[0][1] { (0, 1) } else { (1, 0) }; + let (i2, i3) = if p[3][1] > p[2][1] { (2, 3) } else { (3, 2) }; + [p[i1], p[i2], p[i3], p[i4]] +} + +/// Whether (x, y) lies inside or on the convex quad `q`. +fn inside(q: &[[i64; 2]; 4], x: i64, y: i64) -> bool { + let (mut pos, mut neg) = (false, false); + for k in 0..4 { + let (a, b) = (q[k], q[(k + 1) % 4]); + let c = (b[0] - a[0]) * (y - a[1]) - (b[1] - a[1]) * (x - a[0]); + pos |= c > 0; + neg |= c < 0; + } + !(pos && neg) +} + +/// PaddleOCR's "fast" box score: the mean probability over the pixels the box covers. +fn score_fast(prob: &[f32], ph: usize, pw: usize, q: &[[f32; 2]; 4]) -> f64 { + let lo = |v: f32, n: usize| (v.floor() as i64).clamp(0, n as i64 - 1); + let hi = |v: f32, n: usize| (v.ceil() as i64).clamp(0, n as i64 - 1); + let (xs, ys) = (q.map(|p| p[0]), q.map(|p| p[1])); + let min = |v: [f32; 4]| v.iter().copied().fold(f32::MAX, f32::min); + let max = |v: [f32; 4]| v.iter().copied().fold(f32::MIN, f32::max); + let (xmin, xmax) = (lo(min(xs), pw), hi(max(xs), pw)); + let (ymin, ymax) = (lo(min(ys), ph), hi(max(ys), ph)); + let iq = q.map(|p| [(p[0] - xmin as f32) as i64, (p[1] - ymin as f32) as i64]); + let (mut sum, mut n) = (0f64, 0u64); + for y in ymin..=ymax { + for x in xmin..=xmax { + if inside(&iq, x - xmin, y - ymin) { + sum += prob[y as usize * pw + x as usize] as f64; + n += 1; + } + } + } + if n == 0 { + 0.0 + } else { + sum / n as f64 + } +} + +/// PaddleOCR's `_clockwise` on an integer box: top-left and bottom-right by x + y, the other two +/// by y - x; the first index on ties. +fn clockwise(b: Quad) -> Quad { + let pick = |key: &dyn Fn(&[i32; 2]) -> i32, want_max: bool, from: &[usize]| -> usize { + let mut best = from[0]; + for &i in &from[1..] { + let (k, kb) = (key(&b[i]), key(&b[best])); + if (want_max && k > kb) || (!want_max && k < kb) { + best = i; + } + } + best + }; + let all = [0, 1, 2, 3]; + let s = |p: &[i32; 2]| p[0] + p[1]; + let (tl, br) = (pick(&s, false, &all), pick(&s, true, &all)); + let rest: Vec = all + .iter() + .copied() + .filter(|&i| i != tl && i != br) + .collect(); + let d = |p: &[i32; 2]| p[1] - p[0]; + let (tr, bl) = (pick(&d, false, &rest), pick(&d, true, &rest)); + [b[tl], b[tr], b[br], b[bl]] +} + +fn edge(a: [f32; 2], b: [f32; 2]) -> f32 { + let (dx, dy) = (a[0] - b[0], a[1] - b[1]); + (dx * dx + dy * dy).sqrt() +} + +/// Convex hull of integer points (Andrew's monotone chain), collinear points dropped. Exact. +/// +/// Not rten-imageproc's `convex_hull`: its Graham scan visits the points collinear with the lowest +/// one nearest first, so on an unsimplified contour (a text line's flat bottom edge) the hull +/// folds, `min_area_rect` returns a rectangle of height zero, and the line is lost (35 of 49 on a +/// text page, measured 2026-11-10). +pub fn hull(points: &[[i64; 2]]) -> Vec<[i64; 2]> { + let mut p = points.to_vec(); + p.sort_unstable(); + p.dedup(); + if p.len() < 3 { + return p; + } + let cross = |o: [i64; 2], a: [i64; 2], b: [i64; 2]| { + (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]) + }; + let chain = |pts: &mut dyn Iterator| { + let mut c: Vec<[i64; 2]> = Vec::new(); + for q in pts { + while c.len() >= 2 && cross(c[c.len() - 2], c[c.len() - 1], q) <= 0 { + c.pop(); + } + c.push(q); + } + c.pop(); + c + }; + let mut lower = chain(&mut p.iter().copied()); + lower.extend(chain(&mut p.iter().rev().copied())); + lower +} + +/// A rotated rectangle: centre, unit axes `u` and `v`, and its extents along them. +#[derive(Clone, Copy, Debug)] +pub struct Rect { + c: [f64; 2], + u: [f64; 2], + v: [f64; 2], + pub w: f64, + pub h: f64, +} + +impl Rect { + fn corners(&self) -> [[f32; 2]; 4] { + let at = |a: f64, b: f64| { + [ + (self.c[0] + self.u[0] * a + self.v[0] * b) as f32, + (self.c[1] + self.u[1] * a + self.v[1] * b) as f32, + ] + }; + let (hw, hh) = (self.w / 2.0, self.h / 2.0); + [at(-hw, -hh), at(hw, -hh), at(hw, hh), at(-hw, hh)] + } +} + +/// The minimum-area rectangle around a hull: the one with a side on a hull edge, the first such +/// edge on ties. Areas are compared exactly in integers; only the corners are floating point. +pub fn min_rect(h: &[[i64; 2]]) -> Option { + if h.is_empty() { + return None; + } + if h.len() < 3 { + let (x0, x1) = (h.iter().map(|p| p[0]).min()?, h.iter().map(|p| p[0]).max()?); + let (y0, y1) = (h.iter().map(|p| p[1]).min()?, h.iter().map(|p| p[1]).max()?); + let c = [(x0 + x1) as f64 / 2.0, (y0 + y1) as f64 / 2.0]; + let (w, ht) = ((x1 - x0) as f64, (y1 - y0) as f64); + return Some(Rect { + c, + u: [1.0, 0.0], + v: [0.0, 1.0], + w, + h: ht, + }); + } + // Per edge: the spans along and across it, each scaled by the edge's length, and |edge|^2. + let mut best: Option<(i128, i128, usize, [i64; 4])> = None; + for i in 0..h.len() { + let (a, b) = (h[i], h[(i + 1) % h.len()]); + let e = [b[0] - a[0], b[1] - a[1]]; + let (mut s0, mut s1, mut t0, mut t1) = (i64::MAX, i64::MIN, i64::MAX, i64::MIN); + for q in h { + let d = [q[0] - a[0], q[1] - a[1]]; + let (s, t) = (e[0] * d[0] + e[1] * d[1], e[0] * d[1] - e[1] * d[0]); + (s0, s1, t0, t1) = (s0.min(s), s1.max(s), t0.min(t), t1.max(t)); + } + let len2 = (e[0] * e[0] + e[1] * e[1]) as i128; + let area = (s1 - s0) as i128 * (t1 - t0) as i128; // area * len2 + if best.is_none_or(|(ba, bl, _, _)| area * bl < ba * len2) { + best = Some((area, len2, i, [s0, s1, t0, t1])); + } + } + let (_, len2, i, [s0, s1, t0, t1]) = best?; + let (a, b) = (h[i], h[(i + 1) % h.len()]); + let len = (len2 as f64).sqrt(); + let u = [(b[0] - a[0]) as f64 / len, (b[1] - a[1]) as f64 / len]; + let v = [-u[1], u[0]]; + let (sm, tm) = ((s0 + s1) as f64 / 2.0 / len, (t0 + t1) as f64 / 2.0 / len); + let c = [ + a[0] as f64 + u[0] * sm + v[0] * tm, + a[1] as f64 + u[1] * sm + v[1] * tm, + ]; + Some(Rect { + c, + u, + v, + w: (s1 - s0) as f64 / len, + h: (t1 - t0) as f64 / len, + }) +} + +/// DB post-processing (PaddleOCR `DBPostProcess`, score mode "fast"), then `filter_tag_det_res` +/// and `sorted_boxes`. `prob` is the detection map, `ph` x `pw`; the quads are in the source +/// image's pixels, `h` x `w`. +fn boxes(prob: &[f32], ph: usize, pw: usize, h: usize, w: usize) -> Vec { + let mask = NdTensor::from_data( + [ph, pw], + prob.iter().map(|&p| p > THRESH).collect::>(), + ); + let mut out = Vec::new(); + for poly in find_contours(mask.view(), RetrievalMode::List) + .iter() + .take(MAX_CANDIDATES) + { + let pts: Vec<[i64; 2]> = poly.iter().map(|p| [p.x as i64, p.y as i64]).collect(); + let Some(rect) = min_rect(&hull(&pts)) else { + continue; + }; + if rect.w.min(rect.h) < MIN_SIZE + || score_fast(prob, ph, pw, &mini_box(rect.corners())) < BOX_THRESH + { + continue; + } + let d = rect.w * rect.h * UNCLIP / (2.0 * (rect.w + rect.h)); + let grown = Rect { + w: rect.w + 2.0 * d, + h: rect.h + 2.0 * d, + ..rect + }; + if grown.w.min(grown.h) < MIN_SIZE + 2.0 { + continue; + } + let scale = |v: f32, from: usize, to: usize| { + (v / from as f32 * to as f32) + .round_ties_even() + .clamp(0.0, to as f32) as i32 + }; + let q = mini_box(grown.corners()).map(|p| [scale(p[0], pw, w), scale(p[1], ph, h)]); + let q = clockwise(q).map(|p| [p[0].clamp(0, w as i32 - 1), p[1].clamp(0, h as i32 - 1)]); + let f = q.map(|p| [p[0] as f32, p[1] as f32]); + if (edge(f[0], f[1]) as i32) <= 3 || (edge(f[0], f[3]) as i32) <= 3 { + continue; + } + out.push(q); + } + out.sort_by_key(|q| (q[0][1], q[0][0])); + for i in 0..out.len().saturating_sub(1) { + for j in (0..=i).rev() { + if (out[j + 1][0][1] - out[j][0][1]).abs() < 10 && out[j + 1][0][0] < out[j][0][0] { + out.swap(j, j + 1); + } else { + break; + } + } + } + out +} + +/// A turned line's corners in reading order. `crop` turns a tall line a quarter anticlockwise to read +/// it, so its text's top-left is the box's top-right, and so on round: the quad a caller places +/// the text by, rather than the box's corners in the image's order. +fn reading(q: &Quad) -> Quad { + [q[1], q[2], q[3], q[0]] +} + +/// The size of the crop `crop` cuts for quad `q`: its longer top or bottom edge by its longer side. +fn crop_size(q: &Quad) -> (usize, usize) { + let p = q.map(|v| [v[0] as f32, v[1] as f32]); + let cw = edge(p[0], p[1]).max(edge(p[2], p[3])) as usize; + let ch = edge(p[0], p[3]).max(edge(p[1], p[2])) as usize; + (cw, ch) +} + +/// The line under quad `q`, warped upright: the affine map of corners 0, 1 and 3 onto the crop, +/// sampled bilinearly with the border repeated; turned a quarter, and so reported, when 1.5 times +/// taller than wide. +fn crop(img: &Bgr, q: &Quad) -> Option<(Bgr, bool)> { + let p = q.map(|v| [v[0] as f32, v[1] as f32]); + let (cw, ch) = crop_size(q); + if cw == 0 || ch == 0 { + return None; + } + let mut px = vec![0u8; cw * ch * 3]; + let (ux, uy) = ( + (p[1][0] - p[0][0]) / cw as f32, + (p[1][1] - p[0][1]) / cw as f32, + ); + let (vx, vy) = ( + (p[3][0] - p[0][0]) / ch as f32, + (p[3][1] - p[0][1]) / ch as f32, + ); + let clampi = |v: f32, n: usize| (v as i64).clamp(0, n as i64 - 1) as usize; + for v in 0..ch { + for u in 0..cw { + let sx = p[0][0] + u as f32 * ux + v as f32 * vx; + let sy = p[0][1] + u as f32 * uy + v as f32 * vy; + let (fx, fy) = (sx.floor(), sy.floor()); + let (ax, ay) = (sx - fx, sy - fy); + let (x0, x1) = (clampi(fx, img.w), clampi(fx + 1.0, img.w)); + let (y0, y1) = (clampi(fy, img.h), clampi(fy + 1.0, img.h)); + for c in 0..3 { + let top = img.at(x0, y0, c) as f32 * (1.0 - ax) + img.at(x1, y0, c) as f32 * ax; + let bot = img.at(x0, y1, c) as f32 * (1.0 - ax) + img.at(x1, y1, c) as f32 * ax; + let val = top * (1.0 - ay) + bot * ay; + px[(v * cw + u) * 3 + c] = val.round_ties_even().clamp(0.0, 255.0) as u8; + } + } + } + let out = Bgr { w: cw, h: ch, px }; + if ch as f32 / cw as f32 >= 1.5 { + return Some((turn(&out, 3), true)); // np.rot90: a quarter turn anticlockwise + } + Some((out, false)) +} + +/// The words of a line read from quad `q`: `chars` are its characters with the first and last +/// recognition step each was read at, out of `steps`; the recognition input was `img_w` columns, +/// the crop `fed_w` columns scaled to `rw` of them. +fn words( + q: &Quad, + chars: &[(String, usize, usize)], + steps: usize, + img_w: usize, + rw: usize, + fed_w: usize, + turned: bool, +) -> Vec { + let mut spans: Vec<(String, usize, usize)> = Vec::new(); + let mut open = false; + for (c, s0, s1) in chars { + if c == " " { + open = false; + } else if open { + let w = spans.last_mut().expect("an open word"); + w.0.push_str(c); + w.2 = *s1; + } else { + spans.push((c.clone(), *s0, *s1)); + open = true; + } + } + let (cw, ch) = crop_size(q); + let p = q.map(|v| [v[0] as f64, v[1] as f64]); + let at = |x: f64, y: f64| -> [i32; 2] { + let mut out = [0i32; 2]; + for (k, o) in out.iter_mut().enumerate() { + let v = + p[0][k] + x * (p[1][k] - p[0][k]) / cw as f64 + y * (p[3][k] - p[0][k]) / ch as f64; + *o = v.round_ties_even() as i32; + } + out + }; + // One recognition step covers img_w / steps input columns, which are fed_w / rw crop columns. + let col = |s: usize| { + (s as f64 * img_w as f64 * fed_w as f64 / (steps as f64 * rw as f64)).min(cw as f64) + }; + spans + .into_iter() + .map(|(text, s0, s1)| { + let quad = if turned { + reading(q) + } else { + let (xa, xb) = (col(s0), col(s1 + 1)); + [ + at(xa, 0.0), + at(xb, 0.0), + at(xb, ch as f64), + at(xa, ch as f64), + ] + }; + Word { text, quad } + }) + .collect() +} + +/// `img` turned `quarters` quarter turns clockwise. +pub fn turn(img: &Bgr, quarters: u8) -> Bgr { + let (w, h) = (img.w, img.h); + let q = quarters % 4; + let (nw, nh) = if q % 2 == 1 { (h, w) } else { (w, h) }; + let mut px = vec![0u8; nw * nh * 3]; + for y in 0..nh { + for x in 0..nw { + let (sx, sy) = match q { + 0 => (x, y), + 1 => (y, h - 1 - x), + 2 => (w - 1 - x, h - 1 - y), + _ => (w - 1 - y, x), + }; + for c in 0..3 { + px[(y * nw + x) * 3 + c] = img.at(sx, sy, c); + } + } + } + Bgr { w: nw, h: nh, px } +} + +/// `img` mirrored left to right. +pub fn mirror(img: &Bgr) -> Bgr { + let mut px = vec![0u8; img.px.len()]; + for y in 0..img.h { + for x in 0..img.w { + for c in 0..3 { + px[(y * img.w + x) * 3 + c] = img.at(img.w - 1 - x, y, c); + } + } + } + Bgr { + w: img.w, + h: img.h, + px, + } +} + +/// One line at the recognition height, padded right with zeros to at least 320 columns; with the +/// input's width and the width the line itself was scaled to. +fn rec_input(c: &Bgr) -> (Tensor, usize, usize) { + let ratio = c.w as f64 / c.h as f64; + let img_w = (REC_H as f64 * (REC_W as f64 / REC_H as f64).max(ratio)) as usize; + let want = (REC_H as f64 * ratio).ceil() as usize; + let rw = want.min(img_w).max(1); + let r = resize(c, rw, REC_H); + let mut data = vec![0f32; 3 * REC_H * img_w]; + for ch in 0..3 { + for y in 0..REC_H { + for x in 0..rw { + let v = r.px[(y * rw + x) * 3 + ch] as f32 / 255.0; + data[ch * REC_H * img_w + y * img_w + x] = (v - 0.5) / 0.5; + } + } + } + (Tensor::from_data(&[1, 3, REC_H, img_w], data), img_w, rw) +} + +/// CTC greedy decoding: the first maximum per step, repeats collapsed, blanks dropped. The text, +/// each character with the first and last step it held, and the number of steps. +fn ctc(probs: &Tensor, chars: &[String]) -> (String, Vec<(String, usize, usize)>, usize) { + let (t, k) = (probs.shape()[1], probs.shape()[2]); + let v: Vec = probs.iter().copied().collect(); + let (mut text, mut out, mut prev) = (String::new(), Vec::new(), usize::MAX); + for s in 0..t { + let row = &v[s * k..(s + 1) * k]; + let mut best = 0; + for i in 1..k { + if row[i] > row[best] { + best = i; + } + } + if best != 0 && best != prev { + text.push_str(&chars[best]); + out.push((chars[best].clone(), s, s)); + } else if best != 0 { + if let Some(last) = out.last_mut() { + last.2 = s; + } + } + prev = best; + } + (text, out, t) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_flat_edged_contour_keeps_its_height() { + // A 40 x 6 bar traced point by point: the case rten-imageproc's hull folded. + let mut pts = Vec::new(); + for x in 0..40 { + pts.push([x, 0]); + pts.push([x, 5]); + } + for y in 0..6 { + pts.push([0, y]); + pts.push([39, y]); + } + let r = min_rect(&hull(&pts)).unwrap(); + assert_eq!((r.w.min(r.h), r.w.max(r.h)), (5.0, 39.0)); + } + + #[test] + fn four_quarter_turns_are_the_identity() { + let img = Bgr { + w: 3, + h: 2, + px: (0..18).collect(), + }; + assert_eq!(turn(&turn(&img, 1), 3), img); + assert_eq!(turn(&turn(&img, 2), 2), img); + assert_eq!(mirror(&mirror(&img)), img); + let t = turn(&img, 1); + assert_eq!((t.w, t.h), (2, 3)); + // The top-left pixel of a clockwise turn is the source's bottom-left. + assert_eq!(t.px[..3], img.px[3 * 3..3 * 3 + 3]); + } + + #[test] + fn resizing_to_the_same_size_copies() { + let img = Bgr { + w: 5, + h: 4, + px: (0..60).map(|v| (v * 7 % 251) as u8).collect(), + }; + assert_eq!(resize(&img, 5, 4), img); + } +} diff --git a/ocr/tests/data/expected.jsonl b/ocr/tests/data/expected.jsonl new file mode 100644 index 0000000..4dd9cdf --- /dev/null +++ b/ocr/tests/data/expected.jsonl @@ -0,0 +1,6 @@ +{"ocr_response":1,"id":"png-file","identity":{"recognizer":"ethos-parser-ocr","version":"0.1.0","rule":"ppocrv5-mobile-v1","det_sha256":"4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae","rec_sha256":"c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8","dict_sha256":"e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6","runtime":"rten 0.26.0; rten-vecmath 0.26.0 with fixed-order sums"},"width":1700,"height":2200,"lines":[{"text":"NIST SP 800-171r3","quad":[[199,105],[408,105],[408,131],[199,131]],"words":[{"text":"NIST","quad":[[208,105],[247,105],[247,131],[208,131]]},{"text":"SP","quad":[[260,105],[277,105],[277,131],[260,131]]},{"text":"800-171r3","quad":[[295,105],[404,105],[404,131],[295,131]]}]},{"text":"Protecting Controlled Unclassified Information","quad":[[975,104],[1500,104],[1500,133],[975,133]],"words":[{"text":"Protecting","quad":[[980,104],[1087,104],[1087,133],[980,133]]},{"text":"Controlled","quad":[[1106,104],[1213,104],[1213,133],[1106,133]]},{"text":"Unclassified","quad":[[1233,104],[1359,104],[1359,133],[1233,133]]},{"text":"Information","quad":[[1369,104],[1495,104],[1495,133],[1369,133]]}]},{"text":"May 2024","quad":[[197,138],[314,138],[314,168],[197,168]],"words":[{"text":"May","quad":[[207,138],[247,138],[247,168],[207,168]]},{"text":"2024","quad":[[262,138],[307,138],[307,168],[262,168]]}]},{"text":"REFERENCES","quad":[[349,207],[524,207],[524,236],[349,236]],"words":[{"text":"REFERENCES","quad":[[359,207],[518,207],[518,236],[359,236]]}]},{"text":"Source Control: MP-04","quad":[[349,264],[657,264],[657,294],[349,294]],"words":[{"text":"Source","quad":[[354,264],[440,264],[440,294],[354,294]]},{"text":"Control:","quad":[[460,264],[561,264],[561,294],[460,294]]},{"text":"MP-04","quad":[[581,264],[652,264],[652,294],[581,294]]}]},{"text":"Supporting Publications: SP 800-88 [50], SP 800-111 [51]","quad":[[348,320],[1115,319],[1115,354],[348,355]],"words":[{"text":"Supporting","quad":[[354,320],[494,320],[494,355],[354,355]]},{"text":"Publications:","quad":[[512,320],[682,320],[682,355],[512,355]]},{"text":"SP","quad":[[693,320],[717,320],[717,355],[693,355]]},{"text":"800-88","quad":[[734,319],[816,319],[816,354],[734,354]]},{"text":"[50],","quad":[[834,319],[898,319],[898,354],[834,354]]},{"text":"SP","quad":[[904,319],[928,319],[928,354],[904,354]]},{"text":"800-111","quad":[[945,319],[1051,319],[1051,354],[945,354]]},{"text":"[51]","quad":[[1062,319],[1115,319],[1115,354],[1062,354]]}]},{"text":"03.08.02 Media Access","quad":[[199,417],[537,419],[537,449],[198,447]],"words":[{"text":"03.08.02","quad":[[209,417],[313,418],[312,448],[208,447]]},{"text":"Media","quad":[[358,418],[433,418],[432,448],[357,448]]},{"text":"Access","quad":[[452,418],[532,419],[531,449],[452,448]]}]},{"text":"Restrict access to CUl on system media to authorized personnel or roles.","quad":[[347,482],[1330,482],[1330,516],[347,516]],"words":[{"text":"Restrict","quad":[[358,482],[449,482],[449,516],[358,516]]},{"text":"access","quad":[[466,482],[546,482],[546,516],[466,516]]},{"text":"to","quad":[[557,482],[580,482],[580,516],[557,516]]},{"text":"CUl","quad":[[603,482],[637,482],[637,516],[603,516]]},{"text":"on","quad":[[654,482],[677,482],[677,516],[654,516]]},{"text":"system","quad":[[694,482],[773,482],[773,516],[694,516]]},{"text":"media","quad":[[802,482],[875,482],[875,516],[802,516]]},{"text":"to","quad":[[887,482],[910,482],[910,516],[887,516]]},{"text":"authorized","quad":[[927,482],[1063,482],[1063,516],[927,516]]},{"text":"personnel","quad":[[1080,482],[1211,482],[1211,516],[1080,516]]},{"text":"or","quad":[[1228,482],[1245,482],[1245,516],[1228,516]]},{"text":"roles.","quad":[[1262,482],[1330,482],[1330,516],[1262,516]]}]},{"text":"DISCUSSION","quad":[[349,558],[521,558],[521,586],[349,586]],"words":[{"text":"DISCUSSION","quad":[[358,558],[512,558],[512,586],[358,586]]}]},{"text":"System media include digital and non-digital media. Access to CUl on system media","quad":[[349,615],[1475,615],[1475,647],[349,647]],"words":[{"text":"System","quad":[[354,615],[434,615],[434,647],[354,647]]},{"text":"media","quad":[[461,615],[536,615],[536,647],[461,647]]},{"text":"include","quad":[[552,615],[637,615],[637,647],[552,647]]},{"text":"digital","quad":[[659,615],[733,615],[733,647],[659,647]]},{"text":"and","quad":[[749,615],[792,615],[792,647],[749,647]]},{"text":"non-digital","quad":[[808,615],[947,615],[947,647],[808,647]]},{"text":"media.","quad":[[963,615],[1048,615],[1048,647],[963,647]]},{"text":"Access","quad":[[1064,615],[1144,615],[1144,647],[1064,647]]},{"text":"to","quad":[[1160,615],[1176,615],[1176,647],[1160,647]]},{"text":"CUl","quad":[[1198,615],[1240,615],[1240,647],[1198,647]]},{"text":"on","quad":[[1251,615],[1272,615],[1272,647],[1251,647]]},{"text":"system","quad":[[1294,615],[1368,615],[1368,647],[1294,647]]},{"text":"media","quad":[[1400,615],[1470,615],[1470,647],[1400,647]]}]},{"text":"can be restricted by physically controlling such media. This includes conducting","quad":[[348,653],[1422,655],[1422,689],[348,687]],"words":[{"text":"can","quad":[[354,653],[388,653],[388,687],[354,687]]},{"text":"be","quad":[[411,653],[433,653],[433,687],[411,687]]},{"text":"restricted","quad":[[450,653],[575,653],[575,687],[450,687]]},{"text":"by","quad":[[592,653],[615,653],[615,687],[592,687]]},{"text":"physically","quad":[[632,654],[757,654],[757,688],[632,688]]},{"text":"controlling","quad":[[769,654],[905,654],[905,688],[769,688]]},{"text":"such","quad":[[922,654],[973,654],[973,688],[922,688]]},{"text":"media.","quad":[[996,654],[1081,654],[1081,688],[996,688]]},{"text":"This","quad":[[1098,654],[1144,654],[1144,688],[1098,688]]},{"text":"includes","quad":[[1155,655],[1263,655],[1263,689],[1155,689]]},{"text":"conducting","quad":[[1280,655],[1416,655],[1416,689],[1280,689]]}]},{"text":"inventories, ensuring that procedures are in place to allow individuals to check out","quad":[[348,696],[1469,694],[1469,728],[348,730]],"words":[{"text":"inventories,","quad":[[354,696],[512,696],[512,730],[354,730]]},{"text":"ensuring","quad":[[524,696],[631,695],[631,729],[524,730]]},{"text":"that","quad":[[648,695],[693,695],[693,729],[648,729]]},{"text":"procedures","quad":[[710,695],[858,695],[858,729],[710,729]]},{"text":"are","quad":[[875,695],[903,695],[903,729],[875,729]]},{"text":"in","quad":[[920,695],[937,695],[937,729],[920,729]]},{"text":"place","quad":[[959,695],[1016,695],[1016,729],[959,729]]},{"text":"to","quad":[[1033,695],[1050,695],[1050,729],[1033,729]]},{"text":"allow","quad":[[1073,695],[1129,695],[1129,729],[1073,729]]},{"text":"individuals","quad":[[1146,695],[1288,694],[1288,728],[1146,729]]},{"text":"to","quad":[[1299,694],[1322,694],[1322,728],[1299,728]]},{"text":"check","quad":[[1339,694],[1407,694],[1407,728],[1339,728]]},{"text":"out","quad":[[1424,694],[1463,694],[1463,728],[1424,728]]}]},{"text":"and return media to the media library, and maintaining accountability for stored","quad":[[347,737],[1439,737],[1439,772],[347,772]],"words":[{"text":"and","quad":[[353,737],[400,737],[400,772],[353,772]]},{"text":"return","quad":[[411,737],[487,737],[487,772],[411,772]]},{"text":"media","quad":[[511,737],[581,737],[581,772],[511,772]]},{"text":"to","quad":[[598,737],[616,737],[616,772],[598,772]]},{"text":"the","quad":[[633,737],[668,737],[668,772],[633,772]]},{"text":"media","quad":[[692,737],[762,737],[762,772],[692,772]]},{"text":"library,","quad":[[773,737],[873,737],[873,772],[773,772]]},{"text":"and","quad":[[884,737],[925,737],[925,772],[884,772]]},{"text":"maintaining","quad":[[943,737],[1094,737],[1094,772],[943,772]]},{"text":"accountability","quad":[[1112,737],[1293,737],[1293,772],[1112,772]]},{"text":"for","quad":[[1305,737],[1340,737],[1340,772],[1305,772]]},{"text":"stored","quad":[[1357,737],[1433,737],[1433,772],[1357,772]]}]},{"text":"media. For digital media, access to CUl can be restricted by using cryptographic","quad":[[349,777],[1422,777],[1422,811],[349,811]],"words":[{"text":"media.","quad":[[355,777],[446,777],[446,811],[355,811]]},{"text":"For","quad":[[457,777],[491,777],[491,811],[457,811]]},{"text":"digital","quad":[[508,777],[587,777],[587,811],[508,811]]},{"text":"media,","quad":[[599,777],[684,777],[684,811],[599,811]]},{"text":"access","quad":[[701,777],[775,777],[775,811],[701,811]]},{"text":"to","quad":[[792,777],[809,777],[809,811],[792,811]]},{"text":"CUl","quad":[[832,777],[871,777],[871,811],[832,811]]},{"text":"can","quad":[[888,777],[922,777],[922,811],[888,811]]},{"text":"be","quad":[[939,777],[962,777],[962,811],[939,811]]},{"text":"restricted","quad":[[979,777],[1104,777],[1104,811],[979,811]]},{"text":"by","quad":[[1121,777],[1144,777],[1144,811],[1121,811]]},{"text":"using","quad":[[1161,777],[1223,777],[1223,811],[1161,811]]},{"text":"cryptographic","quad":[[1240,777],[1416,777],[1416,811],[1240,811]]}]},{"text":"means. Encrypting data in storage or at rest is addressed in 03.13.08.","quad":[[350,819],[1282,817],[1282,850],[350,852]],"words":[{"text":"means.","quad":[[356,819],[449,819],[449,852],[356,852]]},{"text":"Encrypting","quad":[[460,819],[593,818],[593,851],[460,852]]},{"text":"data","quad":[[615,818],[664,818],[664,851],[615,851]]},{"text":"in","quad":[[675,818],[692,818],[692,851],[675,851]]},{"text":"storage","quad":[[714,818],[808,818],[808,851],[714,851]]},{"text":"or","quad":[[824,818],[846,818],[846,851],[824,851]]},{"text":"at","quad":[[863,818],[879,818],[879,851],[863,851]]},{"text":"rest","quad":[[896,818],[940,818],[940,851],[896,851]]},{"text":"is","quad":[[951,818],[968,818],[968,851],[951,851]]},{"text":"addressed","quad":[[984,818],[1117,817],[1117,850],[984,851]]},{"text":"in","quad":[[1128,817],[1144,817],[1144,850],[1128,850]]},{"text":"03.13.08.","quad":[[1166,817],[1282,817],[1282,850],[1166,850]]}]},{"text":"REFERENCES","quad":[[349,892],[524,892],[524,921],[349,921]],"words":[{"text":"REFERENCES","quad":[[359,892],[518,892],[518,921],[359,921]]}]},{"text":"Source Control: MP-02","quad":[[348,949],[659,949],[659,981],[348,981]],"words":[{"text":"Source","quad":[[353,949],[439,949],[439,981],[353,981]]},{"text":"Control:","quad":[[461,949],[562,949],[562,981],[461,981]]},{"text":"MP-02","quad":[[579,949],[654,949],[654,981],[579,981]]}]},{"text":"Supporting Publications: SP 800-111 [51]","quad":[[348,1007],[903,1005],[903,1039],[349,1042]],"words":[{"text":"Supporting","quad":[[354,1007],[494,1006],[495,1041],[355,1042]]},{"text":"Publications:","quad":[[512,1006],[681,1006],[682,1041],[513,1041]]},{"text":"SP","quad":[[693,1006],[716,1006],[717,1041],[694,1041]]},{"text":"800-111","quad":[[734,1006],[833,1005],[834,1040],[735,1041]]},{"text":"[51]","quad":[[850,1005],[897,1005],[898,1040],[851,1040]]}]},{"text":"03.08.03 Media Sanitization","quad":[[199,1102],[608,1104],[608,1134],[198,1132]],"words":[{"text":"03.08.03","quad":[[209,1102],[314,1103],[313,1133],[208,1132]]},{"text":"Media","quad":[[359,1103],[433,1103],[432,1133],[358,1133]]},{"text":"Sanitization","quad":[[453,1103],[598,1104],[597,1134],[452,1133]]}]},{"text":"Sanitize system media that contain CUl prior to disposal, release out of","quad":[[348,1169],[1311,1168],[1311,1201],[348,1202]],"words":[{"text":"Sanitize","quad":[[354,1169],[453,1169],[453,1202],[354,1202]]},{"text":"system","quad":[[469,1169],[546,1169],[546,1202],[469,1202]]},{"text":"media","quad":[[574,1169],[645,1169],[645,1202],[574,1202]]},{"text":"that","quad":[[662,1169],[711,1169],[711,1202],[662,1202]]},{"text":"contain","quad":[[728,1169],[816,1169],[816,1202],[728,1202]]},{"text":"CUl","quad":[[838,1168],[876,1168],[876,1201],[838,1201]]},{"text":"prior","quad":[[893,1168],[948,1168],[948,1201],[893,1201]]},{"text":"to","quad":[[964,1168],[981,1168],[981,1201],[964,1201]]},{"text":"disposal,","quad":[[1003,1168],[1113,1168],[1113,1201],[1003,1201]]},{"text":"release","quad":[[1124,1168],[1212,1168],[1212,1201],[1124,1201]]},{"text":"out","quad":[[1234,1168],[1267,1168],[1267,1201],[1234,1201]]},{"text":"of","quad":[[1283,1168],[1305,1168],[1305,1201],[1283,1201]]}]},{"text":"organizational control, or release for reuse.","quad":[[350,1210],[938,1209],[938,1242],[350,1243]],"words":[{"text":"organizational","quad":[[355,1210],[542,1210],[542,1243],[355,1243]]},{"text":"control,","quad":[[559,1210],[658,1209],[658,1242],[559,1243]]},{"text":"or","quad":[[669,1209],[691,1209],[691,1242],[669,1242]]},{"text":"release","quad":[[702,1209],[790,1209],[790,1242],[702,1242]]},{"text":"for","quad":[[806,1209],[845,1209],[845,1242],[806,1242]]},{"text":"reuse.","quad":[[856,1209],[933,1209],[933,1242],[856,1242]]}]},{"text":"DISCUSSION","quad":[[349,1283],[521,1283],[521,1314],[349,1314]],"words":[{"text":"DISCUSSION","quad":[[359,1283],[509,1283],[509,1314],[359,1314]]}]},{"text":"Media sanitization applies to digital and non-digital media that are subject to","quad":[[348,1340],[1391,1341],[1391,1375],[348,1374]],"words":[{"text":"Media","quad":[[359,1340],[433,1340],[433,1374],[359,1374]]},{"text":"sanitization","quad":[[450,1340],[592,1340],[592,1374],[450,1374]]},{"text":"applies","quad":[[614,1340],[699,1340],[699,1374],[614,1374]]},{"text":"to","quad":[[716,1340],[733,1340],[733,1374],[716,1374]]},{"text":"digital","quad":[[756,1340],[830,1340],[830,1374],[756,1374]]},{"text":"and","quad":[[841,1340],[887,1341],[887,1375],[841,1374]]},{"text":"non-digital","quad":[[904,1341],[1040,1341],[1040,1375],[904,1375]]},{"text":"media","quad":[[1057,1341],[1130,1341],[1130,1375],[1057,1375]]},{"text":"that","quad":[[1147,1341],[1193,1341],[1193,1375],[1147,1375]]},{"text":"are","quad":[[1210,1341],[1244,1341],[1244,1375],[1210,1375]]},{"text":"subject","quad":[[1261,1341],[1351,1341],[1351,1375],[1261,1375]]},{"text":"to","quad":[[1368,1341],[1385,1341],[1385,1375],[1368,1375]]}]},{"text":"disposal or reuse, whether or not the media are considered removable. Examples","quad":[[349,1381],[1451,1382],[1451,1416],[348,1415]],"words":[{"text":"disposal","quad":[[355,1381],[462,1381],[461,1415],[354,1415]]},{"text":"or","quad":[[473,1381],[496,1381],[495,1415],[472,1415]]},{"text":"reuse,","quad":[[507,1381],[586,1381],[585,1415],[506,1415]]},{"text":"whether","quad":[[603,1381],[705,1381],[704,1415],[602,1415]]},{"text":"or","quad":[[722,1381],[745,1381],[744,1415],[721,1415]]},{"text":"not","quad":[[762,1381],[795,1381],[794,1415],[761,1415]]},{"text":"the","quad":[[812,1381],[846,1381],[845,1415],[811,1415]]},{"text":"media","quad":[[869,1381],[942,1382],[941,1416],[868,1415]]},{"text":"are","quad":[[959,1382],[993,1382],[992,1416],[958,1416]]},{"text":"considered","quad":[[1010,1382],[1146,1382],[1145,1416],[1009,1416]]},{"text":"removable.","quad":[[1163,1382],[1315,1382],[1314,1416],[1162,1416]]},{"text":"Examples","quad":[[1327,1382],[1445,1382],[1444,1416],[1326,1416]]}]},{"text":"include digital media in scanners, copiers, printers, notebook computers, mobile","quad":[[347,1421],[1438,1420],[1438,1456],[347,1457]],"words":[{"text":"include","quad":[[353,1421],[443,1421],[443,1457],[353,1457]]},{"text":"digital","quad":[[461,1421],[539,1421],[539,1457],[461,1457]]},{"text":"media","quad":[[551,1421],[629,1421],[629,1457],[551,1457]]},{"text":"in","quad":[[641,1421],[659,1421],[659,1457],[641,1457]]},{"text":"scanners,","quad":[[677,1421],[797,1421],[797,1457],[677,1457]]},{"text":"copiers,","quad":[[815,1421],[910,1420],[910,1456],[815,1457]]},{"text":"printers,","quad":[[928,1420],[1036,1420],[1036,1456],[928,1456]]},{"text":"notebook","quad":[[1048,1420],[1168,1420],[1168,1456],[1048,1456]]},{"text":"computers,","quad":[[1186,1420],[1336,1420],[1336,1456],[1186,1456]]},{"text":"mobile","quad":[[1348,1420],[1432,1420],[1432,1456],[1348,1456]]}]},{"text":"devices, workstations, network components, and non-digital media. The sanitization","quad":[[348,1462],[1491,1462],[1491,1497],[348,1497]],"words":[{"text":"devices,","quad":[[360,1462],[459,1462],[459,1497],[360,1497]]},{"text":"workstations,","quad":[[476,1462],[651,1462],[651,1497],[476,1497]]},{"text":"network","quad":[[663,1462],[768,1462],[768,1497],[663,1497]]},{"text":"components,","quad":[[785,1462],[954,1462],[954,1497],[785,1497]]},{"text":"and","quad":[[972,1462],[1007,1462],[1007,1497],[972,1497]]},{"text":"non-digital","quad":[[1024,1462],[1164,1462],[1164,1497],[1024,1497]]},{"text":"media.","quad":[[1182,1462],[1269,1462],[1269,1497],[1182,1497]]},{"text":"The","quad":[[1281,1462],[1322,1462],[1322,1497],[1281,1497]]},{"text":"sanitization","quad":[[1339,1462],[1479,1462],[1479,1497],[1339,1497]]}]},{"text":"process removes Cul from media such that the information cannot be retrieved or","quad":[[348,1505],[1466,1503],[1466,1536],[348,1538]],"words":[{"text":"process","quad":[[354,1505],[453,1505],[453,1538],[354,1538]]},{"text":"removes","quad":[[464,1505],[574,1505],[574,1538],[464,1538]]},{"text":"Cul","quad":[[596,1505],[634,1504],[634,1537],[596,1538]]},{"text":"from","quad":[[645,1504],[695,1504],[695,1537],[645,1537]]},{"text":"media","quad":[[723,1504],[794,1504],[794,1537],[723,1537]]},{"text":"such","quad":[[811,1504],[860,1504],[860,1537],[811,1537]]},{"text":"that","quad":[[882,1504],[932,1504],[932,1537],[882,1537]]},{"text":"the","quad":[[943,1504],[981,1504],[981,1537],[943,1537]]},{"text":"information","quad":[[992,1504],[1147,1504],[1147,1537],[992,1537]]},{"text":"cannot","quad":[[1169,1504],[1251,1503],[1251,1536],[1169,1537]]},{"text":"be","quad":[[1268,1503],[1290,1503],[1290,1536],[1268,1536]]},{"text":"retrieved","quad":[[1312,1503],[1427,1503],[1427,1536],[1312,1536]]},{"text":"or","quad":[[1444,1503],[1460,1503],[1460,1536],[1444,1536]]}]},{"text":"reconstructed. Sanitization techniques (e.g., cryptographically erasing, clearing,","quad":[[347,1543],[1425,1545],[1425,1579],[347,1578]],"words":[{"text":"reconstructed.","quad":[[353,1543],[551,1543],[551,1578],[353,1578]]},{"text":"Sanitization","quad":[[563,1543],[708,1544],[708,1579],[563,1578]]},{"text":"techniques","quad":[[726,1544],[866,1544],[866,1579],[726,1579]]},{"text":"(e.g.,","quad":[[883,1544],[947,1544],[947,1579],[883,1579]]},{"text":"cryptographically","quad":[[959,1544],[1180,1545],[1180,1580],[959,1579]]},{"text":"erasing,","quad":[[1198,1545],[1297,1545],[1297,1580],[1198,1580]]},{"text":"clearing,","quad":[[1314,1545],[1425,1545],[1425,1580],[1314,1580]]}]},{"text":"purging, and destroying) prevent the disclosure of CUl to unauthorized individuals","quad":[[349,1586],[1460,1582],[1460,1615],[349,1619]],"words":[{"text":"purging,","quad":[[354,1586],[459,1586],[459,1619],[354,1619]]},{"text":"and","quad":[[476,1586],[514,1585],[514,1618],[476,1619]]},{"text":"destroying)","quad":[[536,1585],[679,1585],[679,1618],[536,1618]]},{"text":"prevent","quad":[[696,1585],[789,1584],[789,1617],[696,1618]]},{"text":"the","quad":[[806,1584],[844,1584],[844,1617],[806,1617]]},{"text":"disclosure","quad":[[860,1584],[987,1584],[987,1617],[860,1617]]},{"text":"of","quad":[[1004,1584],[1026,1584],[1026,1617],[1004,1617]]},{"text":"CUl","quad":[[1042,1584],[1080,1583],[1080,1616],[1042,1617]]},{"text":"to","quad":[[1092,1583],[1114,1583],[1114,1616],[1092,1616]]},{"text":"unauthorized","quad":[[1130,1583],[1306,1583],[1306,1616],[1130,1616]]},{"text":"individuals","quad":[[1317,1583],[1454,1582],[1454,1615],[1317,1616]]}]},{"text":"when such media are reused or released for disposal. NARA policies control the","quad":[[349,1627],[1425,1625],[1425,1657],[349,1658]],"words":[{"text":"when","quad":[[360,1627],[418,1627],[418,1658],[360,1658]]},{"text":"such","quad":[[440,1627],[487,1627],[487,1658],[440,1658]]},{"text":"media","quad":[[509,1627],[583,1627],[583,1658],[509,1658]]},{"text":"are","quad":[[599,1627],[631,1626],[631,1657],[599,1658]]},{"text":"reused","quad":[[653,1626],[733,1626],[733,1657],[653,1657]]},{"text":"or","quad":[[754,1626],[770,1626],[770,1657],[754,1657]]},{"text":"released","quad":[[786,1626],[892,1626],[892,1657],[786,1657]]},{"text":"for","quad":[[908,1626],[940,1626],[940,1657],[908,1657]]},{"text":"disposal.","quad":[[962,1626],[1073,1626],[1073,1657],[962,1657]]},{"text":"NARA","quad":[[1084,1626],[1148,1626],[1148,1657],[1084,1657]]},{"text":"policies","quad":[[1169,1625],[1260,1625],[1260,1656],[1169,1656]]},{"text":"control","quad":[[1281,1625],[1366,1625],[1366,1656],[1281,1656]]},{"text":"the","quad":[[1382,1625],[1414,1625],[1414,1656],[1382,1656]]}]},{"text":"sanitization process for media that contain CUl and may require destruction when","quad":[[349,1666],[1459,1665],[1459,1698],[349,1700]],"words":[{"text":"sanitization","quad":[[355,1666],[496,1666],[496,1700],[355,1700]]},{"text":"process","quad":[[519,1666],[615,1666],[615,1700],[519,1700]]},{"text":"for","quad":[[626,1666],[660,1666],[660,1700],[626,1700]]},{"text":"media","quad":[[677,1666],[751,1666],[751,1700],[677,1700]]},{"text":"that","quad":[[768,1666],[813,1666],[813,1700],[768,1700]]},{"text":"contain","quad":[[830,1666],[921,1665],[921,1699],[830,1700]]},{"text":"CUl","quad":[[944,1665],[983,1665],[983,1699],[944,1699]]},{"text":"and","quad":[[995,1665],[1034,1665],[1034,1699],[995,1699]]},{"text":"may","quad":[[1057,1665],[1102,1665],[1102,1699],[1057,1699]]},{"text":"require","quad":[[1119,1665],[1210,1665],[1210,1699],[1119,1699]]},{"text":"destruction","quad":[[1227,1665],[1368,1665],[1368,1699],[1227,1699]]},{"text":"when","quad":[[1391,1665],[1453,1665],[1453,1699],[1391,1699]]}]},{"text":"other methods cannot be applied to the media.","quad":[[351,1706],[993,1708],[993,1739],[351,1737]],"words":[{"text":"other","quad":[[356,1706],[423,1706],[423,1737],[356,1737]]},{"text":"methods","quad":[[439,1706],[548,1707],[548,1738],[439,1737]]},{"text":"cannot","quad":[[568,1707],[651,1707],[651,1738],[568,1738]]},{"text":"be","quad":[[667,1707],[688,1707],[688,1738],[667,1738]]},{"text":"applied","quad":[[708,1707],[801,1707],[801,1738],[708,1738]]},{"text":"to","quad":[[817,1707],[832,1708],[832,1738],[817,1738]]},{"text":"the","quad":[[853,1708],[889,1708],[889,1739],[853,1739]]},{"text":"media.","quad":[[910,1708],[993,1708],[993,1739],[910,1739]]}]},{"text":"REFERENCES","quad":[[349,1780],[524,1781],[524,1811],[349,1810]],"words":[{"text":"REFERENCES","quad":[[359,1780],[519,1781],[519,1811],[359,1810]]}]},{"text":"Source Control: MP-06","quad":[[348,1838],[659,1838],[659,1870],[348,1870]],"words":[{"text":"Source","quad":[[353,1838],[439,1838],[439,1870],[353,1870]]},{"text":"Control:","quad":[[461,1838],[562,1838],[562,1870],[461,1870]]},{"text":"MP-06","quad":[[579,1838],[654,1838],[654,1870],[579,1870]]}]},{"text":"Supporting Publications: SP 800-88 [50]","quad":[[349,1896],[884,1894],[884,1928],[349,1930]],"words":[{"text":"Supporting","quad":[[355,1896],[497,1895],[497,1929],[355,1930]]},{"text":"Publications:","quad":[[514,1895],[679,1895],[679,1929],[514,1929]]},{"text":"SP","quad":[[696,1895],[713,1895],[713,1929],[696,1929]]},{"text":"800-88","quad":[[736,1895],[821,1894],[821,1928],[736,1929]]},{"text":"[50]","quad":[[833,1894],[884,1894],[884,1928],[833,1928]]}]},{"text":"45","quad":[[832,2070],[867,2070],[867,2099],[832,2099]],"words":[{"text":"45","quad":[[842,2070],[861,2070],[861,2099],[842,2099]]}]}],"refused":null} +{"ocr_response":1,"id":"png-file-turned","identity":{"recognizer":"ethos-parser-ocr","version":"0.1.0","rule":"ppocrv5-mobile-v1","det_sha256":"4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae","rec_sha256":"c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8","dict_sha256":"e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6","runtime":"rten 0.26.0; rten-vecmath 0.26.0 with fixed-order sums"},"width":2200,"height":1700,"lines":[{"text":"NIST SP 800-171r3","quad":[[105,1501],[105,1292],[131,1292],[131,1501]],"words":[{"text":"NIST","quad":[[105,1492],[105,1453],[131,1453],[131,1492]]},{"text":"SP","quad":[[105,1440],[105,1423],[131,1423],[131,1440]]},{"text":"800-171r3","quad":[[105,1405],[105,1296],[131,1296],[131,1405]]}]},{"text":"Protecting Controlled Unclassified Information","quad":[[104,725],[104,200],[133,200],[133,725]],"words":[{"text":"Protecting","quad":[[104,720],[104,613],[133,613],[133,720]]},{"text":"Controlled","quad":[[104,594],[104,487],[133,487],[133,594]]},{"text":"Unclassified","quad":[[104,467],[104,341],[133,341],[133,467]]},{"text":"Information","quad":[[104,331],[104,205],[133,205],[133,331]]}]},{"text":"May 2024","quad":[[138,1503],[138,1386],[168,1386],[168,1503]],"words":[{"text":"May","quad":[[138,1493],[138,1453],[168,1453],[168,1493]]},{"text":"2024","quad":[[138,1438],[138,1393],[168,1393],[168,1438]]}]},{"text":"REFERENCES","quad":[[207,1351],[207,1176],[236,1176],[236,1351]],"words":[{"text":"REFERENCES","quad":[[207,1341],[207,1182],[236,1182],[236,1341]]}]},{"text":"Source Control: MP-04","quad":[[264,1351],[264,1043],[294,1043],[294,1351]],"words":[{"text":"Source","quad":[[264,1346],[264,1260],[294,1260],[294,1346]]},{"text":"Control:","quad":[[264,1240],[264,1139],[294,1139],[294,1240]]},{"text":"MP-04","quad":[[264,1119],[264,1048],[294,1048],[294,1119]]}]},{"text":"Supporting Publications: SP 800-88 [50], SP 800-111 [51]","quad":[[320,1352],[319,585],[354,585],[355,1352]],"words":[{"text":"Supporting","quad":[[320,1346],[320,1206],[355,1206],[355,1346]]},{"text":"Publications:","quad":[[320,1188],[320,1018],[355,1018],[355,1188]]},{"text":"SP","quad":[[320,1007],[320,983],[355,983],[355,1007]]},{"text":"800-88","quad":[[319,966],[319,884],[354,884],[354,966]]},{"text":"[50],","quad":[[319,866],[319,802],[354,802],[354,866]]},{"text":"SP","quad":[[319,796],[319,772],[354,772],[354,796]]},{"text":"800-111","quad":[[319,755],[319,649],[354,649],[354,755]]},{"text":"[51]","quad":[[319,638],[319,585],[354,585],[354,638]]}]},{"text":"03.08.02 Media Access","quad":[[417,1501],[419,1163],[449,1163],[447,1502]],"words":[{"text":"03.08.02","quad":[[417,1491],[418,1387],[448,1388],[447,1492]]},{"text":"Media","quad":[[418,1342],[418,1267],[448,1268],[448,1343]]},{"text":"Access","quad":[[418,1248],[419,1168],[449,1169],[448,1248]]}]},{"text":"Restrict access to CUl on system media to authorized personnel or roles.","quad":[[482,1353],[482,370],[516,370],[516,1353]],"words":[{"text":"Restrict","quad":[[482,1342],[482,1251],[516,1251],[516,1342]]},{"text":"access","quad":[[482,1234],[482,1154],[516,1154],[516,1234]]},{"text":"to","quad":[[482,1143],[482,1120],[516,1120],[516,1143]]},{"text":"CUl","quad":[[482,1097],[482,1063],[516,1063],[516,1097]]},{"text":"on","quad":[[482,1046],[482,1023],[516,1023],[516,1046]]},{"text":"system","quad":[[482,1006],[482,927],[516,927],[516,1006]]},{"text":"media","quad":[[482,898],[482,825],[516,825],[516,898]]},{"text":"to","quad":[[482,813],[482,790],[516,790],[516,813]]},{"text":"authorized","quad":[[482,773],[482,637],[516,637],[516,773]]},{"text":"personnel","quad":[[482,620],[482,489],[516,489],[516,620]]},{"text":"or","quad":[[482,472],[482,455],[516,455],[516,472]]},{"text":"roles.","quad":[[482,438],[482,370],[516,370],[516,438]]}]},{"text":"DISCUSSION","quad":[[558,1351],[558,1179],[586,1179],[586,1351]],"words":[{"text":"DISCUSSION","quad":[[558,1342],[558,1188],[586,1188],[586,1342]]}]},{"text":"System media include digital and non-digital media. Access to CUl on system media","quad":[[615,1351],[615,225],[647,225],[647,1351]],"words":[{"text":"System","quad":[[615,1346],[615,1266],[647,1266],[647,1346]]},{"text":"media","quad":[[615,1239],[615,1164],[647,1164],[647,1239]]},{"text":"include","quad":[[615,1148],[615,1063],[647,1063],[647,1148]]},{"text":"digital","quad":[[615,1041],[615,967],[647,967],[647,1041]]},{"text":"and","quad":[[615,951],[615,908],[647,908],[647,951]]},{"text":"non-digital","quad":[[615,892],[615,753],[647,753],[647,892]]},{"text":"media.","quad":[[615,737],[615,652],[647,652],[647,737]]},{"text":"Access","quad":[[615,636],[615,556],[647,556],[647,636]]},{"text":"to","quad":[[615,540],[615,524],[647,524],[647,540]]},{"text":"CUl","quad":[[615,502],[615,460],[647,460],[647,502]]},{"text":"on","quad":[[615,449],[615,428],[647,428],[647,449]]},{"text":"system","quad":[[615,406],[615,332],[647,332],[647,406]]},{"text":"media","quad":[[615,300],[615,230],[647,230],[647,300]]}]},{"text":"can be restricted by physically controlling such media. This includes conducting","quad":[[653,1352],[655,278],[689,278],[687,1352]],"words":[{"text":"can","quad":[[653,1346],[653,1312],[687,1312],[687,1346]]},{"text":"be","quad":[[653,1289],[653,1267],[687,1267],[687,1289]]},{"text":"restricted","quad":[[653,1250],[653,1125],[687,1125],[687,1250]]},{"text":"by","quad":[[653,1108],[653,1085],[687,1085],[687,1108]]},{"text":"physically","quad":[[654,1068],[654,943],[688,943],[688,1068]]},{"text":"controlling","quad":[[654,931],[654,795],[688,795],[688,931]]},{"text":"such","quad":[[654,778],[654,727],[688,727],[688,778]]},{"text":"media.","quad":[[654,704],[654,619],[688,619],[688,704]]},{"text":"This","quad":[[654,602],[654,556],[688,556],[688,602]]},{"text":"includes","quad":[[655,545],[655,437],[689,437],[689,545]]},{"text":"conducting","quad":[[655,420],[655,284],[689,284],[689,420]]}]},{"text":"inventories, ensuring that procedures are in place to allow individuals to check out","quad":[[696,1352],[694,231],[728,231],[730,1352]],"words":[{"text":"inventories,","quad":[[696,1346],[696,1188],[730,1188],[730,1346]]},{"text":"ensuring","quad":[[696,1176],[695,1069],[729,1069],[730,1176]]},{"text":"that","quad":[[695,1052],[695,1007],[729,1007],[729,1052]]},{"text":"procedures","quad":[[695,990],[695,842],[729,842],[729,990]]},{"text":"are","quad":[[695,825],[695,797],[729,797],[729,825]]},{"text":"in","quad":[[695,780],[695,763],[729,763],[729,780]]},{"text":"place","quad":[[695,741],[695,684],[729,684],[729,741]]},{"text":"to","quad":[[695,667],[695,650],[729,650],[729,667]]},{"text":"allow","quad":[[695,627],[695,571],[729,571],[729,627]]},{"text":"individuals","quad":[[695,554],[694,412],[728,412],[729,554]]},{"text":"to","quad":[[694,401],[694,378],[728,378],[728,401]]},{"text":"check","quad":[[694,361],[694,293],[728,293],[728,361]]},{"text":"out","quad":[[694,276],[694,237],[728,237],[728,276]]}]},{"text":"and return media to the media library, and maintaining accountability for stored","quad":[[737,1353],[737,261],[772,261],[772,1353]],"words":[{"text":"and","quad":[[737,1347],[737,1300],[772,1300],[772,1347]]},{"text":"return","quad":[[737,1289],[737,1213],[772,1213],[772,1289]]},{"text":"media","quad":[[737,1189],[737,1119],[772,1119],[772,1189]]},{"text":"to","quad":[[737,1102],[737,1084],[772,1084],[772,1102]]},{"text":"the","quad":[[737,1067],[737,1032],[772,1032],[772,1067]]},{"text":"media","quad":[[737,1008],[737,938],[772,938],[772,1008]]},{"text":"library,","quad":[[737,927],[737,827],[772,827],[772,927]]},{"text":"and","quad":[[737,816],[737,775],[772,775],[772,816]]},{"text":"maintaining","quad":[[737,757],[737,606],[772,606],[772,757]]},{"text":"accountability","quad":[[737,588],[737,407],[772,407],[772,588]]},{"text":"for","quad":[[737,395],[737,360],[772,360],[772,395]]},{"text":"stored","quad":[[737,343],[737,267],[772,267],[772,343]]}]},{"text":"media. For digital media, access to CUl can be restricted by using cryptographic","quad":[[777,1351],[777,278],[811,278],[811,1351]],"words":[{"text":"media.","quad":[[777,1345],[777,1254],[811,1254],[811,1345]]},{"text":"For","quad":[[777,1243],[777,1209],[811,1209],[811,1243]]},{"text":"digital","quad":[[777,1192],[777,1113],[811,1113],[811,1192]]},{"text":"media,","quad":[[777,1101],[777,1016],[811,1016],[811,1101]]},{"text":"access","quad":[[777,999],[777,925],[811,925],[811,999]]},{"text":"to","quad":[[777,908],[777,891],[811,891],[811,908]]},{"text":"CUl","quad":[[777,868],[777,829],[811,829],[811,868]]},{"text":"can","quad":[[777,812],[777,778],[811,778],[811,812]]},{"text":"be","quad":[[777,761],[777,738],[811,738],[811,761]]},{"text":"restricted","quad":[[777,721],[777,596],[811,596],[811,721]]},{"text":"by","quad":[[777,579],[777,556],[811,556],[811,579]]},{"text":"using","quad":[[777,539],[777,477],[811,477],[811,539]]},{"text":"cryptographic","quad":[[777,460],[777,284],[811,284],[811,460]]}]},{"text":"means. Encrypting data in storage or at rest is addressed in 03.13.08.","quad":[[819,1350],[817,418],[850,418],[852,1350]],"words":[{"text":"means.","quad":[[819,1344],[819,1251],[852,1251],[852,1344]]},{"text":"Encrypting","quad":[[819,1240],[818,1107],[851,1107],[852,1240]]},{"text":"data","quad":[[818,1085],[818,1036],[851,1036],[851,1085]]},{"text":"in","quad":[[818,1025],[818,1008],[851,1008],[851,1025]]},{"text":"storage","quad":[[818,986],[818,892],[851,892],[851,986]]},{"text":"or","quad":[[818,876],[818,854],[851,854],[851,876]]},{"text":"at","quad":[[818,837],[818,821],[851,821],[851,837]]},{"text":"rest","quad":[[818,804],[818,760],[851,760],[851,804]]},{"text":"is","quad":[[818,749],[818,732],[851,732],[851,749]]},{"text":"addressed","quad":[[818,716],[817,583],[850,583],[851,716]]},{"text":"in","quad":[[817,572],[817,556],[850,556],[850,572]]},{"text":"03.13.08.","quad":[[817,534],[817,418],[850,418],[850,534]]}]},{"text":"REFERENCES","quad":[[892,1351],[892,1176],[921,1176],[921,1351]],"words":[{"text":"REFERENCES","quad":[[892,1341],[892,1182],[921,1182],[921,1341]]}]},{"text":"Source Control: MP-02","quad":[[949,1352],[949,1041],[981,1041],[981,1352]],"words":[{"text":"Source","quad":[[949,1347],[949,1261],[981,1261],[981,1347]]},{"text":"Control:","quad":[[949,1239],[949,1138],[981,1138],[981,1239]]},{"text":"MP-02","quad":[[949,1121],[949,1046],[981,1046],[981,1121]]}]},{"text":"Supporting Publications: SP 800-111 [51]","quad":[[1007,1352],[1005,797],[1039,797],[1042,1351]],"words":[{"text":"Supporting","quad":[[1007,1346],[1006,1206],[1041,1205],[1042,1345]]},{"text":"Publications:","quad":[[1006,1188],[1006,1019],[1041,1018],[1041,1187]]},{"text":"SP","quad":[[1006,1007],[1006,984],[1041,983],[1041,1006]]},{"text":"800-111","quad":[[1006,966],[1005,867],[1040,866],[1041,965]]},{"text":"[51]","quad":[[1005,850],[1005,803],[1040,802],[1040,849]]}]},{"text":"03.08.03 Media Sanitization","quad":[[1102,1501],[1104,1092],[1134,1092],[1132,1502]],"words":[{"text":"03.08.03","quad":[[1102,1491],[1103,1386],[1133,1387],[1132,1492]]},{"text":"Media","quad":[[1103,1341],[1103,1267],[1133,1268],[1133,1342]]},{"text":"Sanitization","quad":[[1103,1247],[1104,1102],[1134,1103],[1133,1248]]}]},{"text":"Sanitize system media that contain CUl prior to disposal, release out of","quad":[[1169,1352],[1168,389],[1201,389],[1202,1352]],"words":[{"text":"Sanitize","quad":[[1169,1346],[1169,1247],[1202,1247],[1202,1346]]},{"text":"system","quad":[[1169,1231],[1169,1154],[1202,1154],[1202,1231]]},{"text":"media","quad":[[1169,1126],[1169,1055],[1202,1055],[1202,1126]]},{"text":"that","quad":[[1169,1038],[1169,989],[1202,989],[1202,1038]]},{"text":"contain","quad":[[1169,972],[1169,884],[1202,884],[1202,972]]},{"text":"CUl","quad":[[1168,862],[1168,824],[1201,824],[1201,862]]},{"text":"prior","quad":[[1168,807],[1168,752],[1201,752],[1201,807]]},{"text":"to","quad":[[1168,736],[1168,719],[1201,719],[1201,736]]},{"text":"disposal,","quad":[[1168,697],[1168,587],[1201,587],[1201,697]]},{"text":"release","quad":[[1168,576],[1168,488],[1201,488],[1201,576]]},{"text":"out","quad":[[1168,466],[1168,433],[1201,433],[1201,466]]},{"text":"of","quad":[[1168,417],[1168,395],[1201,395],[1201,417]]}]},{"text":"organizational control, or release for reuse.","quad":[[1210,1350],[1209,762],[1242,762],[1243,1350]],"words":[{"text":"organizational","quad":[[1210,1345],[1210,1158],[1243,1158],[1243,1345]]},{"text":"control,","quad":[[1210,1141],[1209,1042],[1242,1042],[1243,1141]]},{"text":"or","quad":[[1209,1031],[1209,1009],[1242,1009],[1242,1031]]},{"text":"release","quad":[[1209,998],[1209,910],[1242,910],[1242,998]]},{"text":"for","quad":[[1209,894],[1209,855],[1242,855],[1242,894]]},{"text":"reuse.","quad":[[1209,844],[1209,767],[1242,767],[1242,844]]}]},{"text":"DISCUSSION","quad":[[1283,1351],[1283,1179],[1314,1179],[1314,1351]],"words":[{"text":"DISCUSSION","quad":[[1283,1341],[1283,1191],[1314,1191],[1314,1341]]}]},{"text":"Media sanitization applies to digital and non-digital media that are subject to","quad":[[1340,1352],[1341,309],[1375,309],[1374,1352]],"words":[{"text":"Media","quad":[[1340,1341],[1340,1267],[1374,1267],[1374,1341]]},{"text":"sanitization","quad":[[1340,1250],[1340,1108],[1374,1108],[1374,1250]]},{"text":"applies","quad":[[1340,1086],[1340,1001],[1374,1001],[1374,1086]]},{"text":"to","quad":[[1340,984],[1340,967],[1374,967],[1374,984]]},{"text":"digital","quad":[[1340,944],[1340,870],[1374,870],[1374,944]]},{"text":"and","quad":[[1340,859],[1341,813],[1375,813],[1374,859]]},{"text":"non-digital","quad":[[1341,796],[1341,660],[1375,660],[1375,796]]},{"text":"media","quad":[[1341,643],[1341,570],[1375,570],[1375,643]]},{"text":"that","quad":[[1341,553],[1341,507],[1375,507],[1375,553]]},{"text":"are","quad":[[1341,490],[1341,456],[1375,456],[1375,490]]},{"text":"subject","quad":[[1341,439],[1341,349],[1375,349],[1375,439]]},{"text":"to","quad":[[1341,332],[1341,315],[1375,315],[1375,332]]}]},{"text":"disposal or reuse, whether or not the media are considered removable. Examples","quad":[[1381,1351],[1382,249],[1416,249],[1415,1352]],"words":[{"text":"disposal","quad":[[1381,1345],[1381,1238],[1415,1239],[1415,1346]]},{"text":"or","quad":[[1381,1227],[1381,1204],[1415,1205],[1415,1228]]},{"text":"reuse,","quad":[[1381,1193],[1381,1114],[1415,1115],[1415,1194]]},{"text":"whether","quad":[[1381,1097],[1381,995],[1415,996],[1415,1098]]},{"text":"or","quad":[[1381,978],[1381,955],[1415,956],[1415,979]]},{"text":"not","quad":[[1381,938],[1381,905],[1415,906],[1415,939]]},{"text":"the","quad":[[1381,888],[1381,854],[1415,855],[1415,889]]},{"text":"media","quad":[[1381,831],[1382,758],[1416,759],[1415,832]]},{"text":"are","quad":[[1382,741],[1382,707],[1416,708],[1416,742]]},{"text":"considered","quad":[[1382,690],[1382,554],[1416,555],[1416,691]]},{"text":"removable.","quad":[[1382,537],[1382,385],[1416,386],[1416,538]]},{"text":"Examples","quad":[[1382,373],[1382,255],[1416,256],[1416,374]]}]},{"text":"include digital media in scanners, copiers, printers, notebook computers, mobile","quad":[[1421,1353],[1420,262],[1456,262],[1457,1353]],"words":[{"text":"include","quad":[[1421,1347],[1421,1257],[1457,1257],[1457,1347]]},{"text":"digital","quad":[[1421,1239],[1421,1161],[1457,1161],[1457,1239]]},{"text":"media","quad":[[1421,1149],[1421,1071],[1457,1071],[1457,1149]]},{"text":"in","quad":[[1421,1059],[1421,1041],[1457,1041],[1457,1059]]},{"text":"scanners,","quad":[[1421,1023],[1421,903],[1457,903],[1457,1023]]},{"text":"copiers,","quad":[[1421,885],[1420,790],[1456,790],[1457,885]]},{"text":"printers,","quad":[[1420,772],[1420,664],[1456,664],[1456,772]]},{"text":"notebook","quad":[[1420,652],[1420,532],[1456,532],[1456,652]]},{"text":"computers,","quad":[[1420,514],[1420,364],[1456,364],[1456,514]]},{"text":"mobile","quad":[[1420,352],[1420,268],[1456,268],[1456,352]]}]},{"text":"devices, workstations, network components, and non-digital media. The sanitization","quad":[[1462,1352],[1462,209],[1497,209],[1497,1352]],"words":[{"text":"devices,","quad":[[1462,1340],[1462,1241],[1497,1241],[1497,1340]]},{"text":"workstations,","quad":[[1462,1224],[1462,1049],[1497,1049],[1497,1224]]},{"text":"network","quad":[[1462,1037],[1462,932],[1497,932],[1497,1037]]},{"text":"components,","quad":[[1462,915],[1462,746],[1497,746],[1497,915]]},{"text":"and","quad":[[1462,728],[1462,693],[1497,693],[1497,728]]},{"text":"non-digital","quad":[[1462,676],[1462,536],[1497,536],[1497,676]]},{"text":"media.","quad":[[1462,518],[1462,431],[1497,431],[1497,518]]},{"text":"The","quad":[[1462,419],[1462,378],[1497,378],[1497,419]]},{"text":"sanitization","quad":[[1462,361],[1462,221],[1497,221],[1497,361]]}]},{"text":"process removes Cul from media such that the information cannot be retrieved or","quad":[[1505,1352],[1503,234],[1536,234],[1538,1352]],"words":[{"text":"process","quad":[[1505,1346],[1505,1247],[1538,1247],[1538,1346]]},{"text":"removes","quad":[[1505,1236],[1505,1126],[1538,1126],[1538,1236]]},{"text":"Cul","quad":[[1505,1104],[1504,1066],[1537,1066],[1538,1104]]},{"text":"from","quad":[[1504,1055],[1504,1005],[1537,1005],[1537,1055]]},{"text":"media","quad":[[1504,977],[1504,906],[1537,906],[1537,977]]},{"text":"such","quad":[[1504,889],[1504,840],[1537,840],[1537,889]]},{"text":"that","quad":[[1504,818],[1504,768],[1537,768],[1537,818]]},{"text":"the","quad":[[1504,757],[1504,719],[1537,719],[1537,757]]},{"text":"information","quad":[[1504,708],[1504,553],[1537,553],[1537,708]]},{"text":"cannot","quad":[[1504,531],[1503,449],[1536,449],[1537,531]]},{"text":"be","quad":[[1503,432],[1503,410],[1536,410],[1536,432]]},{"text":"retrieved","quad":[[1503,388],[1503,273],[1536,273],[1536,388]]},{"text":"or","quad":[[1503,256],[1503,240],[1536,240],[1536,256]]}]},{"text":"reconstructed. Sanitization techniques (e.g., cryptographically erasing, clearing,","quad":[[1543,1353],[1545,275],[1579,275],[1578,1353]],"words":[{"text":"reconstructed.","quad":[[1543,1347],[1543,1149],[1578,1149],[1578,1347]]},{"text":"Sanitization","quad":[[1543,1137],[1544,992],[1579,992],[1578,1137]]},{"text":"techniques","quad":[[1544,974],[1544,834],[1579,834],[1579,974]]},{"text":"(e.g.,","quad":[[1544,817],[1544,753],[1579,753],[1579,817]]},{"text":"cryptographically","quad":[[1544,741],[1545,520],[1580,520],[1579,741]]},{"text":"erasing,","quad":[[1545,502],[1545,403],[1580,403],[1580,502]]},{"text":"clearing,","quad":[[1545,386],[1545,275],[1580,275],[1580,386]]}]},{"text":"purging, and destroying) prevent the disclosure of CUl to unauthorized individuals","quad":[[1586,1351],[1582,240],[1615,240],[1619,1351]],"words":[{"text":"purging,","quad":[[1586,1346],[1586,1241],[1619,1241],[1619,1346]]},{"text":"and","quad":[[1586,1224],[1585,1186],[1618,1186],[1619,1224]]},{"text":"destroying)","quad":[[1585,1164],[1585,1021],[1618,1021],[1618,1164]]},{"text":"prevent","quad":[[1585,1004],[1584,911],[1617,911],[1618,1004]]},{"text":"the","quad":[[1584,894],[1584,856],[1617,856],[1617,894]]},{"text":"disclosure","quad":[[1584,840],[1584,713],[1617,713],[1617,840]]},{"text":"of","quad":[[1584,696],[1584,674],[1617,674],[1617,696]]},{"text":"CUl","quad":[[1584,658],[1583,620],[1616,620],[1617,658]]},{"text":"to","quad":[[1583,608],[1583,586],[1616,586],[1616,608]]},{"text":"unauthorized","quad":[[1583,570],[1583,394],[1616,394],[1616,570]]},{"text":"individuals","quad":[[1583,383],[1582,246],[1615,246],[1616,383]]}]},{"text":"when such media are reused or released for disposal. NARA policies control the","quad":[[1627,1351],[1625,275],[1657,275],[1658,1351]],"words":[{"text":"when","quad":[[1627,1340],[1627,1282],[1658,1282],[1658,1340]]},{"text":"such","quad":[[1627,1260],[1627,1213],[1658,1213],[1658,1260]]},{"text":"media","quad":[[1627,1191],[1627,1117],[1658,1117],[1658,1191]]},{"text":"are","quad":[[1627,1101],[1626,1069],[1657,1069],[1658,1101]]},{"text":"reused","quad":[[1626,1047],[1626,967],[1657,967],[1657,1047]]},{"text":"or","quad":[[1626,946],[1626,930],[1657,930],[1657,946]]},{"text":"released","quad":[[1626,914],[1626,808],[1657,808],[1657,914]]},{"text":"for","quad":[[1626,792],[1626,760],[1657,760],[1657,792]]},{"text":"disposal.","quad":[[1626,738],[1626,627],[1657,627],[1657,738]]},{"text":"NARA","quad":[[1626,616],[1626,552],[1657,552],[1657,616]]},{"text":"policies","quad":[[1625,531],[1625,440],[1656,440],[1656,531]]},{"text":"control","quad":[[1625,419],[1625,334],[1656,334],[1656,419]]},{"text":"the","quad":[[1625,318],[1625,286],[1656,286],[1656,318]]}]},{"text":"sanitization process for media that contain CUl and may require destruction when","quad":[[1666,1351],[1665,241],[1698,241],[1700,1351]],"words":[{"text":"sanitization","quad":[[1666,1345],[1666,1204],[1700,1204],[1700,1345]]},{"text":"process","quad":[[1666,1181],[1666,1085],[1700,1085],[1700,1181]]},{"text":"for","quad":[[1666,1074],[1666,1040],[1700,1040],[1700,1074]]},{"text":"media","quad":[[1666,1023],[1666,949],[1700,949],[1700,1023]]},{"text":"that","quad":[[1666,932],[1666,887],[1700,887],[1700,932]]},{"text":"contain","quad":[[1666,870],[1665,779],[1699,779],[1700,870]]},{"text":"CUl","quad":[[1665,756],[1665,717],[1699,717],[1699,756]]},{"text":"and","quad":[[1665,705],[1665,666],[1699,666],[1699,705]]},{"text":"may","quad":[[1665,643],[1665,598],[1699,598],[1699,643]]},{"text":"require","quad":[[1665,581],[1665,490],[1699,490],[1699,581]]},{"text":"destruction","quad":[[1665,473],[1665,332],[1699,332],[1699,473]]},{"text":"when","quad":[[1665,309],[1665,247],[1699,247],[1699,309]]}]},{"text":"other methods cannot be applied to the media.","quad":[[1706,1349],[1708,707],[1739,707],[1737,1349]],"words":[{"text":"other","quad":[[1706,1344],[1706,1277],[1737,1277],[1737,1344]]},{"text":"methods","quad":[[1706,1261],[1707,1152],[1738,1152],[1737,1261]]},{"text":"cannot","quad":[[1707,1132],[1707,1049],[1738,1049],[1738,1132]]},{"text":"be","quad":[[1707,1033],[1707,1012],[1738,1012],[1738,1033]]},{"text":"applied","quad":[[1707,992],[1707,899],[1738,899],[1738,992]]},{"text":"to","quad":[[1707,883],[1708,868],[1738,868],[1738,883]]},{"text":"the","quad":[[1708,847],[1708,811],[1739,811],[1739,847]]},{"text":"media.","quad":[[1708,790],[1708,707],[1739,707],[1739,790]]}]},{"text":"REFERENCES","quad":[[1780,1351],[1781,1176],[1811,1176],[1810,1351]],"words":[{"text":"REFERENCES","quad":[[1780,1341],[1781,1181],[1811,1181],[1810,1341]]}]},{"text":"Source Control: MP-06","quad":[[1838,1352],[1838,1041],[1870,1041],[1870,1352]],"words":[{"text":"Source","quad":[[1838,1347],[1838,1261],[1870,1261],[1870,1347]]},{"text":"Control:","quad":[[1838,1239],[1838,1138],[1870,1138],[1870,1239]]},{"text":"MP-06","quad":[[1838,1121],[1838,1046],[1870,1046],[1870,1121]]}]},{"text":"Supporting Publications: SP 800-88 [50]","quad":[[1896,1351],[1894,816],[1928,816],[1930,1351]],"words":[{"text":"Supporting","quad":[[1896,1345],[1895,1203],[1929,1203],[1930,1345]]},{"text":"Publications:","quad":[[1895,1186],[1895,1021],[1929,1021],[1929,1186]]},{"text":"SP","quad":[[1895,1004],[1895,987],[1929,987],[1929,1004]]},{"text":"800-88","quad":[[1895,964],[1894,879],[1928,879],[1929,964]]},{"text":"[50]","quad":[[1894,867],[1894,816],[1928,816],[1928,867]]}]},{"text":"45","quad":[[2070,868],[2070,833],[2099,833],[2099,868]],"words":[{"text":"45","quad":[[2070,858],[2070,839],[2099,839],[2099,858]]}]}],"refused":null} +{"ocr_response":1,"id":"pdf-dct-gray","identity":{"recognizer":"ethos-parser-ocr","version":"0.1.0","rule":"ppocrv5-mobile-v1","det_sha256":"4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae","rec_sha256":"c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8","dict_sha256":"e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6","runtime":"rten 0.26.0; rten-vecmath 0.26.0 with fixed-order sums"},"width":1700,"height":2200,"lines":[{"text":"NIST SP 800-37, REVISION 2","quad":[[249,103],[502,103],[502,125],[249,125]],"words":[{"text":"NIST","quad":[[256,103],[286,103],[286,125],[256,125]]},{"text":"SP","quad":[[300,103],[315,103],[315,125],[300,125]]},{"text":"800-37,","quad":[[326,103],[392,103],[392,125],[326,125]]},{"text":"REVISION","quad":[[403,103],[476,103],[476,125],[403,125]]},{"text":"2","quad":[[491,103],[495,103],[495,125],[491,125]]}]},{"text":"RISK MANAGEMENT FRAMEWORK FOR INFORMATION SYSTEMS AND ORGANIZATIONS","quad":[[664,104],[1448,104],[1448,123],[664,123]],"words":[{"text":"RISK","quad":[[670,104],[699,104],[699,123],[670,123]]},{"text":"MANAGEMENT","quad":[[715,104],[845,104],[845,123],[715,123]]},{"text":"FRAMEWORK","quad":[[858,104],[972,104],[972,123],[858,123]]},{"text":"FOR","quad":[[985,104],[1013,104],[1013,123],[985,123]]},{"text":"INFORMATION","quad":[[1026,104],[1153,104],[1153,123],[1026,123]]},{"text":"SYSTEMS","quad":[[1166,104],[1242,104],[1242,123],[1166,123]]},{"text":"AND","quad":[[1254,104],[1286,104],[1286,123],[1254,123]]},{"text":"ORGANIZATIONS","quad":[[1302,104],[1445,104],[1445,123],[1302,123]]}]},{"text":"A System Life Cycle Approach for Security and Privacy","quad":[[1023,127],[1450,128],[1450,152],[1023,150]],"words":[{"text":"A","quad":[[1031,127],[1035,127],[1035,150],[1031,150]]},{"text":"System","quad":[[1047,127],[1091,127],[1091,150],[1047,150]]},{"text":"Life","quad":[[1107,127],[1131,127],[1131,150],[1107,150]]},{"text":"Cycle","quad":[[1139,127],[1175,127],[1175,150],[1139,150]]},{"text":"Approach","quad":[[1187,127],[1254,128],[1254,151],[1187,150]]},{"text":"for","quad":[[1266,128],[1286,128],[1286,151],[1266,151]]},{"text":"Security","quad":[[1294,128],[1354,128],[1354,151],[1294,151]]},{"text":"and","quad":[[1362,128],[1386,128],[1386,151],[1362,151]]},{"text":"Privacy","quad":[[1398,128],[1446,128],[1446,151],[1398,151]]}]},{"text":"RMF TASKS","quad":[[383,224],[516,224],[516,248],[383,248]],"words":[{"text":"RMF","quad":[[391,224],[435,224],[435,248],[391,248]]},{"text":"TASKS","quad":[[447,224],[511,224],[511,248],[447,248]]}]},{"text":"PRIMARY RESPONSIBILITY","quad":[[693,225],[992,225],[992,247],[693,247]],"words":[{"text":"PRIMARY","quad":[[700,225],[796,225],[796,247],[700,247]]},{"text":"RESPONSIBILITY","quad":[[815,225],[988,225],[988,247],[815,247]]}]},{"text":"SUPPORTING ROLES","quad":[[1128,225],[1359,225],[1359,247],[1128,247]],"words":[{"text":"SUPPORTING","quad":[[1132,225],[1275,225],[1275,247],[1132,247]]},{"text":"ROLES","quad":[[1293,225],[1355,225],[1355,247],[1293,247]]}]},{"text":"TASK M-5","quad":[[263,292],[368,292],[368,316],[263,316]],"words":[{"text":"TASK","quad":[[271,292],[311,292],[311,316],[271,316]]},{"text":"M-5","quad":[[331,292],[363,292],[363,316],[331,316]]}]},{"text":"System Owner","quad":[[664,294],[834,294],[834,318],[664,318]],"words":[{"text":"System","quad":[[688,294],[749,294],[749,318],[688,318]]},{"text":"Owner","quad":[[773,294],[830,294],[830,318],[773,318]]}]},{"text":"System Security Officer","quad":[[1050,294],[1308,294],[1308,318],[1050,318]],"words":[{"text":"System","quad":[[1078,294],[1135,294],[1135,318],[1078,318]]},{"text":"Security","quad":[[1155,294],[1231,294],[1231,318],[1155,318]]},{"text":"Officer","quad":[[1248,294],[1308,294],[1308,318],[1248,318]]}]},{"text":"Common Control Provider","quad":[[664,323],[952,325],[951,349],[664,348]],"words":[{"text":"Common","quad":[[693,323],[773,324],[773,349],[693,348]]},{"text":"Control","quad":[[789,324],[860,324],[860,349],[789,349]]},{"text":"Provider","quad":[[869,324],[948,325],[948,350],[869,349]]}]},{"text":"System Privacy Officer","quad":[[1050,323],[1301,324],[1301,351],[1050,349]],"words":[{"text":"System","quad":[[1077,323],[1135,323],[1135,349],[1077,349]]},{"text":"Privacy","quad":[[1158,323],[1220,324],[1220,350],[1158,349]]},{"text":"Officer","quad":[[1234,324],[1297,324],[1297,350],[1234,350]]}]},{"text":"Security and Privacy Reporting","quad":[[264,338],[582,340],[582,367],[264,364]],"words":[{"text":"Security","quad":[[268,338],[345,339],[345,365],[268,364]]},{"text":"and","quad":[[358,339],[389,339],[389,365],[358,365]]},{"text":"Privacy","quad":[[403,339],[470,339],[470,365],[403,365]]},{"text":"Reporting","quad":[[483,339],[578,340],[578,366],[483,365]]}]},{"text":"Senior Agency Information","quad":[[665,356],[956,358],[955,382],[665,380]],"words":[{"text":"Senior","quad":[[689,356],[749,357],[749,381],[689,380]]},{"text":"Agency","quad":[[761,357],[824,357],[824,381],[761,381]]},{"text":"Information","quad":[[836,357],[948,358],[948,382],[836,381]]}]},{"text":"Report the security and privacy","quad":[[264,377],[582,379],[582,406],[264,403]],"words":[{"text":"Report","quad":[[268,377],[331,377],[331,403],[268,403]]},{"text":"the","quad":[[340,377],[367,378],[367,404],[340,403]]},{"text":"security","quad":[[380,378],[457,378],[457,404],[380,404]]},{"text":"and","quad":[[470,378],[497,378],[497,404],[470,404]]},{"text":"privacy","quad":[[510,379],[578,379],[578,405],[510,405]]}]},{"text":"Security Officer","quad":[[680,386],[843,386],[843,412],[680,412]],"words":[{"text":"Security","quad":[[689,386],[762,386],[762,412],[689,412]]},{"text":"Officer","quad":[[780,386],[840,386],[840,412],[780,412]]}]},{"text":"posture of the system to the","quad":[[264,408],[554,408],[554,433],[264,433]],"words":[{"text":"posture","quad":[[268,408],[340,408],[340,433],[268,433]]},{"text":"of","quad":[[352,408],[369,408],[369,433],[352,433]]},{"text":"the","quad":[[377,408],[407,408],[407,433],[377,433]]},{"text":"system","quad":[[420,408],[478,408],[478,433],[420,433]]},{"text":"to","quad":[[495,408],[512,408],[512,433],[495,433]]},{"text":"the","quad":[[525,408],[550,408],[550,433],[525,433]]}]},{"text":"Senior Agency Official for Privacy","quad":[[664,417],[1021,419],[1021,445],[664,443]],"words":[{"text":"Senior","quad":[[690,417],[747,417],[747,443],[690,443]]},{"text":"Agency","quad":[[760,418],[825,418],[825,444],[760,444]]},{"text":"Official","quad":[[842,418],[908,418],[908,444],[842,444]]},{"text":"for","quad":[[917,418],[943,419],[943,445],[917,444]]},{"text":"Privacy","quad":[[951,419],[1017,419],[1017,445],[951,445]]}]},{"text":"authorizing official and other","quad":[[263,438],[559,439],[559,465],[263,464]],"words":[{"text":"authorizing","quad":[[267,438],[376,438],[376,464],[267,464]]},{"text":"official","quad":[[389,438],[455,439],[455,465],[389,464]]},{"text":"and","quad":[[463,439],[494,439],[494,465],[463,465]]},{"text":"other","quad":[[507,439],[555,439],[555,465],[507,465]]}]},{"text":"organizational officials on an","quad":[[263,469],[557,468],[557,494],[263,495]],"words":[{"text":"organizational","quad":[[272,469],[410,468],[410,494],[272,495]]},{"text":"officials","quad":[[419,468],[488,468],[488,494],[419,494]]},{"text":"on","quad":[[505,468],[518,468],[518,494],[505,494]]},{"text":"an","quad":[[535,468],[548,468],[548,494],[535,494]]}]},{"text":"ongoing basis in accordance with","quad":[[263,500],[599,500],[599,524],[263,524]],"words":[{"text":"ongoing","quad":[[271,500],[343,500],[343,524],[271,524]]},{"text":"basis","quad":[[355,500],[399,500],[399,524],[355,524]]},{"text":"in","quad":[[411,500],[423,500],[423,524],[411,524]]},{"text":"accordance","quad":[[439,500],[547,500],[547,524],[439,524]]},{"text":"with","quad":[[563,500],[595,500],[595,524],[563,524]]}]},{"text":"the organizational continuous","quad":[[263,530],[568,531],[568,555],[263,554]],"words":[{"text":"the","quad":[[267,530],[295,530],[295,554],[267,554]]},{"text":"organizational","quad":[[307,530],[448,531],[448,555],[307,554]]},{"text":"continuous","quad":[[460,531],[564,531],[564,555],[460,555]]}]},{"text":"monitoring strategy.","quad":[[263,559],[473,563],[473,589],[263,585]],"words":[{"text":"monitoring","quad":[[272,559],[377,561],[377,587],[272,585]]},{"text":"strategy.","quad":[[386,561],[473,563],[473,589],[386,587]]}]},{"text":"This publication is available free of charge from: https://doi. org/10.6028/NIST.SP.800-37r2","quad":[[73,618],[73,1622],[43,1622],[43,618]],"words":[{"text":"This","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"publication","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"is","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"available","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"free","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"of","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"charge","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"from:","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"https://doi.","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"org/10.6028/NIST.SP.800-37r2","quad":[[73,618],[73,1622],[43,1622],[43,618]]}]},{"text":"TASK M-6","quad":[[263,626],[367,626],[367,649],[263,649]],"words":[{"text":"TASK","quad":[[271,626],[313,626],[313,649],[271,649]]},{"text":"M-6","quad":[[332,626],[362,626],[362,649],[332,649]]}]},{"text":"Authorizing Official","quad":[[664,627],[880,627],[880,651],[664,651]],"words":[{"text":"Authorizing","quad":[[692,627],[796,627],[796,651],[692,651]]},{"text":"Official","quad":[[816,627],[876,627],[876,651],[816,651]]}]},{"text":"Senior Accountable Official for Risk","quad":[[1050,625],[1428,625],[1428,650],[1050,650]],"words":[{"text":"Senior","quad":[[1075,625],[1133,625],[1133,650],[1075,650]]},{"text":"Accountable","quad":[[1146,625],[1266,625],[1266,650],[1146,650]]},{"text":"Official","quad":[[1283,625],[1345,625],[1345,650],[1283,650]]},{"text":"for","quad":[[1353,625],[1378,625],[1378,650],[1353,650]]},{"text":"Risk","quad":[[1391,625],[1424,625],[1424,650],[1391,650]]}]},{"text":"Management or Risk Executive","quad":[[1071,657],[1385,658],[1384,683],[1071,681]],"words":[{"text":"Management","quad":[[1079,657],[1205,657],[1205,681],[1079,681]]},{"text":"or","quad":[[1218,657],[1234,658],[1234,682],[1218,681]]},{"text":"Risk","quad":[[1247,658],[1280,658],[1280,682],[1247,682]]},{"text":"Executive","quad":[[1293,658],[1381,658],[1381,682],[1293,682]]}]},{"text":"Ongoing Authorization","quad":[[264,672],[500,672],[500,697],[264,697]],"words":[{"text":"Ongoing","quad":[[272,672],[347,672],[347,697],[272,697]]},{"text":"Authorization","quad":[[363,672],[492,672],[492,697],[363,697]]}]},{"text":"(Function)","quad":[[1069,687],[1178,687],[1178,713],[1069,713]],"words":[{"text":"(Function)","quad":[[1073,687],[1177,687],[1177,713],[1073,713]]}]},{"text":"Review the security and privacy","quad":[[263,710],[585,713],[585,739],[263,736]],"words":[{"text":"Review","quad":[[272,710],[328,711],[328,737],[272,736]]},{"text":"the","quad":[[346,711],[372,711],[372,737],[346,737]]},{"text":"security","quad":[[385,711],[463,712],[463,738],[385,737]]},{"text":"and","quad":[[476,712],[502,712],[502,738],[476,738]]},{"text":"privacy","quad":[[520,712],[585,713],[585,739],[520,738]]}]},{"text":"Chief Information Officer","quad":[[1051,718],[1329,719],[1329,745],[1051,744]],"words":[{"text":"Chief","quad":[[1077,718],[1125,718],[1125,744],[1077,744]]},{"text":"Information","quad":[[1134,718],[1246,719],[1246,745],[1134,744]]},{"text":"Officer","quad":[[1264,719],[1325,719],[1325,745],[1264,745]]}]},{"text":"posture of the system on an","quad":[[263,743],[551,743],[551,767],[263,767]],"words":[{"text":"posture","quad":[[271,743],[339,743],[339,767],[271,767]]},{"text":"of","quad":[[355,743],[367,743],[367,767],[355,767]]},{"text":"the","quad":[[379,743],[403,743],[403,767],[379,767]]},{"text":"system","quad":[[419,743],[475,743],[475,767],[419,767]]},{"text":"on","quad":[[499,743],[515,743],[515,767],[499,767]]},{"text":"an","quad":[[531,743],[543,743],[543,767],[531,767]]}]},{"text":"Senior Agency Information Security","quad":[[1051,751],[1432,752],[1432,777],[1051,775]],"words":[{"text":"Senior","quad":[[1076,751],[1135,751],[1135,775],[1076,775]]},{"text":"Agency","quad":[[1147,751],[1214,751],[1214,775],[1147,775]]},{"text":"Information","quad":[[1223,751],[1340,752],[1340,776],[1223,775]]},{"text":"Security","quad":[[1352,752],[1428,752],[1428,776],[1352,776]]}]},{"text":"ongoing basis to determine whether","quad":[[263,773],[635,773],[635,797],[263,797]],"words":[{"text":"ongoing","quad":[[271,773],[343,773],[343,797],[271,797]]},{"text":"basis","quad":[[355,773],[399,773],[399,797],[355,797]]},{"text":"to","quad":[[411,773],[427,773],[427,797],[411,797]]},{"text":"determine","quad":[[443,773],[539,773],[539,797],[443,797]]},{"text":"whether","quad":[[555,773],[631,773],[631,797],[555,797]]}]},{"text":"Officer","quad":[[1069,780],[1144,783],[1143,807],[1068,805]],"words":[{"text":"Officer","quad":[[1077,780],[1140,783],[1139,808],[1076,805]]}]},{"text":"the risk remains acceptable.","quad":[[263,802],[551,803],[551,829],[263,828]],"words":[{"text":"the","quad":[[267,802],[294,802],[294,828],[267,828]]},{"text":"risk","quad":[[307,802],[337,802],[337,828],[307,828]]},{"text":"remains","quad":[[350,802],[424,803],[424,829],[350,828]]},{"text":"acceptable.","quad":[[438,803],[551,803],[551,829],[438,829]]}]},{"text":"Senior Agency Official for Privacy","quad":[[1050,811],[1409,813],[1409,841],[1050,839]],"words":[{"text":"Senior","quad":[[1078,811],[1134,811],[1134,839],[1078,839]]},{"text":"Agency","quad":[[1148,812],[1213,812],[1213,840],[1148,840]]},{"text":"Official","quad":[[1227,812],[1292,812],[1292,840],[1227,840]]},{"text":"for","quad":[[1302,812],[1325,813],[1325,841],[1302,840]]},{"text":"Privacy","quad":[[1339,813],[1404,813],[1404,841],[1339,841]]}]},{"text":"Authorizing Official Designated","quad":[[1054,846],[1387,846],[1387,870],[1054,870]],"words":[{"text":"Authorizing","quad":[[1078,846],[1186,846],[1186,870],[1078,870]]},{"text":"Official","quad":[[1202,846],[1267,846],[1267,870],[1202,870]]},{"text":"Designated","quad":[[1279,846],[1383,846],[1383,870],[1279,870]]}]},{"text":"Representative","quad":[[1072,878],[1225,878],[1225,900],[1072,900]],"words":[{"text":"Representative","quad":[[1079,878],[1221,878],[1221,900],[1079,900]]}]},{"text":"TASK M-7","quad":[[262,941],[367,941],[367,967],[262,967]],"words":[{"text":"TASK","quad":[[271,941],[314,941],[314,967],[271,967]]},{"text":"M-7","quad":[[331,941],[362,941],[362,967],[331,967]]}]},{"text":"System Owner","quad":[[664,944],[834,944],[834,968],[664,968]],"words":[{"text":"System","quad":[[688,944],[749,944],[749,968],[688,968]]},{"text":"Owner","quad":[[773,944],[830,944],[830,968],[773,968]]}]},{"text":"Authorizing Official or Authorizing","quad":[[1052,941],[1418,943],[1418,969],[1052,967]],"words":[{"text":"Authorizing","quad":[[1078,941],[1187,942],[1187,968],[1078,967]]},{"text":"Official","quad":[[1204,942],[1266,942],[1266,968],[1204,968]]},{"text":"or","quad":[[1279,942],[1292,942],[1292,968],[1279,968]]},{"text":"Authorizing","quad":[[1305,942],[1414,943],[1414,969],[1305,968]]}]},{"text":"Official Designated Representative","quad":[[1068,972],[1423,973],[1423,999],[1068,998]],"words":[{"text":"Official","quad":[[1081,972],[1142,972],[1142,998],[1081,998]]},{"text":"Designated","quad":[[1155,972],[1258,973],[1258,999],[1155,998]]},{"text":"Representative","quad":[[1276,973],[1419,973],[1419,999],[1276,999]]}]},{"text":"System Disposal","quad":[[262,989],[434,988],[434,1015],[263,1017]],"words":[{"text":"System","quad":[[267,989],[327,989],[328,1017],[268,1017]]},{"text":"Disposal","quad":[[351,988],[430,988],[431,1016],[352,1016]]}]},{"text":"Information Owner or Steward","quad":[[1051,1005],[1386,1005],[1386,1030],[1051,1030]],"words":[{"text":"Information","quad":[[1076,1005],[1189,1005],[1189,1030],[1076,1030]]},{"text":"Owner","quad":[[1206,1005],[1265,1005],[1265,1030],[1206,1030]]},{"text":"or","quad":[[1281,1005],[1294,1005],[1294,1030],[1281,1030]]},{"text":"Steward","quad":[[1306,1005],[1382,1005],[1382,1030],[1306,1030]]}]},{"text":"Implement a system disposal","quad":[[263,1027],[561,1027],[561,1054],[263,1054]],"words":[{"text":"Implement","quad":[[268,1027],[376,1027],[376,1054],[268,1054]]},{"text":"a","quad":[[385,1027],[389,1027],[389,1054],[385,1054]]},{"text":"system","quad":[[403,1027],[462,1027],[462,1054],[403,1054]]},{"text":"disposal","quad":[[484,1027],[561,1027],[561,1054],[484,1054]]}]},{"text":"System Security Officer","quad":[[1051,1035],[1311,1035],[1311,1062],[1051,1062]],"words":[{"text":"System","quad":[[1078,1035],[1136,1035],[1136,1062],[1078,1062]]},{"text":"Security","quad":[[1154,1035],[1230,1035],[1230,1062],[1154,1062]]},{"text":"Officer","quad":[[1244,1035],[1307,1035],[1307,1062],[1244,1062]]}]},{"text":"strategy and execute required","quad":[[262,1058],[570,1057],[571,1084],[263,1085]],"words":[{"text":"strategy","quad":[[267,1058],[344,1058],[345,1085],[268,1085]]},{"text":"and","quad":[[357,1058],[389,1058],[390,1085],[358,1085]]},{"text":"execute","quad":[[402,1058],[475,1057],[476,1084],[403,1085]]},{"text":"required","quad":[[488,1057],[565,1057],[566,1084],[489,1084]]}]},{"text":"System Privacy Officer","quad":[[1052,1068],[1301,1067],[1301,1093],[1052,1094]],"words":[{"text":"System","quad":[[1078,1068],[1135,1068],[1135,1094],[1078,1094]]},{"text":"Privacy","quad":[[1157,1068],[1222,1067],[1222,1093],[1157,1094]]},{"text":"Officer","quad":[[1235,1067],[1297,1067],[1297,1093],[1235,1093]]}]},{"text":"actions when a system is removed","quad":[[263,1089],[614,1089],[614,1113],[263,1113]],"words":[{"text":"actions","quad":[[271,1089],[335,1089],[335,1113],[271,1113]]},{"text":"when","quad":[[351,1089],[395,1089],[395,1113],[351,1113]]},{"text":"a","quad":[[411,1089],[415,1089],[415,1113],[411,1113]]},{"text":"system","quad":[[427,1089],[482,1089],[482,1113],[427,1113]]},{"text":"is","quad":[[502,1089],[514,1089],[514,1113],[502,1113]]},{"text":"removed","quad":[[522,1089],[610,1089],[610,1113],[522,1113]]}]},{"text":"Senior Accountable Official for Risk","quad":[[1056,1100],[1428,1100],[1428,1124],[1056,1124]],"words":[{"text":"Senior","quad":[[1076,1100],[1136,1100],[1136,1124],[1076,1124]]},{"text":"Accountable","quad":[[1148,1100],[1264,1100],[1264,1124],[1148,1124]]},{"text":"Official","quad":[[1280,1100],[1344,1100],[1344,1124],[1280,1124]]},{"text":"for","quad":[[1356,1100],[1380,1100],[1380,1124],[1356,1124]]},{"text":"Risk","quad":[[1392,1100],[1424,1100],[1424,1124],[1392,1124]]}]},{"text":"from operation.","quad":[[263,1119],[428,1119],[428,1146],[263,1146]],"words":[{"text":"from","quad":[[267,1119],[303,1119],[303,1146],[267,1146]]},{"text":"operation.","quad":[[326,1119],[425,1119],[425,1146],[326,1146]]}]},{"text":"Management or Risk Executive","quad":[[1070,1131],[1385,1132],[1384,1156],[1070,1155]],"words":[{"text":"Management","quad":[[1082,1131],[1206,1131],[1206,1155],[1082,1155]]},{"text":"or","quad":[[1218,1131],[1233,1132],[1233,1156],[1218,1155]]},{"text":"Risk","quad":[[1245,1132],[1277,1132],[1277,1156],[1245,1156]]},{"text":"Executive","quad":[[1293,1132],[1381,1132],[1381,1156],[1293,1156]]}]},{"text":"(Function)","quad":[[1070,1162],[1178,1162],[1178,1185],[1070,1185]],"words":[{"text":"(Function)","quad":[[1074,1162],[1177,1162],[1177,1185],[1074,1185]]}]},{"text":"Senior Agency Information Security","quad":[[1049,1193],[1431,1193],[1431,1218],[1049,1218]],"words":[{"text":"Senior","quad":[[1078,1193],[1136,1193],[1136,1218],[1078,1218]]},{"text":"Agency","quad":[[1149,1193],[1211,1193],[1211,1218],[1149,1218]]},{"text":"Information","quad":[[1223,1193],[1336,1193],[1336,1218],[1223,1218]]},{"text":"Security","quad":[[1352,1193],[1427,1193],[1427,1218],[1352,1218]]}]},{"text":"Officer","quad":[[1069,1223],[1144,1225],[1143,1249],[1068,1247]],"words":[{"text":"Officer","quad":[[1081,1223],[1141,1225],[1140,1249],[1080,1247]]}]},{"text":"Senior Agency Official for Privacy","quad":[[1049,1253],[1409,1256],[1409,1284],[1049,1282]],"words":[{"text":"Senior","quad":[[1078,1253],[1137,1254],[1137,1283],[1078,1282]]},{"text":"Agency","quad":[[1146,1254],[1214,1254],[1214,1283],[1146,1283]]},{"text":"Official","quad":[[1229,1254],[1292,1255],[1292,1284],[1229,1284]]},{"text":"for","quad":[[1302,1255],[1326,1255],[1326,1284],[1302,1284]]},{"text":"Privacy","quad":[[1341,1255],[1404,1256],[1404,1285],[1341,1284]]}]},{"text":"APPENDIX E","quad":[[247,2075],[362,2075],[362,2098],[247,2098]],"words":[{"text":"APPENDIX","quad":[[255,2075],[339,2075],[339,2098],[255,2098]]},{"text":"E","quad":[[350,2075],[354,2075],[354,2098],[350,2098]]}]},{"text":"PAGE 138","quad":[[1361,2075],[1452,2075],[1452,2098],[1361,2098]],"words":[{"text":"PAGE","quad":[[1369,2075],[1411,2075],[1411,2098],[1369,2098]]},{"text":"138","quad":[[1422,2075],[1449,2075],[1449,2098],[1422,2098]]}]}],"refused":null} +{"ocr_response":1,"id":"pdf-flate-gray-png-up","identity":{"recognizer":"ethos-parser-ocr","version":"0.1.0","rule":"ppocrv5-mobile-v1","det_sha256":"4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae","rec_sha256":"c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8","dict_sha256":"e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6","runtime":"rten 0.26.0; rten-vecmath 0.26.0 with fixed-order sums"},"width":1700,"height":2200,"lines":[{"text":"NIST SP 800-37, REVISION 2","quad":[[248,102],[501,101],[502,126],[248,127]],"words":[{"text":"NIST","quad":[[256,102],[285,102],[285,127],[256,127]]},{"text":"SP","quad":[[298,102],[314,102],[314,127],[298,127]]},{"text":"800-37,","quad":[[327,102],[393,101],[393,126],[327,127]]},{"text":"REVISION","quad":[[401,101],[476,101],[476,126],[401,126]]},{"text":"2","quad":[[493,101],[497,101],[497,126],[493,126]]}]},{"text":"RISK MANAGEMENT FRAMEWORK FOR INFORMATION SYSTEMS AND ORGANIZATIONS","quad":[[663,103],[1448,103],[1448,125],[663,125]],"words":[{"text":"RISK","quad":[[667,103],[700,103],[700,125],[667,125]]},{"text":"MANAGEMENT","quad":[[714,103],[843,103],[843,125],[714,125]]},{"text":"FRAMEWORK","quad":[[857,103],[971,103],[971,125],[857,125]]},{"text":"FOR","quad":[[986,103],[1015,103],[1015,125],[986,125]]},{"text":"INFORMATION","quad":[[1026,103],[1151,103],[1151,125],[1026,125]]},{"text":"SYSTEMS","quad":[[1166,103],[1243,103],[1243,125],[1166,125]]},{"text":"AND","quad":[[1254,103],[1287,103],[1287,125],[1254,125]]},{"text":"ORGANIZATIONS","quad":[[1301,103],[1444,103],[1444,125],[1301,125]]}]},{"text":"A System Life Cycle Approach for Security and Privacy","quad":[[1024,127],[1451,127],[1451,152],[1024,152]],"words":[{"text":"A","quad":[[1028,127],[1032,127],[1032,152],[1028,152]]},{"text":"System","quad":[[1045,127],[1091,127],[1091,152],[1045,152]]},{"text":"Life","quad":[[1108,127],[1129,127],[1129,152],[1108,152]]},{"text":"Cycle","quad":[[1141,127],[1179,127],[1179,152],[1141,152]]},{"text":"Approach","quad":[[1187,127],[1258,127],[1258,152],[1187,152]]},{"text":"for","quad":[[1267,127],[1288,127],[1288,152],[1267,152]]},{"text":"Security","quad":[[1296,127],[1355,127],[1355,152],[1296,152]]},{"text":"and","quad":[[1363,127],[1388,127],[1388,152],[1363,152]]},{"text":"Privacy","quad":[[1401,127],[1451,127],[1451,152],[1401,152]]}]},{"text":"The authors also wish to recognize Matt Barrett, Kathleen Coupe, Jeff Eisensmith, Chris Enloe,","quad":[[248,233],[1417,235],[1417,269],[248,267]],"words":[{"text":"The","quad":[[254,233],[293,233],[293,267],[254,267]]},{"text":"authors","quad":[[310,233],[396,233],[396,267],[310,267]]},{"text":"also","quad":[[413,233],[452,233],[452,267],[413,267]]},{"text":"wish","quad":[[469,233],[515,233],[515,267],[469,267]]},{"text":"to","quad":[[532,233],[549,234],[549,268],[532,267]]},{"text":"recognize","quad":[[566,234],[674,234],[674,268],[566,268]]},{"text":"Matt","quad":[[696,234],[747,234],[747,268],[696,268]]},{"text":"Barrett,","quad":[[759,234],[850,234],[850,268],[759,268]]},{"text":"Kathleen","quad":[[861,234],[963,234],[963,268],[861,268]]},{"text":"Coupe,","quad":[[980,234],[1065,234],[1065,268],[980,268]]},{"text":"Jeff","quad":[[1071,234],[1116,234],[1116,268],[1071,268]]},{"text":"Eisensmith,","quad":[[1122,234],[1264,235],[1264,269],[1122,268]]},{"text":"Chris","quad":[[1275,235],[1326,235],[1326,269],[1275,269]]},{"text":"Enloe,","quad":[[1343,235],[1417,235],[1417,269],[1343,269]]}]},{"text":"Ned Goren, Matthew Halstead, Jody Jacobs, Ralph Jones, Martin Kihiko, Raquel Leone, and the","quad":[[248,273],[1425,274],[1425,305],[248,303]],"words":[{"text":"Ned","quad":[[258,273],[294,273],[294,303],[258,303]]},{"text":"Goren,","quad":[[315,273],[393,273],[393,303],[315,303]]},{"text":"Matthew","quad":[[408,273],[506,273],[506,303],[408,303]]},{"text":"Halstead,","quad":[[527,273],[635,273],[635,303],[527,303]]},{"text":"Jody","quad":[[645,273],[697,273],[697,303],[645,303]]},{"text":"Jacobs,","quad":[[707,273],[795,273],[795,303],[707,303]]},{"text":"Ralph","quad":[[811,273],[867,274],[867,304],[811,303]]},{"text":"Jones,","quad":[[883,274],[955,274],[955,304],[883,304]]},{"text":"Martin","quad":[[971,274],[1043,274],[1043,304],[971,304]]},{"text":"Kihiko,","quad":[[1058,274],[1136,274],[1136,304],[1058,304]]},{"text":"Raquel","quad":[[1151,274],[1229,274],[1229,304],[1151,304]]},{"text":"Leone,","quad":[[1244,274],[1322,274],[1322,304],[1244,304]]},{"text":"and","quad":[[1332,274],[1368,274],[1368,304],[1332,304]]},{"text":"the","quad":[[1384,274],[1415,274],[1415,304],[1384,304]]}]},{"text":"scientists, engineers, and research staff from the Computer Security Division and the Applied","quad":[[248,309],[1404,309],[1404,341],[248,341]],"words":[{"text":"scientists,","quad":[[253,309],[371,309],[371,341],[253,341]]},{"text":"engineers,","quad":[[387,309],[509,309],[509,341],[387,341]]},{"text":"and","quad":[[520,309],[557,309],[557,341],[520,341]]},{"text":"research","quad":[[573,309],[669,309],[669,341],[573,341]]},{"text":"staff","quad":[[685,309],[738,309],[738,341],[685,341]]},{"text":"from","quad":[[749,309],[791,309],[791,341],[749,341]]},{"text":"the","quad":[[813,309],[850,309],[850,341],[813,341]]},{"text":"Computer","quad":[[866,309],[983,309],[983,341],[866,341]]},{"text":"Security","quad":[[999,309],[1090,309],[1090,341],[999,341]]},{"text":"Division","quad":[[1106,309],[1191,309],[1191,341],[1106,341]]},{"text":"and","quad":[[1207,309],[1250,309],[1250,341],[1207,341]]},{"text":"the","quad":[[1260,309],[1292,309],[1292,341],[1260,341]]},{"text":"Applied","quad":[[1313,309],[1399,309],[1399,341],[1313,341]]}]},{"text":"Cybersecurity Division for their exceptional contributions in helping to improve the content of","quad":[[248,346],[1418,346],[1418,381],[248,381]],"words":[{"text":"Cybersecurity","quad":[[260,346],[418,346],[418,381],[260,381]]},{"text":"Division","quad":[[435,346],[517,346],[517,381],[435,381]]},{"text":"for","quad":[[535,346],[564,346],[564,381],[535,381]]},{"text":"their","quad":[[581,346],[634,346],[634,381],[581,381]]},{"text":"exceptional","quad":[[646,346],[786,346],[786,381],[646,381]]},{"text":"contributions","quad":[[798,346],[956,346],[956,381],[798,381]]},{"text":"in","quad":[[968,346],[985,346],[985,381],[968,381]]},{"text":"helping","quad":[[1003,346],[1085,346],[1085,381],[1003,381]]},{"text":"to","quad":[[1096,346],[1114,346],[1114,381],[1096,381]]},{"text":"improve","quad":[[1131,346],[1225,346],[1225,381],[1131,381]]},{"text":"the","quad":[[1242,346],[1272,346],[1272,381],[1242,381]]},{"text":"content","quad":[[1295,346],[1383,346],[1383,381],[1295,381]]},{"text":"of","quad":[[1395,346],[1412,346],[1412,381],[1395,381]]}]},{"text":"the publication. A special note of thanks to Jim Foti and the NiST web team for their outstanding","quad":[[247,384],[1447,385],[1447,417],[247,416]],"words":[{"text":"the","quad":[[252,384],[284,384],[284,416],[252,416]]},{"text":"publication.","quad":[[306,384],[444,384],[444,416],[306,416]]},{"text":"A","quad":[[460,384],[466,384],[466,416],[460,416]]},{"text":"special","quad":[[482,384],[562,384],[562,416],[482,416]]},{"text":"note","quad":[[572,384],[620,384],[620,416],[572,416]]},{"text":"of","quad":[[636,384],[658,384],[658,416],[636,416]]},{"text":"thanks","quad":[[668,384],[748,384],[748,416],[668,416]]},{"text":"to","quad":[[759,384],[775,384],[775,416],[759,416]]},{"text":"Jim","quad":[[791,384],[818,384],[818,416],[791,416]]},{"text":"Foti","quad":[[844,384],[882,385],[882,417],[844,416]]},{"text":"and","quad":[[898,385],[935,385],[935,417],[898,417]]},{"text":"the","quad":[[946,385],[983,385],[983,417],[946,417]]},{"text":"NiST","quad":[[999,385],[1042,385],[1042,417],[999,417]]},{"text":"web","quad":[[1063,385],[1100,385],[1100,417],[1063,417]]},{"text":"team","quad":[[1122,385],[1170,385],[1170,417],[1122,417]]},{"text":"for","quad":[[1191,385],[1223,385],[1223,417],[1191,417]]},{"text":"their","quad":[[1234,385],[1287,385],[1287,417],[1234,417]]},{"text":"outstanding","quad":[[1303,385],[1442,385],[1442,417],[1303,417]]}]},{"text":"administrative support.","quad":[[250,422],[541,423],[541,454],[249,453]],"words":[{"text":"administrative","quad":[[255,422],[427,423],[426,454],[254,453]]},{"text":"support.","quad":[[442,423],[541,423],[540,454],[441,454]]}]},{"text":"In addition, the authors wish to acknowledge the United States Air Force and the \"RMF Next\"","quad":[[248,492],[1405,491],[1405,525],[248,526]],"words":[{"text":"In","quad":[[254,492],[271,492],[271,526],[254,526]]},{"text":"addition,","quad":[[288,492],[390,492],[390,526],[288,526]]},{"text":"the","quad":[[401,492],[435,492],[435,526],[401,526]]},{"text":"authors","quad":[[452,492],[543,492],[543,526],[452,526]]},{"text":"wish","quad":[[560,492],[600,492],[600,526],[560,526]]},{"text":"to","quad":[[617,492],[634,492],[634,526],[617,526]]},{"text":"acknowledge","quad":[[651,492],[804,492],[804,526],[651,526]]},{"text":"the","quad":[[821,492],[855,491],[855,525],[821,526]]},{"text":"United","quad":[[872,491],[946,491],[946,525],[872,525]]},{"text":"States","quad":[[963,491],[1031,491],[1031,525],[963,525]]},{"text":"Air","quad":[[1048,491],[1076,491],[1076,525],[1048,525]]},{"text":"Force","quad":[[1087,491],[1144,491],[1144,525],[1087,525]]},{"text":"and","quad":[[1161,491],[1201,491],[1201,525],[1161,525]]},{"text":"the","quad":[[1212,491],[1246,491],[1246,525],[1212,525]]},{"text":"\"RMF","quad":[[1263,491],[1326,491],[1326,525],[1263,525]]},{"text":"Next\"","quad":[[1343,491],[1405,491],[1405,525],[1343,525]]}]},{"text":"initiative, facilitated by Air Force CyberWorx, that provided the inspiration for some of the new","quad":[[249,530],[1429,530],[1429,560],[249,560]],"words":[{"text":"initiative,","quad":[[249,530],[364,530],[364,560],[249,560]]},{"text":"facilitated","quad":[[379,530],[494,530],[494,560],[379,560]]},{"text":"by","quad":[[509,530],[529,530],[529,560],[509,560]]},{"text":"Air","quad":[[549,530],[574,530],[574,560],[549,560]]},{"text":"Force","quad":[[589,530],[649,530],[649,560],[589,560]]},{"text":"CyberWorx,","quad":[[669,530],[804,530],[804,560],[669,560]]},{"text":"that","quad":[[819,530],[859,530],[859,560],[819,560]]},{"text":"provided","quad":[[879,530],[979,530],[979,560],[879,560]]},{"text":"the","quad":[[994,530],[1024,530],[1024,560],[994,560]]},{"text":"inspiration","quad":[[1039,530],[1164,530],[1164,560],[1039,560]]},{"text":"for","quad":[[1179,530],[1209,530],[1209,560],[1179,560]]},{"text":"some","quad":[[1224,530],[1284,530],[1284,560],[1224,560]]},{"text":"of","quad":[[1299,530],[1319,530],[1319,560],[1299,560]]},{"text":"the","quad":[[1329,530],[1364,530],[1364,560],[1329,560]]},{"text":"new","quad":[[1379,530],[1419,530],[1419,560],[1379,560]]}]},{"text":"ideas in this update to the RMF. The working group, led by Lauren Knausenberger, Bill Bryant,","quad":[[248,564],[1412,568],[1412,601],[248,598]],"words":[{"text":"ideas","quad":[[248,564],[310,564],[310,598],[248,598]]},{"text":"in","quad":[[322,564],[339,564],[339,598],[322,598]]},{"text":"this","quad":[[356,564],[396,565],[396,599],[356,598]]},{"text":"update","quad":[[413,565],[486,565],[486,599],[413,599]]},{"text":"to","quad":[[504,565],[521,565],[521,599],[504,599]]},{"text":"the","quad":[[538,565],[572,565],[572,599],[538,599]]},{"text":"RMF.","quad":[[589,565],[645,565],[645,599],[589,599]]},{"text":"The","quad":[[657,565],[697,566],[697,600],[657,599]]},{"text":"working","quad":[[714,566],[804,566],[804,600],[714,600]]},{"text":"group,","quad":[[816,566],[895,566],[895,600],[816,600]]},{"text":"led","quad":[[901,566],[935,566],[935,600],[901,600]]},{"text":"by","quad":[[952,566],[975,566],[975,600],[952,600]]},{"text":"Lauren","quad":[[986,567],[1060,567],[1060,601],[986,601]]},{"text":"Knausenberger,","quad":[[1083,567],[1270,568],[1270,602],[1083,601]]},{"text":"Bill","quad":[[1287,568],[1321,568],[1321,602],[1287,602]]},{"text":"Bryant,","quad":[[1327,568],[1412,568],[1412,602],[1327,602]]}]},{"text":"and Venice Goodwine, included government and industry representatives Jake Ames, Chris","quad":[[248,605],[1381,604],[1381,637],[248,638]],"words":[{"text":"and","quad":[[254,605],[292,605],[292,638],[254,638]]},{"text":"Venice","quad":[[314,605],[386,605],[386,638],[314,638]]},{"text":"Goodwine,","quad":[[402,605],[528,605],[528,638],[402,638]]},{"text":"included","quad":[[540,605],[638,605],[638,638],[540,638]]},{"text":"government","quad":[[655,605],[798,605],[798,638],[655,638]]},{"text":"and","quad":[[814,604],[853,604],[853,637],[814,638]]},{"text":"industry","quad":[[864,604],[963,604],[963,637],[864,637]]},{"text":"representatives","quad":[[974,604],[1161,604],[1161,637],[974,637]]},{"text":"Jake","quad":[[1178,604],[1222,604],[1222,637],[1178,637]]},{"text":"Ames,","quad":[[1238,604],[1310,604],[1310,637],[1238,637]]},{"text":"Chris","quad":[[1326,604],[1376,604],[1376,637],[1326,637]]}]},{"text":"This publication is available free of charge from: https://doi.org/10.6028/NIST.SP.800-37r2","quad":[[73,618],[73,1622],[43,1622],[43,618]],"words":[{"text":"This","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"publication","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"is","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"available","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"free","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"of","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"charge","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"from:","quad":[[73,618],[73,1622],[43,1622],[43,618]]},{"text":"https://doi.org/10.6028/NIST.SP.800-37r2","quad":[[73,618],[73,1622],[43,1622],[43,618]]}]},{"text":"Bailey, James Barnett, Steve Bogue, Wes Chiu, Kurt Danis, Shane Deichman; Joe Erskine, Terence","quad":[[248,642],[1449,643],[1449,675],[248,674]],"words":[{"text":"Bailey,","quad":[[253,642],[333,642],[333,674],[253,674]]},{"text":"James","quad":[[344,642],[413,642],[413,674],[344,674]]},{"text":"Barnett,","quad":[[429,642],[526,642],[526,674],[429,674]]},{"text":"Steve","quad":[[536,642],[595,642],[595,674],[536,674]]},{"text":"Bogue,","quad":[[611,642],[691,642],[691,674],[611,674]]},{"text":"Wes","quad":[[707,642],[750,642],[750,674],[707,674]]},{"text":"Chiu,","quad":[[766,642],[824,642],[824,674],[766,674]]},{"text":"Kurt","quad":[[835,642],[878,643],[878,675],[835,674]]},{"text":"Danis,","quad":[[894,643],[969,643],[969,675],[894,675]]},{"text":"Shane","quad":[[979,643],[1043,643],[1043,675],[979,675]]},{"text":"Deichman;","quad":[[1065,643],[1187,643],[1187,675],[1065,675]]},{"text":"Joe","quad":[[1198,643],[1230,643],[1230,675],[1198,675]]},{"text":"Erskine,","quad":[[1252,643],[1342,643],[1342,675],[1252,675]]},{"text":"Terence","quad":[[1353,643],[1444,643],[1444,675],[1353,675]]}]},{"text":"Goodman, Jason Howe, Brandon Howell, Todd Jacobs, Peter Klabe, William Kramer, Bryon","quad":[[249,678],[1368,679],[1368,711],[249,710]],"words":[{"text":"Goodman,","quad":[[260,678],[382,678],[382,710],[260,710]]},{"text":"Jason","quad":[[393,678],[451,678],[451,710],[393,710]]},{"text":"Howe,","quad":[[467,678],[542,678],[542,710],[467,710]]},{"text":"Brandon","quad":[[558,678],[649,678],[649,710],[558,710]]},{"text":"Howell,","quad":[[670,678],[755,678],[755,710],[670,710]]},{"text":"Todd","quad":[[766,678],[824,679],[824,711],[766,710]]},{"text":"Jacobs,","quad":[[835,679],[920,679],[920,711],[835,711]]},{"text":"Peter","quad":[[936,679],[995,679],[995,711],[936,711]]},{"text":"Klabe,","quad":[[1006,679],[1080,679],[1080,711],[1006,711]]},{"text":"William","quad":[[1096,679],[1166,679],[1166,711],[1096,711]]},{"text":"Kramer,","quad":[[1192,679],[1288,679],[1288,711],[1192,711]]},{"text":"Bryon","quad":[[1299,679],[1357,679],[1357,711],[1299,711]]}]},{"text":"Kroger, Kevin LaSalle, Dinh Le, Noam Liran, Sam Miles, Michael Morrison, Raymond Tom Nagley,","quad":[[247,714],[1446,716],[1446,751],[247,748]],"words":[{"text":"Kroger,","quad":[[253,714],[341,714],[341,748],[253,748]]},{"text":"Kevin","quad":[[352,714],[411,714],[411,748],[352,748]]},{"text":"LaSalle,","quad":[[428,714],[516,714],[516,748],[428,748]]},{"text":"Dinh","quad":[[528,714],[575,715],[575,749],[528,748]]},{"text":"Le,","quad":[[592,715],[621,715],[621,749],[592,749]]},{"text":"Noam","quad":[[639,715],[692,715],[692,749],[639,749]]},{"text":"Liran,","quad":[[715,715],[785,715],[785,749],[715,749]]},{"text":"Sam","quad":[[791,715],[832,715],[832,749],[791,749]]},{"text":"Miles,","quad":[[855,715],[925,715],[925,749],[855,749]]},{"text":"Michael","quad":[[937,715],[1031,715],[1031,749],[937,749]]},{"text":"Morrison,","quad":[[1042,715],[1159,716],[1159,750],[1042,749]]},{"text":"Raymond","quad":[[1171,716],[1282,716],[1282,750],[1171,750]]},{"text":"Tom","quad":[[1294,716],[1335,716],[1335,750],[1294,750]]},{"text":"Nagley,","quad":[[1358,716],[1446,716],[1446,750],[1358,750]]}]},{"text":"Wendy Nather, Jasmine Neal, Ryan Perry, Eugene Peterson, Lawrence Rampaul, Jessica","quad":[[249,754],[1331,755],[1331,786],[249,785]],"words":[{"text":"Wendy","quad":[[259,754],[337,754],[337,785],[259,785]]},{"text":"Nather,","quad":[[353,754],[441,754],[441,785],[353,785]]},{"text":"Jasmine","quad":[[451,754],[539,754],[539,785],[451,785]]},{"text":"Neal,","quad":[[560,754],[617,754],[617,785],[560,785]]},{"text":"Ryan","quad":[[632,754],[679,754],[679,785],[632,785]]},{"text":"Perry,","quad":[[699,754],[767,754],[767,785],[699,785]]},{"text":"Eugene","quad":[[777,754],[860,755],[860,786],[777,785]]},{"text":"Peterson,","quad":[[881,755],[989,755],[989,786],[881,786]]},{"text":"Lawrence","quad":[[1005,755],[1114,755],[1114,786],[1005,786]]},{"text":"Rampaul,","quad":[[1129,755],[1243,755],[1243,786],[1129,786]]},{"text":"Jessica","quad":[[1253,755],[1326,755],[1326,786],[1253,786]]}]},{"text":"Rheinschmidt, Greg Roman, Susanna Scarveles, Justin Schoenthal, Christian Sorenson, Stacy","quad":[[249,788],[1387,792],[1387,825],[248,822]],"words":[{"text":"Rheinschmidt,","quad":[[255,788],[430,789],[429,823],[254,822]]},{"text":"Greg","quad":[[441,789],[492,789],[491,823],[440,823]]},{"text":"Roman,","quad":[[509,789],[594,789],[593,823],[508,823]]},{"text":"Susanna","quad":[[611,789],[702,790],[701,824],[610,823]]},{"text":"Scarveles,","quad":[[719,790],[838,790],[837,824],[718,824]]},{"text":"Justin","quad":[[849,790],[911,790],[910,824],[848,824]]},{"text":"Schoenthal,","quad":[[928,790],[1070,791],[1069,825],[927,824]]},{"text":"Christian","quad":[[1081,791],[1178,791],[1177,825],[1080,825]]},{"text":"Sorenson,","quad":[[1195,791],[1313,792],[1312,826],[1194,825]]},{"text":"Stacy","quad":[[1325,792],[1381,792],[1380,826],[1324,826]]}]},{"text":"Studstill, Charles Wade, Shawn Whitney, David Wilcox, and Thomas Woodring.","quad":[[248,828],[1226,829],[1226,860],[248,859]],"words":[{"text":"Studstill,","quad":[[253,828],[357,828],[357,859],[253,859]]},{"text":"Charles","quad":[[372,828],[455,828],[455,859],[372,859]]},{"text":"Wade,","quad":[[476,828],[543,828],[543,859],[476,859]]},{"text":"Shawn","quad":[[558,828],[626,828],[626,859],[558,859]]},{"text":"Whitney,","quad":[[652,828],[755,829],[755,860],[652,859]]},{"text":"David","quad":[[765,829],[828,829],[828,860],[765,860]]},{"text":"Wilcox,","quad":[[848,829],[931,829],[931,860],[848,860]]},{"text":"and","quad":[[941,829],[978,829],[978,860],[941,860]]},{"text":"Thomas","quad":[[998,829],[1086,829],[1086,860],[998,860]]},{"text":"Woodring.","quad":[[1107,829],[1226,829],[1226,860],[1107,860]]}]},{"text":"Finally, the authors also gratefully acknowledge the significant contributions from individuals","quad":[[248,899],[1405,898],[1405,932],[248,933]],"words":[{"text":"Finally,","quad":[[254,899],[339,899],[339,933],[254,933]]},{"text":"the","quad":[[350,899],[378,899],[378,933],[350,933]]},{"text":"authors","quad":[[395,899],[486,899],[486,933],[395,933]]},{"text":"also","quad":[[498,899],[537,899],[537,933],[498,933]]},{"text":"gratefully","quad":[[554,899],[668,899],[668,933],[554,933]]},{"text":"acknowledge","quad":[[685,899],[838,898],[838,932],[685,933]]},{"text":"the","quad":[[849,898],[883,898],[883,932],[849,932]]},{"text":"significant","quad":[[900,898],[1019,898],[1019,932],[900,932]]},{"text":"contributions","quad":[[1036,898],[1189,898],[1189,932],[1036,932]]},{"text":"from","quad":[[1206,898],[1252,898],[1252,932],[1206,932]]},{"text":"individuals","quad":[[1275,898],[1399,898],[1399,932],[1275,932]]}]},{"text":"and organizations in both the public and private sectors, nationally and internationally, whose","quad":[[248,935],[1416,935],[1416,970],[248,970]],"words":[{"text":"and","quad":[[254,935],[295,935],[295,970],[254,970]]},{"text":"organizations","quad":[[312,935],[470,935],[470,970],[312,970]]},{"text":"in","quad":[[482,935],[493,935],[493,970],[482,970]]},{"text":"both","quad":[[511,935],[558,935],[558,970],[511,970]]},{"text":"the","quad":[[575,935],[610,935],[610,970],[575,970]]},{"text":"public","quad":[[628,935],[692,935],[692,970],[628,970]]},{"text":"and","quad":[[709,935],[750,935],[750,970],[709,970]]},{"text":"private","quad":[[762,935],[838,935],[838,970],[762,970]]},{"text":"sectors,","quad":[[855,935],[949,935],[949,970],[855,970]]},{"text":"nationally","quad":[[960,935],[1077,935],[1077,970],[960,970]]},{"text":"and","quad":[[1089,935],[1130,935],[1130,970],[1089,970]]},{"text":"internationally,","quad":[[1142,935],[1328,935],[1328,970],[1142,970]]},{"text":"whose","quad":[[1340,935],[1410,935],[1410,970],[1340,970]]}]},{"text":"thoughtful and constructive comments improved the overall quality, thoroughness, and","quad":[[247,972],[1339,974],[1339,1007],[247,1006]],"words":[{"text":"thoughtful","quad":[[253,972],[383,972],[383,1006],[253,1006]]},{"text":"and","quad":[[394,972],[434,972],[434,1006],[394,1006]]},{"text":"constructive","quad":[[445,972],[592,973],[592,1007],[445,1006]]},{"text":"comments","quad":[[609,973],[728,973],[728,1007],[609,1007]]},{"text":"improved","quad":[[739,973],[858,973],[858,1007],[739,1007]]},{"text":"the","quad":[[869,973],[903,973],[903,1007],[869,1007]]},{"text":"overall","quad":[[920,973],[1000,973],[1000,1007],[920,1007]]},{"text":"quality,","quad":[[1011,973],[1096,974],[1096,1008],[1011,1007]]},{"text":"thoroughness,","quad":[[1107,974],[1282,974],[1282,1008],[1107,1008]]},{"text":"and","quad":[[1294,974],[1333,974],[1333,1008],[1294,1008]]}]},{"text":"usefulness of this publication.","quad":[[249,1011],[620,1012],[620,1044],[249,1042]],"words":[{"text":"usefulness","quad":[[254,1011],[378,1011],[378,1042],[254,1042]]},{"text":"of","quad":[[394,1011],[410,1011],[410,1042],[394,1042]]},{"text":"this","quad":[[426,1011],[464,1012],[464,1043],[426,1042]]},{"text":"publication.","quad":[[480,1012],[620,1012],[620,1043],[480,1043]]}]},{"text":"HISTORICAL CONTRIBUTIONS TO NIST SPECIAL PUBLICATION 800-37","quad":[[416,1161],[1277,1161],[1277,1191],[416,1191]],"words":[{"text":"HISTORICAL","quad":[[421,1161],[566,1161],[566,1191],[421,1191]]},{"text":"CONTRIBUTIONS","quad":[[581,1161],[781,1161],[781,1191],[581,1191]]},{"text":"TO","quad":[[801,1161],[821,1161],[821,1191],[801,1191]]},{"text":"NIST","quad":[[846,1161],[892,1161],[892,1191],[846,1191]]},{"text":"SPECIAL","quad":[[907,1161],[1002,1161],[1002,1191],[907,1191]]},{"text":"PUBLICATION","quad":[[1017,1161],[1177,1161],[1177,1191],[1017,1191]]},{"text":"800-37","quad":[[1197,1161],[1272,1161],[1272,1191],[1197,1191]]}]},{"text":"The authors acknowledge the many individuals who contributed to previous versions of Special","quad":[[308,1212],[1386,1212],[1386,1247],[308,1247]],"words":[{"text":"The","quad":[[314,1212],[349,1212],[349,1247],[314,1247]]},{"text":"authors","quad":[[360,1212],[442,1212],[442,1247],[360,1247]]},{"text":"acknowledge","quad":[[454,1212],[594,1212],[594,1247],[454,1247]]},{"text":"the","quad":[[611,1212],[640,1212],[640,1247],[611,1247]]},{"text":"many","quad":[[658,1212],[710,1212],[710,1247],[658,1247]]},{"text":"individuals","quad":[[722,1212],[838,1212],[838,1247],[722,1247]]},{"text":"who","quad":[[850,1212],[891,1212],[891,1247],[850,1247]]},{"text":"contributed","quad":[[908,1212],[1031,1212],[1031,1247],[908,1247]]},{"text":"to","quad":[[1042,1212],[1060,1212],[1060,1247],[1042,1247]]},{"text":"previous","quad":[[1077,1212],[1165,1212],[1165,1247],[1077,1247]]},{"text":"versions","quad":[[1176,1212],[1264,1212],[1264,1247],[1176,1247]]},{"text":"of","quad":[[1275,1212],[1293,1212],[1293,1247],[1275,1247]]},{"text":"Special","quad":[[1304,1212],[1380,1212],[1380,1247],[1304,1247]]}]},{"text":"Publication 800-37 since its inception in 2005. They include Marshall Abrams, William Barker,","quad":[[310,1246],[1385,1248],[1385,1279],[310,1277]],"words":[{"text":"Publication","quad":[[315,1246],[434,1246],[434,1277],[315,1277]]},{"text":"800-37","quad":[[450,1246],[522,1246],[522,1277],[450,1277]]},{"text":"since","quad":[[537,1246],[584,1247],[584,1278],[537,1277]]},{"text":"its","quad":[[599,1247],[620,1247],[620,1278],[599,1278]]},{"text":"inception","quad":[[636,1247],[734,1247],[734,1278],[636,1278]]},{"text":"in","quad":[[749,1247],[765,1247],[765,1278],[749,1278]]},{"text":"2005.","quad":[[780,1247],[842,1247],[842,1278],[780,1278]]},{"text":"They","quad":[[853,1247],[899,1247],[899,1278],[853,1278]]},{"text":"include","quad":[[910,1247],[987,1247],[987,1278],[910,1278]]},{"text":"Marshall","quad":[[1008,1247],[1101,1247],[1101,1278],[1008,1278]]},{"text":"Abrams,","quad":[[1111,1247],[1199,1248],[1199,1279],[1111,1278]]},{"text":"William","quad":[[1214,1248],[1282,1248],[1282,1279],[1214,1279]]},{"text":"Barker,","quad":[[1307,1248],[1385,1248],[1385,1279],[1307,1279]]}]},{"text":"Beckie Koonge, Roger Caslow, John Gilligan, Peter Gouldmann, Richard Graubart, John Grimes,","quad":[[309,1281],[1385,1282],[1385,1314],[309,1312]],"words":[{"text":"Beckie","quad":[[314,1281],[378,1281],[378,1312],[314,1312]]},{"text":"Koonge,","quad":[[394,1281],[485,1281],[485,1312],[394,1312]]},{"text":"Roger","quad":[[495,1281],[554,1281],[554,1312],[495,1312]]},{"text":"Caslow,","quad":[[570,1281],[650,1281],[650,1312],[570,1312]]},{"text":"John","quad":[[661,1281],[703,1281],[703,1312],[661,1312]]},{"text":"Gilligan,","quad":[[724,1281],[810,1281],[810,1312],[724,1312]]},{"text":"Peter","quad":[[820,1281],[874,1282],[874,1313],[820,1312]]},{"text":"Gouldmann,","quad":[[890,1282],[1023,1282],[1023,1313],[890,1313]]},{"text":"Richard","quad":[[1033,1282],[1108,1282],[1108,1313],[1033,1313]]},{"text":"Graubart,","quad":[[1129,1282],[1231,1282],[1231,1313],[1129,1313]]},{"text":"John","quad":[[1241,1282],[1284,1282],[1284,1313],[1241,1313]]},{"text":"Grimes,","quad":[[1305,1282],[1385,1282],[1385,1313],[1305,1313]]}]},{"text":"Gus Guissanie, Priscilla Guthrie, Jennifer Fabius, Cita Furlani, Richard Hale, Peggy Himes, William","quad":[[310,1316],[1384,1316],[1384,1346],[310,1346]],"words":[{"text":"Gus","quad":[[320,1316],[350,1316],[350,1346],[320,1346]]},{"text":"Guissanie,","quad":[[365,1316],[475,1316],[475,1346],[365,1346]]},{"text":"Priscilla","quad":[[485,1316],[560,1316],[560,1346],[485,1346]]},{"text":"Guthrie,","quad":[[580,1316],[665,1316],[665,1346],[580,1346]]},{"text":"Jennifer","quad":[[670,1316],[755,1316],[755,1346],[670,1346]]},{"text":"Fabius,","quad":[[770,1316],[845,1316],[845,1346],[770,1346]]},{"text":"Cita","quad":[[854,1316],[889,1316],[889,1346],[854,1346]]},{"text":"Furlani,","quad":[[904,1316],[984,1316],[984,1346],[904,1346]]},{"text":"Richard","quad":[[994,1316],[1069,1316],[1069,1346],[994,1346]]},{"text":"Hale,","quad":[[1084,1316],[1134,1316],[1134,1346],[1084,1346]]},{"text":"Peggy","quad":[[1144,1316],[1204,1316],[1204,1346],[1144,1346]]},{"text":"Himes,","quad":[[1219,1316],[1289,1316],[1289,1346],[1219,1346]]},{"text":"William","quad":[[1304,1316],[1369,1316],[1369,1346],[1304,1346]]}]},{"text":"Hunteman, Arnold Johnson, Donald Jones, Stuart Katzke, Eustace King, Mark Morrison, Sherrill","quad":[[310,1350],[1384,1349],[1384,1379],[310,1380]],"words":[{"text":"Hunteman,","quad":[[315,1350],[440,1350],[440,1380],[315,1380]]},{"text":"Arnold","quad":[[450,1350],[515,1350],[515,1380],[450,1380]]},{"text":"Johnson,","quad":[[530,1350],[625,1350],[625,1380],[530,1380]]},{"text":"Donald","quad":[[640,1350],[710,1350],[710,1380],[640,1380]]},{"text":"Jones,","quad":[[725,1350],[790,1350],[790,1380],[725,1380]]},{"text":"Stuart","quad":[[805,1350],[864,1349],[864,1379],[805,1380]]},{"text":"Katzke,","quad":[[879,1349],[954,1349],[954,1379],[879,1379]]},{"text":"Eustace","quad":[[969,1349],[1044,1349],[1044,1379],[969,1379]]},{"text":"King,","quad":[[1059,1349],[1109,1349],[1109,1379],[1059,1379]]},{"text":"Mark","quad":[[1124,1349],[1174,1349],[1174,1379],[1124,1379]]},{"text":"Morrison,","quad":[[1194,1349],[1294,1349],[1294,1379],[1194,1379]]},{"text":"Sherrill","quad":[[1309,1349],[1384,1349],[1384,1379],[1309,1379]]}]},{"text":"Nicely, Karen Quigg, George Rogers, Cheryl Roby, Gary Stoneburner, Marianne Swanson, Glenda","quad":[[310,1384],[1384,1384],[1384,1414],[310,1414]],"words":[{"text":"Nicely,","quad":[[315,1384],[385,1384],[385,1414],[315,1414]]},{"text":"Karen","quad":[[395,1384],[450,1384],[450,1414],[395,1414]]},{"text":"Quigg,","quad":[[470,1384],[535,1384],[535,1414],[470,1414]]},{"text":"George","quad":[[550,1384],[620,1384],[620,1414],[550,1414]]},{"text":"Rogers,","quad":[[635,1384],[710,1384],[710,1414],[635,1414]]},{"text":"Cheryl","quad":[[725,1384],[790,1384],[790,1414],[725,1414]]},{"text":"Roby,","quad":[[800,1384],[859,1384],[859,1414],[800,1414]]},{"text":"Gary","quad":[[869,1384],[914,1384],[914,1414],[869,1414]]},{"text":"Stoneburner,","quad":[[924,1384],[1069,1384],[1069,1414],[924,1414]]},{"text":"Marianne","quad":[[1084,1384],[1179,1384],[1179,1414],[1084,1414]]},{"text":"Swanson,","quad":[[1194,1384],[1294,1384],[1294,1414],[1194,1414]]},{"text":"Glenda","quad":[[1309,1384],[1379,1384],[1379,1414],[1309,1414]]}]},{"text":"Turner, and Peter Williams.","quad":[[311,1419],[619,1419],[619,1446],[311,1446]],"words":[{"text":"Turner,","quad":[[316,1419],[393,1419],[393,1446],[316,1446]]},{"text":"and","quad":[[406,1419],[438,1419],[438,1446],[406,1446]]},{"text":"Peter","quad":[[456,1419],[506,1419],[506,1446],[456,1446]]},{"text":"Williams.","quad":[[524,1419],[619,1419],[619,1446],[524,1446]]}]},{"text":"PAGE iv","quad":[[812,2075],[888,2075],[888,2098],[812,2098]],"words":[{"text":"PAGE","quad":[[820,2075],[862,2075],[862,2098],[820,2098]]},{"text":"iv","quad":[[873,2075],[885,2075],[885,2098],[873,2098]]}]}],"refused":null} +{"ocr_response":1,"id":"pdf-flate-1bit","identity":{"recognizer":"ethos-parser-ocr","version":"0.1.0","rule":"ppocrv5-mobile-v1","det_sha256":"4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae","rec_sha256":"c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8","dict_sha256":"e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6","runtime":"rten 0.26.0; rten-vecmath 0.26.0 with fixed-order sums"},"width":1700,"height":2200,"lines":[{"text":"Schedule D (Form 1040) 2025","quad":[[99,101],[359,101],[359,126],[99,126]],"words":[{"text":"Schedule","quad":[[103,101],[179,101],[179,126],[103,126]]},{"text":"D","quad":[[191,101],[195,101],[195,126],[191,126]]},{"text":"(Form","quad":[[208,101],[250,101],[250,126],[208,126]]},{"text":"1040)","quad":[[267,101],[313,101],[313,126],[267,126]]},{"text":"2025","quad":[[321,101],[359,101],[359,126],[321,126]]}]},{"text":"Page 2","quad":[[1527,98],[1607,94],[1609,131],[1529,135]],"words":[{"text":"Page","quad":[[1539,97],[1576,96],[1578,133],[1541,134]]},{"text":"2","quad":[[1589,95],[1595,95],[1597,132],[1591,132]]}]},{"text":"Part III","quad":[[99,147],[202,147],[202,183],[99,183]],"words":[{"text":"Part","quad":[[111,147],[159,147],[159,183],[111,183]]},{"text":"III","quad":[[165,147],[201,147],[201,183],[165,183]]}]},{"text":"Summary","quad":[[237,149],[370,154],[369,188],[236,183]],"words":[{"text":"Summary","quad":[[248,149],[362,154],[361,188],[247,183]]}]},{"text":"16","quad":[[110,238],[147,238],[147,263],[110,263]],"words":[{"text":"16","quad":[[118,238],[135,238],[135,263],[118,263]]}]},{"text":"Combine lines 7 and 15 and enter the result","quad":[[176,237],[670,238],[670,264],[176,263]],"words":[{"text":"Combine","quad":[[189,237],[276,237],[276,263],[189,263]]},{"text":"lines","quad":[[289,237],[336,237],[336,263],[289,263]]},{"text":"7","quad":[[349,237],[354,237],[354,263],[349,263]]},{"text":"and","quad":[[371,237],[406,237],[406,263],[371,263]]},{"text":"15","quad":[[423,238],[440,238],[440,264],[423,264]]},{"text":"and","quad":[[458,238],[488,238],[488,264],[458,264]]},{"text":"enter","quad":[[505,238],[553,238],[553,264],[505,264]]},{"text":"the","quad":[[566,238],[596,238],[596,264],[566,264]]},{"text":"result","quad":[[609,238],[666,238],[666,264],[609,264]]}]},{"text":"16","quad":[[1310,234],[1351,234],[1351,266],[1310,266]],"words":[{"text":"16","quad":[[1321,234],[1342,234],[1342,266],[1321,266]]}]},{"text":"• If line 16 is a gain, enter the amount from line 16 on Form 1040, 1040-SR, or 1040-NR, line 7a.","quad":[[178,304],[1282,307],[1281,338],[178,336]],"words":[{"text":"•","quad":[[183,304],[189,304],[189,336],[183,336]]},{"text":"If","quad":[[199,304],[215,304],[215,336],[199,336]]},{"text":"line","quad":[[226,304],[258,304],[258,336],[226,336]]},{"text":"16","quad":[[274,304],[295,304],[295,336],[274,336]]},{"text":"is","quad":[[306,304],[322,304],[322,336],[306,336]]},{"text":"a","quad":[[338,304],[343,304],[343,336],[338,336]]},{"text":"gain,","quad":[[359,304],[413,305],[413,337],[359,336]]},{"text":"enter","quad":[[423,305],[477,305],[477,337],[423,337]]},{"text":"the","quad":[[487,305],[514,305],[514,337],[487,337]]},{"text":"amount","quad":[[535,305],[610,305],[610,337],[535,337]]},{"text":"from","quad":[[626,305],[663,305],[663,337],[626,337]]},{"text":"line","quad":[[685,305],[717,305],[717,337],[685,337]]},{"text":"16","quad":[[733,306],[754,306],[754,338],[733,338]]},{"text":"on","quad":[[770,306],[786,306],[786,338],[770,338]]},{"text":"Form","quad":[[807,306],[850,306],[850,338],[807,338]]},{"text":"1040,","quad":[[871,306],[930,306],[930,338],[871,338]]},{"text":"1040-SR,","quad":[[941,306],[1047,306],[1047,338],[941,338]]},{"text":"or","quad":[[1058,306],[1074,306],[1074,338],[1058,338]]},{"text":"1040-NR,","quad":[[1090,306],[1191,307],[1191,339],[1090,338]]},{"text":"line","quad":[[1202,307],[1234,307],[1234,339],[1202,339]]},{"text":"7a.","quad":[[1250,307],[1282,307],[1282,339],[1250,339]]}]},{"text":"Then, go to line 17 below.","quad":[[197,337],[490,337],[490,366],[197,366]],"words":[{"text":"Then,","quad":[[207,337],[260,337],[260,366],[207,366]]},{"text":"go","quad":[[275,337],[295,337],[295,366],[275,366]]},{"text":"to","quad":[[309,337],[324,337],[324,366],[309,366]]},{"text":"line","quad":[[334,337],[368,337],[368,366],[334,366]]},{"text":"17","quad":[[387,337],[407,337],[407,366],[387,366]]},{"text":"below.","quad":[[422,337],[490,337],[490,366],[422,366]]}]},{"text":" If line 16 is a loss, skip lines 17 through 20 below. Then, go to line 21. Also be sure to complete","quad":[[178,370],[1282,373],[1282,405],[178,402]],"words":[{"text":"If","quad":[[199,370],[215,370],[215,402],[199,402]]},{"text":"line","quad":[[221,370],[258,370],[258,402],[221,402]]},{"text":"16","quad":[[274,370],[290,370],[290,402],[274,402]]},{"text":"is","quad":[[306,370],[317,370],[317,402],[306,402]]},{"text":"a","quad":[[333,370],[338,370],[338,402],[333,402]]},{"text":"loss,","quad":[[354,370],[407,371],[407,403],[354,402]]},{"text":"skip","quad":[[418,371],[455,371],[455,403],[418,403]]},{"text":"lines","quad":[[466,371],[514,371],[514,403],[466,403]]},{"text":"17","quad":[[530,371],[551,371],[551,403],[530,403]]},{"text":"through","quad":[[562,371],[642,371],[642,403],[562,403]]},{"text":"20","quad":[[663,371],[679,371],[679,403],[663,403]]},{"text":"below.","quad":[[695,371],[765,372],[765,404],[695,403]]},{"text":"Then,","quad":[[781,372],[834,372],[834,404],[781,404]]},{"text":"go","quad":[[850,372],[866,372],[866,404],[850,404]]},{"text":"to","quad":[[882,372],[898,372],[898,404],[882,404]]},{"text":"line","quad":[[909,372],[941,372],[941,404],[909,404]]},{"text":"21.","quad":[[962,372],[994,372],[994,404],[962,404]]},{"text":"Also","quad":[[1005,372],[1042,372],[1042,404],[1005,404]]},{"text":"be","quad":[[1058,372],[1079,372],[1079,404],[1058,404]]},{"text":"sure","quad":[[1095,372],[1133,373],[1133,405],[1095,404]]},{"text":"to","quad":[[1149,373],[1165,373],[1165,405],[1149,405]]},{"text":"complete","quad":[[1181,373],[1277,373],[1277,405],[1181,405]]}]},{"text":"line 22.","quad":[[196,403],[283,403],[283,430],[196,430]],"words":[{"text":"line","quad":[[200,403],[232,403],[232,430],[200,430]]},{"text":"22.","quad":[[250,403],[281,403],[281,430],[250,430]]}]},{"text":"If line 16 is zero, skip lines 17 through 21 below and enter -0- on Form 1040, 1040-SR, or","quad":[[177,437],[1283,439],[1283,471],[177,469]],"words":[{"text":"If","quad":[[204,437],[220,437],[220,469],[204,469]]},{"text":"line","quad":[[230,437],[262,437],[262,469],[230,469]]},{"text":"16","quad":[[284,437],[300,437],[300,469],[284,469]]},{"text":"is","quad":[[316,437],[332,437],[332,469],[316,469]]},{"text":"zero,","quad":[[348,437],[407,437],[407,469],[348,469]]},{"text":"skip","quad":[[423,437],[460,438],[460,470],[423,469]]},{"text":"lines","quad":[[476,438],[524,438],[524,470],[476,470]]},{"text":"17","quad":[[540,438],[562,438],[562,470],[540,470]]},{"text":"through","quad":[[578,438],[658,438],[658,470],[578,470]]},{"text":"21","quad":[[679,438],[701,438],[701,470],[679,470]]},{"text":"below","quad":[[717,438],[775,438],[775,470],[717,470]]},{"text":"and","quad":[[797,438],[829,438],[829,470],[797,470]]},{"text":"enter","quad":[[850,438],[904,438],[904,470],[850,470]]},{"text":"-0-","quad":[[914,438],[946,438],[946,470],[914,470]]},{"text":"on","quad":[[962,438],[978,438],[978,470],[962,470]]},{"text":"Form","quad":[[1005,438],[1048,439],[1048,471],[1005,470]]},{"text":"1040,","quad":[[1069,439],[1133,439],[1133,471],[1069,471]]},{"text":"1040-SR,","quad":[[1144,439],[1251,439],[1251,471],[1144,471]]},{"text":"or","quad":[[1262,439],[1278,439],[1278,471],[1262,471]]}]},{"text":"1040-NR, line 7a. Then, go to line 22.","quad":[[199,470],[625,472],[624,499],[198,497]],"words":[{"text":"1040-NR,","quad":[[203,470],[311,471],[310,498],[202,497]]},{"text":"line","quad":[[316,471],[351,471],[350,498],[315,498]]},{"text":"7a.","quad":[[369,471],[396,471],[395,498],[368,498]]},{"text":"Then,","quad":[[410,471],[468,471],[467,498],[409,498]]},{"text":"go","quad":[[482,471],[499,471],[498,498],[481,498]]},{"text":"to","quad":[[513,471],[531,472],[530,499],[512,498]]},{"text":"line","quad":[[544,472],[576,472],[575,499],[543,499]]},{"text":"22.","quad":[[594,472],[621,472],[620,499],[593,499]]}]},{"text":"17","quad":[[110,536],[143,536],[143,561],[110,561]],"words":[{"text":"17","quad":[[118,536],[135,536],[135,561],[118,561]]}]},{"text":"Are lines 15 and 16 both gains?","quad":[[175,536],[535,537],[535,564],[175,563]],"words":[{"text":"Are","quad":[[184,536],[216,536],[216,563],[184,563]]},{"text":"lines","quad":[[224,536],[270,536],[270,563],[224,563]]},{"text":"15","quad":[[288,536],[306,536],[306,563],[288,563]]},{"text":"and","quad":[[324,536],[355,536],[355,564],[324,563]]},{"text":"16","quad":[[373,537],[391,537],[391,564],[373,564]]},{"text":"both","quad":[[409,537],[450,537],[450,564],[409,564]]},{"text":"gains?","quad":[[468,537],[530,537],[530,564],[468,564]]}]},{"text":" Yes. Go to line 18.","quad":[[184,569],[428,571],[428,598],[183,596]],"words":[{"text":"Yes.","quad":[[229,569],[270,570],[269,597],[228,596]]},{"text":"Go","quad":[[288,570],[311,570],[310,597],[287,597]]},{"text":"to","quad":[[324,570],[338,570],[337,597],[323,597]]},{"text":"line","quad":[[351,570],[383,571],[382,598],[350,597]]},{"text":"18.","quad":[[401,571],[428,571],[427,598],[400,598]]}]},{"text":" No. Skip lines 18 through 21, and go to line 22.","quad":[[184,603],[745,604],[745,632],[184,631]],"words":[{"text":"No.","quad":[[226,603],[259,603],[259,631],[226,631]]},{"text":"Skip","quad":[[273,603],[310,603],[310,631],[273,631]]},{"text":"lines","quad":[[324,603],[371,603],[371,631],[324,631]]},{"text":"18","quad":[[385,603],[404,603],[404,631],[385,631]]},{"text":"through","quad":[[418,603],[497,604],[497,632],[418,631]]},{"text":"21,","quad":[[516,604],[544,604],[544,632],[516,632]]},{"text":"and","quad":[[558,604],[591,604],[591,632],[558,632]]},{"text":"go","quad":[[609,604],[623,604],[623,632],[609,632]]},{"text":"to","quad":[[637,604],[656,604],[656,632],[637,632]]},{"text":"line","quad":[[666,604],[698,604],[698,632],[666,632]]},{"text":"22.","quad":[[717,604],[745,604],[745,632],[717,632]]}]},{"text":"18","quad":[[111,674],[145,674],[145,701],[111,701]],"words":[{"text":"18","quad":[[115,674],[133,674],[133,701],[115,701]]}]},{"text":"If you are required to complete the 28% Rate Gain Worksheet (see instructions), enter the","quad":[[174,674],[1283,672],[1283,701],[174,703]],"words":[{"text":"If","quad":[[179,674],[193,674],[193,703],[179,703]]},{"text":"you","quad":[[213,674],[242,674],[242,703],[213,703]]},{"text":"are","quad":[[266,674],[290,674],[290,703],[266,703]]},{"text":"required","quad":[[310,674],[397,674],[397,703],[310,703]]},{"text":"to","quad":[[416,674],[431,674],[431,703],[416,703]]},{"text":"complete","quad":[[455,673],[552,673],[552,702],[455,702]]},{"text":"the","quad":[[571,673],[600,673],[600,702],[571,702]]},{"text":"28%","quad":[[624,673],[658,673],[658,702],[624,702]]},{"text":"Rate","quad":[[687,673],[731,673],[731,702],[687,702]]},{"text":"Gain","quad":[[760,673],[799,673],[799,702],[760,702]]},{"text":"Worksheet","quad":[[828,673],[944,673],[944,702],[828,702]]},{"text":"(see","quad":[[963,673],[1002,673],[1002,702],[963,702]]},{"text":"instructions),","quad":[[1021,672],[1167,672],[1167,701],[1021,701]]},{"text":"enter","quad":[[1181,672],[1230,672],[1230,701],[1181,701]]},{"text":"the","quad":[[1249,672],[1278,672],[1278,701],[1249,701]]}]},{"text":"amount, if any, from line 7 of that worksheet","quad":[[179,706],[678,706],[678,731],[179,731]],"words":[{"text":"amount,","quad":[[187,706],[270,706],[270,731],[187,731]]},{"text":"if","quad":[[279,706],[291,706],[291,731],[279,731]]},{"text":"any,","quad":[[304,706],[345,706],[345,731],[304,731]]},{"text":"from","quad":[[354,706],[391,706],[391,731],[354,731]]},{"text":"line","quad":[[412,706],[445,706],[445,731],[412,731]]},{"text":"7","quad":[[462,706],[466,706],[466,731],[462,731]]},{"text":"of","quad":[[483,706],[495,706],[495,731],[483,731]]},{"text":"that","quad":[[508,706],[545,706],[545,731],[508,731]]},{"text":"worksheet","quad":[[562,706],[670,706],[670,731],[562,731]]}]},{"text":"18","quad":[[1309,700],[1351,700],[1351,732],[1309,732]],"words":[{"text":"18","quad":[[1320,700],[1341,700],[1341,732],[1320,732]]}]},{"text":"19","quad":[[110,774],[145,774],[145,801],[110,801]],"words":[{"text":"19","quad":[[119,774],[137,774],[137,801],[119,801]]}]},{"text":"If you are required to complete the Unrecaptured Section 1250 Gain Worksheet (see","quad":[[175,773],[1284,773],[1284,803],[175,803]],"words":[{"text":"If","quad":[[180,773],[195,773],[195,803],[180,803]]},{"text":"you","quad":[[215,773],[245,773],[245,803],[215,803]]},{"text":"are","quad":[[275,773],[300,773],[300,803],[275,803]]},{"text":"required","quad":[[325,773],[410,773],[410,803],[325,803]]},{"text":"to","quad":[[435,773],[450,773],[450,803],[435,803]]},{"text":"complete","quad":[[480,773],[575,773],[575,803],[480,803]]},{"text":"the","quad":[[600,773],[630,773],[630,803],[600,803]]},{"text":"Unrecaptured","quad":[[655,773],[809,773],[809,803],[655,803]]},{"text":"Section","quad":[[839,773],[914,773],[914,803],[839,803]]},{"text":"1250","quad":[[944,773],[989,773],[989,803],[944,803]]},{"text":"Gain","quad":[[1024,773],[1064,773],[1064,803],[1024,803]]},{"text":"Worksheet","quad":[[1094,773],[1214,773],[1214,803],[1094,803]]},{"text":"(see","quad":[[1234,773],[1274,773],[1274,803],[1234,803]]}]},{"text":"instructions), enter the amount, if any, from line 18 of that worksheet","quad":[[178,804],[943,805],[943,832],[178,831]],"words":[{"text":"instructions),","quad":[[182,804],[322,804],[322,831],[182,831]]},{"text":"enter","quad":[[336,804],[385,804],[385,831],[336,831]]},{"text":"the","quad":[[394,804],[426,804],[426,831],[394,831]]},{"text":"amount,","quad":[[444,804],[529,804],[529,831],[444,831]]},{"text":"if","quad":[[538,804],[547,804],[547,831],[538,831]]},{"text":"any,","quad":[[560,804],[601,805],[601,832],[560,832]]},{"text":"from","quad":[[610,805],[646,805],[646,832],[610,832]]},{"text":"line","quad":[[668,805],[700,805],[700,832],[668,832]]},{"text":"18","quad":[[718,805],[736,805],[736,832],[718,832]]},{"text":"of","quad":[[754,805],[768,805],[768,832],[754,832]]},{"text":"that","quad":[[776,805],[817,805],[817,832],[776,832]]},{"text":"worksheet","quad":[[830,805],[938,805],[938,832],[830,832]]}]},{"text":"19","quad":[[1310,800],[1351,800],[1351,832],[1310,832]],"words":[{"text":"19","quad":[[1321,800],[1342,800],[1342,832],[1321,832]]}]},{"text":"20","quad":[[109,871],[145,871],[145,896],[109,896]],"words":[{"text":"20","quad":[[117,871],[138,871],[138,896],[117,896]]}]},{"text":"Are lines 18 and 19 both zero or blank and you are not filing Form 4952?","quad":[[175,870],[988,870],[988,900],[175,900]],"words":[{"text":"Are","quad":[[185,870],[215,870],[215,900],[185,900]]},{"text":"lines","quad":[[225,870],[270,870],[270,900],[225,900]]},{"text":"18","quad":[[290,870],[305,870],[305,900],[290,900]]},{"text":"and","quad":[[321,870],[356,870],[356,900],[321,900]]},{"text":"19","quad":[[371,870],[391,870],[391,900],[371,900]]},{"text":"both","quad":[[406,870],[446,870],[446,900],[406,900]]},{"text":"zero","quad":[[466,870],[506,870],[506,900],[466,900]]},{"text":"or","quad":[[521,870],[536,870],[536,900],[521,900]]},{"text":"blank","quad":[[551,870],[602,870],[602,900],[551,900]]},{"text":"and","quad":[[617,870],[652,870],[652,900],[617,900]]},{"text":"you","quad":[[667,870],[697,870],[697,900],[667,900]]},{"text":"are","quad":[[717,870],[742,870],[742,900],[717,900]]},{"text":"not","quad":[[757,870],[787,870],[787,900],[757,900]]},{"text":"filing","quad":[[797,870],[842,870],[842,900],[797,900]]},{"text":"Form","quad":[[863,870],[903,870],[903,900],[863,900]]},{"text":"4952?","quad":[[923,870],[988,870],[988,900],[923,900]]}]},{"text":" Yes. Complete the Qualified Dividends and Capital Gain Tax Worksheet in the instructions","quad":[[182,903],[1281,905],[1281,934],[182,932]],"words":[{"text":"Yes.","quad":[[226,903],[269,903],[269,932],[226,932]]},{"text":"Complete","quad":[[289,903],[385,903],[385,932],[289,932]]},{"text":"the","quad":[[400,903],[429,903],[429,932],[400,932]]},{"text":"Qualified","quad":[[448,903],[540,904],[540,933],[448,932]]},{"text":"Dividends","quad":[[564,904],[671,904],[671,933],[564,933]]},{"text":"and","quad":[[686,904],[719,904],[719,933],[686,933]]},{"text":"Capital","quad":[[744,904],[816,904],[816,933],[744,933]]},{"text":"Gain","quad":[[836,904],[879,904],[879,933],[836,933]]},{"text":"Tax","quad":[[899,904],[932,904],[932,933],[899,933]]},{"text":"Worksheet","quad":[[952,904],[1068,905],[1068,934],[952,933]]},{"text":"in","quad":[[1083,905],[1092,905],[1092,934],[1083,934]]},{"text":"the","quad":[[1112,905],[1136,905],[1136,934],[1112,934]]},{"text":"instructions","quad":[[1155,905],[1276,905],[1276,934],[1155,934]]}]},{"text":"for Form 1040, line 16. Don't complete lines 21 and 22 below.","quad":[[217,934],[915,935],[915,964],[217,962]],"words":[{"text":"for","quad":[[222,934],[246,934],[246,962],[222,962]]},{"text":"Form","quad":[[261,934],[304,934],[304,962],[261,962]]},{"text":"1040,","quad":[[328,934],[387,934],[387,962],[328,962]]},{"text":"line","quad":[[392,934],[425,934],[425,962],[392,962]]},{"text":"16.","quad":[[445,934],[474,934],[474,962],[445,962]]},{"text":"Don't","quad":[[488,934],[542,934],[542,962],[488,962]]},{"text":"complete","quad":[[556,934],[653,935],[653,963],[556,962]]},{"text":"lines","quad":[[663,935],[711,935],[711,963],[663,963]]},{"text":"21","quad":[[726,935],[745,935],[745,963],[726,963]]},{"text":"and","quad":[[760,935],[794,935],[794,963],[760,963]]},{"text":"22","quad":[[808,935],[828,935],[828,963],[808,963]]},{"text":"below.","quad":[[847,935],[915,935],[915,963],[847,963]]}]},{"text":" No. Complete the Schedule D Tax Worksheet in the instructions. Don't complete lines 21","quad":[[182,1002],[1281,1006],[1281,1040],[182,1036]],"words":[{"text":"No.","quad":[[227,1002],[261,1002],[261,1036],[227,1036]]},{"text":"Complete","quad":[[278,1002],[375,1003],[375,1037],[278,1036]]},{"text":"the","quad":[[392,1003],[420,1003],[420,1037],[392,1037]]},{"text":"Schedule","quad":[[443,1003],[545,1003],[545,1037],[443,1037]]},{"text":"D","quad":[[562,1003],[567,1003],[567,1037],[562,1037]]},{"text":"Tax","quad":[[590,1003],[624,1004],[624,1038],[590,1037]]},{"text":"Worksheet","quad":[[647,1004],[765,1004],[765,1038],[647,1038]]},{"text":"in","quad":[[777,1004],[794,1004],[794,1038],[777,1038]]},{"text":"the","quad":[[811,1004],[839,1004],[839,1038],[811,1038]]},{"text":"instructions.","quad":[[856,1004],[992,1005],[992,1039],[856,1038]]},{"text":"Don't","quad":[[1009,1005],[1066,1005],[1066,1039],[1009,1039]]},{"text":"complete","quad":[[1083,1005],[1179,1006],[1179,1040],[1083,1039]]},{"text":"lines","quad":[[1190,1006],[1236,1006],[1236,1040],[1190,1040]]},{"text":"21","quad":[[1253,1006],[1275,1006],[1275,1040],[1253,1040]]}]},{"text":"and 22 below.","quad":[[218,1039],[380,1039],[380,1065],[218,1065]],"words":[{"text":"and","quad":[[227,1039],[257,1039],[257,1065],[227,1065]]},{"text":"22","quad":[[274,1039],[291,1039],[291,1065],[274,1065]]},{"text":"below.","quad":[[309,1039],[378,1039],[378,1065],[309,1065]]}]},{"text":"21","quad":[[109,1104],[183,1104],[183,1130],[109,1130]],"words":[{"text":"21","quad":[[118,1104],[135,1104],[135,1130],[118,1130]]}]},{"text":"If line 16 is a loss, enter here and on Form 1040, 1040-SR, or 1040-NR, line 7a, the smaller of:","quad":[[169,1103],[1234,1104],[1234,1131],[169,1130]],"words":[{"text":"If","quad":[[182,1103],[191,1103],[191,1130],[182,1130]]},{"text":"line","quad":[[200,1103],[236,1103],[236,1130],[200,1130]]},{"text":"16","quad":[[250,1103],[268,1103],[268,1130],[250,1130]]},{"text":"is","quad":[[281,1103],[295,1103],[295,1130],[281,1130]]},{"text":"a","quad":[[308,1103],[317,1103],[317,1130],[308,1130]]},{"text":"loss,","quad":[[326,1103],[380,1103],[380,1130],[326,1130]]},{"text":"enter","quad":[[389,1103],[439,1103],[439,1130],[389,1130]]},{"text":"here","quad":[[452,1103],[493,1103],[493,1130],[452,1130]]},{"text":"and","quad":[[511,1103],[542,1103],[542,1130],[511,1130]]},{"text":"on","quad":[[560,1103],[578,1103],[578,1130],[560,1130]]},{"text":"Form","quad":[[596,1103],[636,1103],[636,1130],[596,1130]]},{"text":"1040,","quad":[[659,1103],[717,1104],[717,1131],[659,1130]]},{"text":"1040-SR,","quad":[[726,1104],[830,1104],[830,1131],[726,1131]]},{"text":"or","quad":[[843,1104],[857,1104],[857,1131],[843,1131]]},{"text":"1040-NR,","quad":[[870,1104],[973,1104],[973,1131],[870,1131]]},{"text":"line","quad":[[982,1104],[1014,1104],[1014,1131],[982,1131]]},{"text":"7a,","quad":[[1032,1104],[1059,1104],[1059,1131],[1032,1131]]},{"text":"the","quad":[[1068,1104],[1095,1104],[1095,1131],[1068,1131]]},{"text":"smaller","quad":[[1113,1104],[1194,1104],[1194,1131],[1113,1131]]},{"text":"of:","quad":[[1207,1104],[1230,1104],[1230,1131],[1207,1131]]}]},{"text":"The loss on line 16; or","quad":[[179,1161],[446,1164],[446,1191],[179,1189]],"words":[{"text":"The","quad":[[207,1161],[235,1162],[235,1190],[207,1189]]},{"text":"loss","quad":[[249,1162],[291,1162],[291,1190],[249,1190]]},{"text":"on","quad":[[305,1162],[324,1163],[324,1191],[305,1190]]},{"text":"line","quad":[[338,1163],[371,1163],[371,1191],[338,1191]]},{"text":"16;","quad":[[385,1163],[418,1164],[418,1192],[385,1191]]},{"text":"or","quad":[[427,1164],[441,1164],[441,1192],[427,1192]]}]},{"text":"21","quad":[[1308,1164],[1376,1164],[1376,1200],[1308,1200]],"words":[{"text":"21","quad":[[1320,1164],[1344,1164],[1344,1200],[1320,1200]]}]},{"text":" ($3,000), or if married filing separately, ($1,500)","quad":[[177,1194],[725,1191],[725,1228],[177,1231]],"words":[{"text":"($3,000),","quad":[[202,1194],[300,1193],[300,1230],[202,1231]]},{"text":"or","quad":[[306,1193],[325,1193],[325,1230],[306,1230]]},{"text":"if","quad":[[331,1193],[343,1193],[343,1230],[331,1230]]},{"text":"married","quad":[[356,1193],[436,1193],[436,1230],[356,1230]]},{"text":"filing","quad":[[442,1193],[491,1192],[491,1229],[442,1230]]},{"text":"separately,","quad":[[509,1192],[626,1192],[626,1229],[509,1229]]},{"text":"($1,500)","quad":[[633,1192],[725,1191],[725,1228],[633,1229]]}]},{"text":"Note: When figuring which amount is smaller, treat both amounts as positive numbers.","quad":[[177,1261],[1148,1262],[1148,1293],[177,1292]],"words":[{"text":"Note:","quad":[[187,1261],[244,1261],[244,1292],[187,1292]]},{"text":"When","quad":[[260,1261],[311,1261],[311,1292],[260,1292]]},{"text":"figuring","quad":[[322,1261],[399,1261],[399,1292],[322,1292]]},{"text":"which","quad":[[420,1261],[471,1261],[471,1292],[420,1292]]},{"text":"amount","quad":[[492,1261],[570,1261],[570,1292],[492,1292]]},{"text":"is","quad":[[580,1261],[590,1261],[590,1292],[580,1292]]},{"text":"smaller,","quad":[[606,1261],[688,1262],[688,1293],[606,1292]]},{"text":"treat","quad":[[699,1262],[745,1262],[745,1293],[699,1293]]},{"text":"both","quad":[[761,1262],[797,1262],[797,1293],[761,1293]]},{"text":"amounts","quad":[[817,1262],[905,1262],[905,1293],[817,1293]]},{"text":"as","quad":[[921,1262],[936,1262],[936,1293],[921,1293]]},{"text":"positive","quad":[[952,1262],[1029,1262],[1029,1293],[952,1293]]},{"text":"numbers.","quad":[[1045,1262],[1148,1262],[1148,1293],[1045,1293]]}]},{"text":"22","quad":[[111,1339],[145,1339],[145,1365],[111,1365]],"words":[{"text":"22","quad":[[120,1339],[137,1339],[137,1365],[120,1365]]}]},{"text":"Do you have qualified dividends on Form 1040, 1040-SR, or 1040-NR, line 3a?","quad":[[174,1336],[1055,1336],[1055,1366],[174,1366]],"words":[{"text":"Do","quad":[[184,1336],[209,1336],[209,1366],[184,1366]]},{"text":"you","quad":[[224,1336],[254,1336],[254,1366],[224,1366]]},{"text":"have","quad":[[269,1336],[314,1336],[314,1366],[269,1366]]},{"text":"qualified","quad":[[334,1336],[419,1336],[419,1366],[334,1366]]},{"text":"dividends","quad":[[434,1336],[534,1336],[534,1366],[434,1366]]},{"text":"on","quad":[[549,1336],[564,1336],[564,1366],[549,1366]]},{"text":"Form","quad":[[584,1336],[625,1336],[625,1366],[584,1366]]},{"text":"1040,","quad":[[650,1336],[705,1336],[705,1366],[650,1366]]},{"text":"1040-SR,","quad":[[720,1336],[820,1336],[820,1366],[720,1366]]},{"text":"or","quad":[[830,1336],[845,1336],[845,1366],[830,1366]]},{"text":"1040-NR,","quad":[[860,1336],[965,1336],[965,1366],[860,1366]]},{"text":"line","quad":[[970,1336],[1005,1336],[1005,1366],[970,1366]]},{"text":"3a?","quad":[[1020,1336],[1055,1336],[1055,1366],[1020,1366]]}]},{"text":" Yes. Complete the Qualified Dividends and Capital Gain Tax Worksheet in the instructions","quad":[[183,1403],[1282,1405],[1282,1434],[183,1432]],"words":[{"text":"Yes.","quad":[[227,1403],[270,1403],[270,1432],[227,1432]]},{"text":"Complete","quad":[[290,1403],[386,1403],[386,1432],[290,1432]]},{"text":"the","quad":[[401,1403],[430,1403],[430,1432],[401,1432]]},{"text":"Qualified","quad":[[449,1403],[541,1404],[541,1433],[449,1432]]},{"text":"Dividends","quad":[[561,1404],[667,1404],[667,1433],[561,1433]]},{"text":"and","quad":[[687,1404],[720,1404],[720,1433],[687,1433]]},{"text":"Capital","quad":[[745,1404],[817,1404],[817,1433],[745,1433]]},{"text":"Gain","quad":[[837,1404],[875,1404],[875,1433],[837,1433]]},{"text":"Tax","quad":[[900,1404],[929,1404],[929,1433],[900,1433]]},{"text":"Worksheet","quad":[[953,1404],[1069,1405],[1069,1434],[953,1433]]},{"text":"in","quad":[[1084,1405],[1093,1405],[1093,1434],[1084,1434]]},{"text":"the","quad":[[1108,1405],[1137,1405],[1137,1434],[1108,1434]]},{"text":"instructions","quad":[[1151,1405],[1277,1405],[1277,1434],[1151,1434]]}]},{"text":"for Form 1040, line 16.","quad":[[218,1435],[473,1435],[473,1462],[218,1462]],"words":[{"text":"for","quad":[[222,1435],[249,1435],[249,1462],[222,1462]]},{"text":"Form","quad":[[263,1435],[303,1435],[303,1462],[263,1462]]},{"text":"1040,","quad":[[325,1435],[384,1435],[384,1462],[325,1462]]},{"text":"line","quad":[[392,1435],[424,1435],[424,1462],[392,1462]]},{"text":"16.","quad":[[442,1435],[469,1435],[469,1462],[442,1462]]}]},{"text":" No. Complete the rest of Form 1040, 1040-SR, or 1040-NR.","quad":[[183,1503],[889,1502],[889,1531],[183,1532]],"words":[{"text":"No.","quad":[[227,1503],[260,1503],[260,1532],[227,1532]]},{"text":"Complete","quad":[[275,1503],[372,1503],[372,1532],[275,1532]]},{"text":"the","quad":[[386,1503],[415,1503],[415,1532],[386,1532]]},{"text":"rest","quad":[[430,1503],[468,1503],[468,1532],[430,1532]]},{"text":"of","quad":[[478,1503],[497,1503],[497,1532],[478,1532]]},{"text":"Form","quad":[[507,1503],[551,1502],[551,1531],[507,1532]]},{"text":"1040,","quad":[[575,1502],[633,1502],[633,1531],[575,1531]]},{"text":"1040-SR,","quad":[[642,1502],[744,1502],[744,1531],[642,1531]]},{"text":"or","quad":[[758,1502],[773,1502],[773,1531],[758,1531]]},{"text":"1040-NR.","quad":[[787,1502],[889,1502],[889,1531],[787,1531]]}]},{"text":"Schedule D (Form 1040) 2025","quad":[[1329,1575],[1600,1575],[1600,1599],[1329,1599]],"words":[{"text":"Schedule","quad":[[1337,1575],[1413,1575],[1413,1599],[1337,1599]]},{"text":"D","quad":[[1429,1575],[1433,1575],[1433,1599],[1429,1599]]},{"text":"(Form","quad":[[1445,1575],[1488,1575],[1488,1599],[1445,1599]]},{"text":"1040)","quad":[[1504,1575],[1548,1575],[1548,1599],[1504,1599]]},{"text":"2025","quad":[[1560,1575],[1596,1575],[1596,1599],[1560,1599]]}]}],"refused":null} +{"ocr_response":1,"id":"pdf-jpx-refused","identity":{"recognizer":"ethos-parser-ocr","version":"0.1.0","rule":"ppocrv5-mobile-v1","det_sha256":"4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae","rec_sha256":"c3461add59bb4323ecba96a492ab75e06dda42467c9e3d0c18db5d1d21924be8","dict_sha256":"e025a66d31f327ba0c232e03f407ae8d105e1e709e7ccb3f408aa778c24e70d6","runtime":"rten 0.26.0; rten-vecmath 0.26.0 with fixed-order sums"},"width":0,"height":0,"lines":[],"refused":"filter-not-decoded:JPXDecode"} diff --git a/ocr/tests/data/f1040sd-p2.flate b/ocr/tests/data/f1040sd-p2.flate new file mode 100644 index 0000000..5aa6836 Binary files /dev/null and b/ocr/tests/data/f1040sd-p2.flate differ diff --git a/ocr/tests/data/nist-171r3-p55-turned.png b/ocr/tests/data/nist-171r3-p55-turned.png new file mode 100644 index 0000000..4970f70 Binary files /dev/null and b/ocr/tests/data/nist-171r3-p55-turned.png differ diff --git a/ocr/tests/data/nist-171r3-p55.png b/ocr/tests/data/nist-171r3-p55.png new file mode 100644 index 0000000..213802b Binary files /dev/null and b/ocr/tests/data/nist-171r3-p55.png differ diff --git a/ocr/tests/data/nist-37r2-p157.jpg b/ocr/tests/data/nist-37r2-p157.jpg new file mode 100644 index 0000000..8e94843 Binary files /dev/null and b/ocr/tests/data/nist-37r2-p157.jpg differ diff --git a/ocr/tests/data/nist-37r2-p6.flate b/ocr/tests/data/nist-37r2-p6.flate new file mode 100644 index 0000000..32f00ac Binary files /dev/null and b/ocr/tests/data/nist-37r2-p6.flate differ diff --git a/ocr/tests/data/requests.json b/ocr/tests/data/requests.json new file mode 100644 index 0000000..01a4497 --- /dev/null +++ b/ocr/tests/data/requests.json @@ -0,0 +1,111 @@ +[ + { + "id": "png-file", + "file": "nist-171r3-p55.png", + "image": { + "kind": "file" + }, + "orientation": 0 + }, + { + "id": "png-file-turned", + "file": "nist-171r3-p55-turned.png", + "image": { + "kind": "file" + }, + "orientation": 1 + }, + { + "id": "pdf-dct-gray", + "file": "nist-37r2-p157.jpg", + "orientation": 0, + "image": { + "kind": "pdf", + "width": 1700, + "height": 2200, + "bits_per_component": 8, + "colour": { + "space": "gray" + }, + "filters": [ + "DCTDecode" + ], + "decode_parms": [ + null + ], + "decode": null, + "image_mask": false + } + }, + { + "id": "pdf-flate-gray-png-up", + "file": "nist-37r2-p6.flate", + "orientation": 0, + "image": { + "kind": "pdf", + "width": 1700, + "height": 2200, + "bits_per_component": 8, + "colour": { + "space": "gray" + }, + "filters": [ + "FlateDecode" + ], + "decode_parms": [ + { + "predictor": 12, + "colors": 1, + "bits_per_component": 8, + "columns": 1700 + } + ], + "decode": null, + "image_mask": false + } + }, + { + "id": "pdf-flate-1bit", + "file": "f1040sd-p2.flate", + "orientation": 0, + "image": { + "kind": "pdf", + "width": 1700, + "height": 2200, + "bits_per_component": 1, + "colour": { + "space": "gray" + }, + "filters": [ + "FlateDecode" + ], + "decode_parms": [ + null + ], + "decode": null, + "image_mask": false + } + }, + { + "id": "pdf-jpx-refused", + "file": "f1040sd-p2.flate", + "orientation": 0, + "image": { + "kind": "pdf", + "width": 10, + "height": 10, + "bits_per_component": 8, + "colour": { + "space": "gray" + }, + "filters": [ + "JPXDecode" + ], + "decode_parms": [ + null + ], + "decode": null, + "image_mask": false + } + } +] \ No newline at end of file diff --git a/ocr/tests/fixtures.rs b/ocr/tests/fixtures.rs new file mode 100644 index 0000000..fca26ac --- /dev/null +++ b/ocr/tests/fixtures.rs @@ -0,0 +1,134 @@ +// Copyright 2026 The ethos-parser maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The recognizer answers the fixture requests with exactly the bytes `data/expected.jsonl` holds, +//! on whatever CPU runs this: `docs/35-OCR-SCOPE.md` §10.1, run in CI on every supported platform. +//! The requests cover each decode path (a PNG file, a turned PNG stood upright, a gray JPEG stream, +//! an 8-bit Flate stream with a PNG predictor, a 1-bit Flate stream) and a refusal. +//! +//! `expected.jsonl` was written on Apple silicon. To write it again after a deliberate change, run +//! with `ETHOS_OCR_BLESS=1`, and say in the commit why the bytes moved. + +use std::io::{BufRead, BufReader, Write}; +use std::path::Path; +use std::process::{Command, Stdio}; + +use serde_json::Value; + +fn data(name: &str) -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/data") + .join(name) +} + +/// Every fixture request through one `serve` process; the answers, one per line. +fn answers() -> Vec { + let reqs: Vec = + serde_json::from_slice(&std::fs::read(data("requests.json")).unwrap()).unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_ethos-parser-ocr")) + .arg("serve") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + let mut out = Vec::new(); + for r in &reqs { + let bytes = std::fs::read(data(r["file"].as_str().unwrap())).unwrap(); + let header = serde_json::json!({"ocr_request": 1, "id": r["id"], "image": r["image"], + "orientation": r["orientation"], "bytes": bytes.len()}); + writeln!(stdin, "{header}").unwrap(); + stdin.write_all(&bytes).unwrap(); + stdin.flush().unwrap(); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + out.push(line.trim_end().to_string()); + } + drop(stdin); + assert!(child.wait().unwrap().success()); + out +} + +/// Whether this CPU is one the recognizer refuses (`main.rs`: x86-64 without AVX2 and FMA). +fn refused_cpu() -> bool { + #[cfg(target_arch = "x86_64")] + return !(is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma")); + #[cfg(not(target_arch = "x86_64"))] + return false; +} + +#[test] +fn the_fixtures_read_the_same_bytes_on_every_cpu() { + if refused_cpu() { + // Then the refusal is what is tested: by name, with its own exit code, before any read. + let out = Command::new(env!("CARGO_BIN_EXE_ethos-parser-ocr")) + .arg("serve") + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(3)); + assert!(String::from_utf8_lossy(&out.stderr).contains("refused: cpu-lacks-avx2-fma")); + return; + } + let got = answers(); + let path = data("expected.jsonl"); + if std::env::var_os("ETHOS_OCR_BLESS").is_some() { + std::fs::write(&path, got.join("\n") + "\n").unwrap(); + } + let want: Vec = std::fs::read_to_string(&path) + .unwrap() + .lines() + .map(String::from) + .collect(); + assert_eq!(got.len(), want.len()); + for (g, w) in got.iter().zip(&want) { + let id = serde_json::from_str::(w).unwrap()["id"].clone(); + assert!(g == w, "{id}: the answer's bytes moved"); + } + + // What the bytes say, so a blessed regression cannot pass unread. + let by_id = |id: &str| -> Value { + got.iter() + .map(|l| serde_json::from_str::(l).unwrap()) + .find(|v| v["id"] == id) + .unwrap() + }; + let texts = |v: &Value| -> Vec { + v["lines"] + .as_array() + .unwrap() + .iter() + .map(|l| l["text"].as_str().unwrap().to_string()) + .collect() + }; + let upright = by_id("png-file"); + assert!(texts(&upright).len() > 20, "a text page reads as lines"); + // A raster stored turned reads as the upright one does, its quads in the stored grid. + let turned = by_id("png-file-turned"); + assert_eq!(texts(&turned), texts(&upright)); + assert_eq!( + (turned["width"].clone(), turned["height"].clone()), + (upright["height"].clone(), upright["width"].clone()) + ); + for id in ["pdf-dct-gray", "pdf-flate-gray-png-up", "pdf-flate-1bit"] { + let v = by_id(id); + assert!( + v["refused"].is_null() && texts(&v).len() > 10, + "{id} is read" + ); + } + let refused = by_id("pdf-jpx-refused"); + assert_eq!(refused["refused"], "filter-not-decoded:JPXDecode"); + assert!(refused["lines"].as_array().unwrap().is_empty()); +} diff --git a/ocr/vendor/rten-vecmath/Cargo.toml b/ocr/vendor/rten-vecmath/Cargo.toml new file mode 100644 index 0000000..c062ace --- /dev/null +++ b/ocr/vendor/rten-vecmath/Cargo.toml @@ -0,0 +1,53 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +name = "rten-vecmath" +version = "0.26.0" +authors = ["Robert Knight"] +build = false +include = [ + "/src", + "/README.md", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "SIMD vectorized implementations of various math functions used in ML models" +homepage = "https://github.com/robertknight/rten" +readme = "README.md" +license = "MIT OR Apache-2.0" +repository = "https://github.com/robertknight/rten" +resolver = "2" + +[lib] +name = "rten_vecmath" +crate-type = ["lib"] +path = "src/lib.rs" + +[dependencies.rten-base] +version = "0.26.0" + +[dependencies.rten-simd] +version = "0.26.0" + +[dev-dependencies.fastrand] +version = "2.0.2" + +[dev-dependencies.libm] +version = "0.2.6" + +[lints.clippy] +manual_memcpy = "allow" +needless_range_loop = "allow" diff --git a/ocr/vendor/rten-vecmath/ETHOS-PATCH.md b/ocr/vendor/rten-vecmath/ETHOS-PATCH.md new file mode 100644 index 0000000..39c6277 --- /dev/null +++ b/ocr/vendor/rten-vecmath/ETHOS-PATCH.md @@ -0,0 +1,15 @@ +# rten-vecmath 0.26.0, patched + +Upstream: , crate `rten-vecmath` 0.26.0, licensed +MIT OR Apache-2.0 (used here under Apache-2.0). Vendored so ethos-parser's recognizer gives the +same bytes on every CPU (docs/35-OCR-SCOPE.md). + +**The one change.** Upstream sums a slice per SIMD lane and then across the lanes, so the order of +the additions, and the result's last bits, depend on the CPU's vector width: 4 lanes on NEON, 8 on +AVX2, 16 on AVX-512. `src/sum.rs` adds `fixed_order_sum`: element `i` adds into partial +`i % 16` in input order, and the 16 partials add in one fixed tree. Every lane count divides 16, +so every CPU performs the same additions. `Sum`, `SumSquare`, `SumAbs`, `SumSquareSub`, +`SumExpSub` (`src/sum.rs`) and the exponential sums of `Softmax` and `LogSoftmax` +(`src/softmax.rs`) use it; the changed places are marked "Ethos fork". Nothing else differs from +the published crate but one comment: `src/ulp.rs` cites Java's `Math.ulp` documentation by name +rather than by its URL. diff --git a/ocr/vendor/rten-vecmath/README.md b/ocr/vendor/rten-vecmath/README.md new file mode 100644 index 0000000..ca57b8e --- /dev/null +++ b/ocr/vendor/rten-vecmath/README.md @@ -0,0 +1,12 @@ +# rten-vecmath + +This crate contains SIMD-vectorized kernels ("vectorized math") for various +operations used in machine learning models. This includes: + + - Math functions such as exp, erf, tanh + - Activation function such as gelu + - Normalization functions such as softmax and mean-variance normalization + - Reduction functions such as sums and sum-of-square + +SIMD operations are implemented using portable SIMD types from the rten-simd +crate. diff --git a/ocr/vendor/rten-vecmath/src/convert.rs b/ocr/vendor/rten-vecmath/src/convert.rs new file mode 100644 index 0000000..e0d1304 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/convert.rs @@ -0,0 +1,194 @@ +use std::mem::MaybeUninit; + +use rten_simd::ops::{BitOps, Extend, NarrowSaturate}; +use rten_simd::{Isa, SimdOp, SliceWriter, f16}; + +/// Convert a slice of `f16` values to `f32`. +pub struct F16ToF32<'s, 'd> { + src: &'s [f16], + dest: &'d mut [MaybeUninit], +} + +impl<'s, 'd> F16ToF32<'s, 'd> { + /// Create a conversion operation which reads from `src` and writes the + /// converted values to `dest`. + /// + /// Panics if `src` and `dest` have different lengths. + pub fn new(src: &'s [f16], dest: &'d mut [MaybeUninit]) -> Self { + assert_eq!(src.len(), dest.len()); + F16ToF32 { src, dest } + } +} + +impl<'d> SimdOp for F16ToF32<'_, 'd> { + type Output = &'d mut [f32]; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let f16_ops = isa.f16(); + let f32_ops = isa.f32(); + let f16_v_len = f16_ops.len(); + + let mut dest_writer = SliceWriter::new(self.dest); + + // Main loop, unrolled by two. + let mut chunks = self.src.chunks_exact(f16_v_len * 2); + for chunk in chunks.by_ref() { + let xs = f16_ops.load_many::<2>(chunk); + let lo0 = f16_ops.extend_low(xs[0]); + let hi0 = f16_ops.extend_high(xs[0]); + let lo1 = f16_ops.extend_low(xs[1]); + let hi1 = f16_ops.extend_high(xs[1]); + // Store all four f32 vectors with a single bounds check. + dest_writer.write_vecs(f32_ops, [lo0, hi0, lo1, hi1]); + } + + // Convert a remaining whole `f16` vector, if any. + let mut chunks = chunks.remainder().chunks_exact(f16_v_len); + for chunk in chunks.by_ref() { + let x = f16_ops.load(chunk); + let low = f16_ops.extend_low(x); + let high = f16_ops.extend_high(x); + dest_writer.write_vec(f32_ops, low); + dest_writer.write_vec(f32_ops, high); + } + + // Convert tail elements which don't fill a whole vector. + for &x in chunks.remainder() { + dest_writer.write_scalar(x.to_f32()); + } + + dest_writer.into_mut_slice() + } +} + +/// Convert a slice of `f32` values to `f16`. +/// +/// Values are rounded to the nearest `f16`, with ties to even. Values whose +/// magnitude exceeds the `f16` range are rounded to infinity. +pub struct F32ToF16<'s, 'd> { + src: &'s [f32], + dest: &'d mut [MaybeUninit], +} + +impl<'s, 'd> F32ToF16<'s, 'd> { + /// Create a conversion operation which reads from `src` and writes the + /// converted values to `dest`. + /// + /// Panics if `src` and `dest` have different lengths. + pub fn new(src: &'s [f32], dest: &'d mut [MaybeUninit]) -> Self { + assert_eq!(src.len(), dest.len()); + F32ToF16 { src, dest } + } +} + +impl<'d> SimdOp for F32ToF16<'_, 'd> { + type Output = &'d mut [f16]; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let f32_ops = isa.f32(); + let f16_ops = isa.f16(); + let f32_v_len = f32_ops.len(); + + let mut src_chunks = self.src.chunks_exact(f32_v_len * 2); + let mut dest_writer = SliceWriter::new(self.dest); + + for src_chunk in src_chunks.by_ref() { + let xs = f32_ops.load_many::<2>(src_chunk); + let half = f32_ops.narrow_saturate(xs[0], xs[1]); + dest_writer.write_vec(f16_ops, half); + } + + for &x in src_chunks.remainder() { + dest_writer.write_scalar(f16::from_f32(x)); + } + + dest_writer.into_mut_slice() + } +} + +#[cfg(test)] +mod tests { + use rten_simd::ops::BitOps; + use rten_simd::{Isa, SimdOp, f16}; + + use super::{F16ToF32, F32ToF16}; + + /// Return the number of `f16` lanes in a SIMD vector. + fn f16_vec_len() -> usize { + struct F16VecLen {} + impl SimdOp for F16VecLen { + type Output = usize; + fn eval(self, isa: I) -> usize { + isa.f16().len() + } + } + F16VecLen {}.dispatch() + } + + #[test] + fn test_f16_to_f32() { + // Length chosen to exercise all three code paths: the main loop + // (unrolled by two, so it needs at least two whole vectors), the + // single-vector cleanup loop, and the scalar tail. + let len = f16_vec_len() * 3 + 1; + let src: Vec = (0..len) + .map(|i| f16::from_f32(i as f32 * 0.5 - 3.0)) + .collect(); + let expected: Vec = src.iter().map(|x| x.to_f32()).collect(); + + let mut buf = Vec::with_capacity(src.len()); + let actual = F16ToF32::new(&src, buf.spare_capacity_mut()).dispatch(); + + assert_eq!(actual, expected); + } + + #[test] + fn test_f16_to_f32_empty() { + let src: Vec = Vec::new(); + let mut buf: Vec = Vec::new(); + let actual = F16ToF32::new(&src, buf.spare_capacity_mut()).dispatch(); + assert!(actual.is_empty()); + } + + #[test] + fn test_f32_to_f16() { + // Length larger than the max `f16` vector width, and not an exact + // multiple, so we exercise both the vectorized body and the scalar + // tail. + let len = f16_vec_len() + 1; + let src: Vec = (0..len).map(|i| i as f32 * 0.5 - 3.0).collect(); + let expected: Vec = src.iter().map(|&x| f16::from_f32(x)).collect(); + + let mut buf = Vec::with_capacity(src.len()); + let actual = F32ToF16::new(&src, buf.spare_capacity_mut()).dispatch(); + + assert_eq!(actual, expected); + } + + #[test] + fn test_f32_to_f16_empty() { + let src: Vec = Vec::new(); + let mut buf: Vec = Vec::new(); + let actual = F32ToF16::new(&src, buf.spare_capacity_mut()).dispatch(); + assert!(actual.is_empty()); + } + + // Round-trip f32 -> f16 -> f32 for values exactly representable in f16. + #[test] + fn test_roundtrip() { + let len = f16_vec_len() * 2 + 3; + let src: Vec = (0..len).map(|i| i as f32 * 0.25 - 5.0).collect(); + + let mut half_buf = Vec::with_capacity(len); + let half = F32ToF16::new(&src, half_buf.spare_capacity_mut()).dispatch(); + let half: Vec = half.to_vec(); + + let mut back_buf = Vec::with_capacity(len); + let back = F16ToF32::new(&half, back_buf.spare_capacity_mut()).dispatch(); + + let expected: Vec = src.iter().map(|&x| f16::from_f32(x).to_f32()).collect(); + assert_eq!(back, expected); + } +} diff --git a/ocr/vendor/rten-vecmath/src/erf.rs b/ocr/vendor/rten-vecmath/src/erf.rs new file mode 100644 index 0000000..d92253d --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/erf.rs @@ -0,0 +1,189 @@ +//! Error function ("erf") and closely related operations. + +#![allow(clippy::excessive_precision)] + +use std::f32::consts::SQRT_2; + +use rten_simd::ops::{BitOps, FloatOps, NumOps}; +use rten_simd::{Isa, SimdUnaryOp}; + +use crate::exp::ReducedRangeExp; +use crate::tanh::Tanh; + +/// Vectorized error function (erf). +/// +/// The implementation uses an approximation from Abramowitz and Stegun, +/// see . +/// +/// This has a maximum absolute error of 6.631017e-7 when comparing to +/// `libm::erff` as a source of truth. +#[derive(Default)] +pub struct Erf {} + +impl SimdUnaryOp for Erf { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + + let neg_mask = ops.lt(x, ops.zero()); + + let x = ops.abs(x); + + let p = ops.splat(0.3275911); + + // Coefficients for polynomial approximation. + let a0 = ops.splat(0.254829592); + let a1 = ops.splat(-0.284496736); + let a2 = ops.splat(1.421413741); + let a3 = ops.splat(-1.453152027); + let a4 = ops.splat(1.061405429); + + // t = 1. / (1. + p * x); + let t = ops.reciprocal(ops.mul_add(x, p, ops.one())); + let at = ops.poly_eval(t, &[a0, a1, a2, a3, a4]); + + // exp_mx2 = e^(-x^2). `-(x^2)` is always <= 0, so we can use + // reduced-range exp. + let x_m2 = ops.neg(ops.mul(x, x)); + let exp_mx2 = ReducedRangeExp::apply(isa, x_m2); + + // y = 1. - at * exp_mx2; + let y = ops.sub(ops.one(), ops.mul(at, exp_mx2)); + + // Approximation is valid only for x >= 0. For negative values approximation + // can be computed as -erf(-x). + ops.select(ops.neg(y), y, neg_mask) + } +} + +const SQRT_2_RCP: f32 = 1.0 / SQRT_2; + +/// Computes the [GELU](https://onnx.ai/onnx/operators/onnx__Gelu.html) +/// function. +pub struct Gelu {} + +impl SimdUnaryOp for Gelu { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + + let half_x = ops.mul(x, ops.splat(0.5)); + let sqrt_2_rcp = ops.splat(SQRT_2_RCP); + let y = ops.mul(x, sqrt_2_rcp); + let y = ops.add(Erf::apply(isa, y), ops.splat(1.0)); + ops.mul(half_x, y) + } +} + +// sqrt(2 / pi) +const SQRT_2_PI: f32 = 0.7978845608028654; + +/// Approximate Gelu function. +/// +/// See . +pub struct ApproxGelu {} + +impl SimdUnaryOp for ApproxGelu { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + + // 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))) + let half_x = ops.mul(x, ops.splat(0.5)); + let x_cubed = ops.mul(ops.mul(x, x), x); + let y = ops.mul_add(x_cubed, ops.splat(0.044715), x); + let y = ops.mul(y, ops.splat(SQRT_2_PI)); + let y = Tanh::apply(isa, y); + let y = ops.add(y, ops.splat(1.)); + ops.mul(half_x, y) + } +} + +#[cfg(test)] +mod tests { + use rten_simd::SimdUnaryOp; + + use super::{ApproxGelu, Erf, Gelu}; + use crate::testing::{AllF32s, Tolerance, UnaryOpTester, arange, benchmark_op}; + + fn reference_gelu(x: f32) -> f32 { + 0.5 * x * (1. + libm::erff(x / (2.0f32).sqrt())) + } + + fn reference_approx_gelu(x: f32) -> f32 { + let x_cubed = x * x * x; + let approx_erf = ((2.0f32 / std::f32::consts::PI).sqrt() * (x + 0.044715 * x_cubed)).tanh(); + 0.5 * x * (1. + approx_erf) + } + + // Maximum difference between our erf function and `libm::erf` found + // through an exhaustive test. + // + // We use a max-difference test rather than comparing ULPs because the ULP + // difference is large when the input is near zero, but the absolute + // difference is still small enough to be acceptable for the practical uses + // this library is most concerned with. + const MAX_EXPECTED_DIFF: f32 = 6.631017e-7; + + #[test] + fn test_erf() { + let test = UnaryOpTester { + reference: libm::erff, + simd: Erf {}, + // This range is sufficient to cover the regions where the function + // is not saturated and where it is saturated at +/- 1. + range: arange(-6., 6., 0.001), + tolerance: Tolerance::Absolute(MAX_EXPECTED_DIFF), + }; + test.run(); + } + + #[test] + #[ignore] // Ignored by default due to long runtime + fn test_erf_exhaustive() { + let test = UnaryOpTester { + reference: libm::erff, + simd: Erf {}, + range: AllF32s::new(), + tolerance: Tolerance::Absolute(MAX_EXPECTED_DIFF), + }; + test.run_with_progress(); + } + + #[test] + fn test_gelu() { + let test = UnaryOpTester { + reference: reference_gelu, + simd: Gelu {}, + range: arange(-6., 6., 0.001), + tolerance: Tolerance::Absolute(MAX_EXPECTED_DIFF), + }; + test.run(); + } + + #[test] + fn test_approx_gelu() { + let test = UnaryOpTester { + reference: reference_approx_gelu, + simd: ApproxGelu {}, + range: arange(-6., 6., 0.001), + tolerance: Tolerance::Absolute(5e-7), + }; + test.run(); + } + + #[test] + #[ignore] + fn bench_erf() { + benchmark_op( + |xs, ys| { + xs.iter() + .zip(ys.iter_mut()) + .for_each(|(x, y)| *y = libm::erff(*x)) + }, + |xs, ys| { + Erf {}.map(xs, ys); + }, + ); + } +} diff --git a/ocr/vendor/rten-vecmath/src/exp.rs b/ocr/vendor/rten-vecmath/src/exp.rs new file mode 100644 index 0000000..ff10236 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/exp.rs @@ -0,0 +1,460 @@ +//! Vectorized version of the exponential and closely related functions. + +#![allow(clippy::excessive_precision)] + +use rten_simd::ops::{BitOps, FloatOps, IntOps, NumOps}; +use rten_simd::{Isa, Simd, SimdUnaryOp}; + +const INV_LOG2: f32 = std::f32::consts::LOG2_E; // aka. 1 / ln2 +const ROUNDING_MAGIC: f32 = 12582912.; // 0x3 << 22 + +// `log(2)` split into large and small parts for Cody-Waite range reduction. +const LOG2_HI: f32 = -6.93145752e-1; +const LOG2_LO: f32 = -1.42860677e-6; + +// Coefficients of polynomial used to approximate `exp(x)` in `[0, ln2/2]`. +// +// These values are very close to, but not exactly the same as the coefficients +// of the Taylor series around 0 (1, 1/1!, 1/2!, 1/3!, 1/4! ...). +const EXP_POLY_0: f32 = 1.0; +const EXP_POLY_1: f32 = 1.0; +const EXP_POLY_2: f32 = 4.99999851e-1; // ~ 1/2! +const EXP_POLY_3: f32 = 1.66664720e-1; // ~ 1/3! or 1/6 +const EXP_POLY_4: f32 = 4.16695364e-2; // ~ 1/4! or 1/24 +const EXP_POLY_5: f32 = 8.37312452e-3; // ~ 1/5! or 1/120 +const EXP_POLY_6: f32 = 1.37805939e-3; // ~ 1/6! or 1/720 + +/// Vectorized exponential function. +/// +/// This has a maximum error of 1 ULP compared to `f32::exp` in the Rust standard +/// library. +#[derive(Default)] +pub struct Exp {} + +// Implementation based on work by Norbert Juffa in +// https://forums.developer.nvidia.com/t/a-more-accurate-performance-competitive-implementation-of-expf/47528. +// +// See also +// https://justinwillmert.com/articles/2020/numerically-computing-the-exponential-function-with-polynomial-approximations/. +// +// Method outline: +// +// 1. Use the identity `exp(a + b) = exp(a) * exp(b)` to reduce the range for +// which a polynomial approximation needs to be valid: +// +// ```text +// exp(x) = exp(ln2 * k) * exp(r); +// = 2**k * exp(r) +// ``` +// +// Such that `k` is an integer and `|r| <= 1/2 ln 2`. +// +// ```text +// k = rintf(x / ln2) +// r = x - k * ln 2 +// ``` +// +// 2. Compute `exp(r)` using a polynomial approximation. +// +// 3. Compute result as `exp(x) = exp(r) * 2**k`. The reconstruction is split +// into multiple steps to extend the domain. +impl SimdUnaryOp for Exp { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + let int_ops = isa.i32(); + + // Load constants + let inv_log_2 = ops.splat(INV_LOG2); + let rounding_magic = ops.splat(ROUNDING_MAGIC); + let ln2_hi = ops.splat(LOG2_HI); + let ln2_lo = ops.splat(LOG2_LO); + + let p6 = ops.splat(EXP_POLY_6); + let p5 = ops.splat(EXP_POLY_5); + let p4 = ops.splat(EXP_POLY_4); + let p3 = ops.splat(EXP_POLY_3); + let p2 = ops.splat(EXP_POLY_2); + let p1 = ops.splat(EXP_POLY_1); + let p0 = ops.splat(EXP_POLY_0); + + // Compute `k = rintf(x / ln2), r = x - k * ln2`. + let j = ops.mul_add(x, inv_log_2, rounding_magic); + let j = ops.sub(j, rounding_magic); + let r = ops.mul_add(j, ln2_hi, x); + let r = ops.mul_add(j, ln2_lo, r); + let k = ops.to_int_trunc(j); + + // Approximate `exp(r)` on interval [-ln2 / 2, +ln2 / 2] + let mut tmp = p6; + tmp = ops.mul_add(tmp, r, p5); + tmp = ops.mul_add(tmp, r, p4); + tmp = ops.mul_add(tmp, r, p3); + tmp = ops.mul_add(tmp, r, p2); + tmp = ops.mul_add(tmp, r, p1); + let r = ops.mul_add(tmp, r, p0); + + // Reconstruct `exp(x) = 2**k * exp(r`). + // + // Reconstruction is split into steps to extend the input domain of the + // function. The split reconstruction is effectively: + // + // When k > 0: exp(r) * exp2(127) * exp2(k - 127) + // When k <= 0: exp(r) * exp2(-123) * exp2(k + 123) + // + // Where 127 is the exponent bias for f32. + let ia = int_ops.gt(k, int_ops.zero()); + let x7f = int_ops.splat(0x7f000000); + #[allow(overflowing_literals)] + let x83 = int_ops.splat(0x83000000); + let ia = int_ops.select(int_ops.zero(), x83, ia); + let is = int_ops.add(ia, x7f); + + let it = int_ops.shift_left::<23>(k); + let it = int_ops.sub(it, ia); + + let s: I::F32 = is.reinterpret_cast(); + let t: I::F32 = it.reinterpret_cast(); + let r = ops.mul(r, s); + let r = ops.mul(r, t); + + // Handle overflow and underflow when `x.abs() >= 104.` + let overflow_mask = ops.ge(x, ops.splat(104.0)); + let underflow_mask = ops.le(x, ops.splat(-104.0)); + let r = ops.select(ops.splat(f32::INFINITY), r, overflow_mask); + ops.select(ops.zero(), r, underflow_mask) + } +} + +/// Cutoff value chosen such that if `k = round(x / ln2)`, `2**k` is a normal +/// number. +const EXP_LOWER_CUTOFF: f32 = -126.5 * std::f32::consts::LN_2 + 0.01; // ~87.67 + +/// A simplified and faster version of [`Exp`] with a reduced domain and range. +/// +/// 1. The input value must be <= 0 +/// 2. The lower cutoff for which `exp(x)` returns 0 is higher (~87.67 instead of ~104). +#[derive(Default)] +pub struct ReducedRangeExp {} + +impl SimdUnaryOp for ReducedRangeExp { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + let int_ops = isa.i32(); + + // Load constants + let inv_log_2 = ops.splat(INV_LOG2); + let rounding_magic = ops.splat(ROUNDING_MAGIC); + let ln2_hi = ops.splat(LOG2_HI); + let ln2_lo = ops.splat(LOG2_LO); + + let p6 = ops.splat(EXP_POLY_6); + let p5 = ops.splat(EXP_POLY_5); + let p4 = ops.splat(EXP_POLY_4); + let p3 = ops.splat(EXP_POLY_3); + let p2 = ops.splat(EXP_POLY_2); + let p1 = ops.splat(EXP_POLY_1); + let p0 = ops.splat(EXP_POLY_0); + + // Compute `k = rintf(x / ln2), r = x - k * ln2`. + // + // Since x <= 0, also k <= 0. + let j = ops.mul_add(x, inv_log_2, rounding_magic); + let j = ops.sub(j, rounding_magic); + let r = ops.mul_add(j, ln2_hi, x); + let r = ops.mul_add(j, ln2_lo, r); + let k = ops.to_int_trunc(j); + + // Approximate `exp(r)` on interval [-ln2 / 2, +ln2 / 2] + let mut tmp = p6; + tmp = ops.mul_add(tmp, r, p5); + tmp = ops.mul_add(tmp, r, p4); + tmp = ops.mul_add(tmp, r, p3); + tmp = ops.mul_add(tmp, r, p2); + tmp = ops.mul_add(tmp, r, p1); + let r = ops.mul_add(tmp, r, p0); + + // Reconstruct `exp(x) = 2**k * exp(r)`. + // + // This is valid as long as `k >= -126`, so that `2**k` as f32 is a + // normal number. + let exponent_bias = int_ops.splat(127); + let k_pow2 = int_ops.shift_left::<23>(int_ops.add(k, exponent_bias)); + let k_pow2: I::F32 = k_pow2.reinterpret_cast(); + let r = ops.mul(r, k_pow2); + + // Handle underflow. We don't need to handle overflow since x <= 0. + let underflow_mask = ops.lt(x, ops.splat(EXP_LOWER_CUTOFF)); + ops.select(ops.zero(), r, underflow_mask) + } +} + +/// Computes the [sigmoid function][sigmoid], aka. the standard logistic function, `1. / +/// (1. + (-x).exp())`. +/// +/// This has a maximum error of 4 ULPs compared to a reference implementation +/// using `1. / (1. + (-x).exp())`. +/// +/// [sigmoid]: https://en.wikipedia.org/wiki/Logistic_function#Mathematical_properties +#[derive(Default)] +pub struct Sigmoid {} + +impl SimdUnaryOp for Sigmoid { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + + // 1. + exp(-x) + let denom = ops.add(ops.one(), Exp::apply(isa, ops.neg(x))); + ops.reciprocal(denom) + } +} + +/// Vectorized Sigmoid Linear Unit (SiLU) function. +/// +/// This computes `x * sigmoid(x)` for all lanes in `x`. +pub struct Silu {} + +impl SimdUnaryOp for Silu { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + + // 1. + exp(-x) + let denom = ops.add(ops.one(), Exp::apply(isa, ops.neg(x))); + ops.div(x, denom) + } +} + +/// Vectorized Swish function. +/// +/// This computes `x * sigmoid(alpha * x)` for each element. +pub struct Swish { + pub alpha: f32, +} + +impl SimdUnaryOp for Swish { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + + let alpha = ops.splat(self.alpha); + ops.mul(x, Sigmoid::apply(isa, ops.mul(x, alpha))) + } +} + +/// Computes the Exponential Linear Unit function. +/// +/// Computes `if x >= 0 { x } else { alpha * (exp(x) - 1) }`. +pub struct Elu { + pub alpha: f32, +} + +impl SimdUnaryOp for Elu { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + // The ONNX spec and the original paper [1] define Elu in slightly + // different, but equivalent ways: + // + // Original: `f(x) = x if x > 0 else alpha * (exp(x) - 1)` + // ONNX: `f(x) = x if x >= 0 else alpha * (exp(x) - 1)` + // + // [1] https://arxiv.org/pdf/1511.07289 + + let ops = isa.f32(); + let x_pos = ops.ge(x, ops.zero()); + let x_exp = ops.mul( + ops.splat(self.alpha), + ops.sub(Exp::apply(isa, x), ops.splat(1.)), + ); + ops.select(x, x_exp, x_pos) + } +} + +#[cfg(test)] +mod tests { + use rten_simd::SimdUnaryOp; + + use super::{EXP_LOWER_CUTOFF, ReducedRangeExp}; + use crate::testing::{AllF32s, Tolerance, UnaryOpTester, arange, benchmark_op}; + use crate::{Elu, Exp, Sigmoid, Silu, Swish}; + + // Maximum error of `Exp` compared to Rust standard library implementation. + const MAX_EXP_ERROR_ULPS: f32 = 1.0; + + // Maximum error of `Sigmoid` compared to reference implementation below. + const MAX_SIGMOID_ERROR_ULPS: f32 = 4.0; + + fn reference_elu(x: f32, alpha: f32) -> f32 { + if x >= 0. { x } else { alpha * (x.exp() - 1.) } + } + + fn reference_sigmoid(x: f32) -> f32 { + 1. / (1. + (-x).exp()) + } + + fn reference_silu(x: f32) -> f32 { + x * reference_sigmoid(x) + } + + fn reference_swish(x: f32, alpha: f32) -> f32 { + x * reference_sigmoid(alpha * x) + } + + #[test] + fn test_exp_basic() { + // A few simple test cases, including "typical" +/-ve inputs with + // |x| above/below ln2, zero and values below/above min/max cutoffs. + let cases = [-2.0f32, -1., -0.5, 0.1, 0., 0.1, 0.5, 1., 2., -105., 105.]; + + let exp_op = Exp {}; + for case in cases { + let expected = case.exp(); + let actual = exp_op.scalar_eval(case); + let diff = (expected - actual).abs(); + + if actual.is_infinite() || expected.is_infinite() { + assert_eq!(actual, expected); + } else { + // The expected precision is less than 1 ULP, so the diff should + // be exactly zero. + assert_eq!(diff, 0.); + }; + } + } + + #[test] + fn test_exp() { + let test = UnaryOpTester { + reference: f32::exp, + simd: Exp {}, + range: arange(-6., 6., 0.001), + tolerance: Tolerance::Ulp(MAX_EXP_ERROR_ULPS), + }; + test.run(); + } + + #[test] + fn test_reduced_range_exp() { + let test = UnaryOpTester { + reference: f32::exp, + simd: ReducedRangeExp {}, + range: arange(EXP_LOWER_CUTOFF, 0., 0.015), + tolerance: Tolerance::Ulp(MAX_EXP_ERROR_ULPS), + }; + test.run(); + } + + #[test] + fn test_elu() { + let alpha = 0.5; + let test = UnaryOpTester { + reference: |x| reference_elu(x, alpha), + simd: Elu { alpha }, + range: [-2., -1., 0., 1., 2.].into_iter(), + tolerance: Tolerance::Ulp(1.0), + }; + test.run(); + } + + #[test] + #[ignore] // Ignored by default due to long runtime + fn test_exp_exhaustive() { + let test = UnaryOpTester { + reference: f32::exp, + simd: Exp {}, + range: AllF32s::new(), + tolerance: Tolerance::Ulp(MAX_EXP_ERROR_ULPS), + }; + test.run_with_progress(); + } + + #[test] + fn test_sigmoid() { + let test = UnaryOpTester { + reference: reference_sigmoid, + simd: Sigmoid {}, + range: arange(-6., 6., 0.001), + tolerance: Tolerance::Ulp(MAX_SIGMOID_ERROR_ULPS), + }; + test.run(); + } + + #[test] + #[ignore] // Ignored by default due to long runtime + fn test_sigmoid_exhaustive() { + let test = UnaryOpTester { + reference: reference_sigmoid, + simd: Sigmoid {}, + range: AllF32s::new(), + tolerance: Tolerance::Ulp(MAX_SIGMOID_ERROR_ULPS), + }; + test.run_with_progress(); + } + + #[test] + fn test_silu() { + let test = UnaryOpTester { + reference: reference_silu, + simd: Silu {}, + range: arange(-6., 6., 0.001), + tolerance: Tolerance::Ulp(MAX_SIGMOID_ERROR_ULPS), + }; + test.run(); + } + + #[test] + fn test_swish() { + let alpha = 1.7; + let test = UnaryOpTester { + reference: |x| reference_swish(x, alpha), + simd: Swish { alpha }, + range: arange(-6., 6., 0.001), + tolerance: Tolerance::Ulp(MAX_SIGMOID_ERROR_ULPS), + }; + test.run(); + } + + #[test] + #[ignore] + fn bench_elu() { + let alpha = 0.5; + benchmark_op( + |xs, ys| { + xs.iter() + .zip(ys.iter_mut()) + .for_each(|(x, y)| *y = reference_elu(*x, alpha)) + }, + |xs, ys| { + Elu { alpha }.map(xs, ys); + }, + ); + } + + #[test] + #[ignore] + fn bench_exp() { + benchmark_op( + |xs, ys| xs.iter().zip(ys.iter_mut()).for_each(|(x, y)| *y = x.exp()), + |xs, ys| { + Exp {}.map(xs, ys); + }, + ); + } + + #[test] + #[ignore] + fn bench_sigmoid() { + benchmark_op( + |xs, ys| { + xs.iter() + .zip(ys.iter_mut()) + .for_each(|(x, y)| *y = reference_sigmoid(*x)) + }, + |xs, ys| { + Sigmoid {}.map(xs, ys); + }, + ); + } +} diff --git a/ocr/vendor/rten-vecmath/src/extend_init.rs b/ocr/vendor/rten-vecmath/src/extend_init.rs new file mode 100644 index 0000000..e5761f2 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/extend_init.rs @@ -0,0 +1,85 @@ +use std::mem::MaybeUninit; + +/// Extend a buffer by incrementally initializing spare capacity. +/// +/// This is implemented for [`Vec`], where it provides a safe API to +/// initialize the spare capacity returned by +/// [`spare_capacity_mut`](Vec::spare_capacity_mut). +pub trait ExtendInit { + /// Element type in the buffer. + type Elem; + + /// Extend the buffer by initializing a portion of the buffer's spare + /// capacity. + /// + /// The function `f` is passed the uninitialized portion of the buffer and + /// should return the portion that it has initialized. `extend_init` can + /// be called many times, until the entire buffer has been initialized. + /// + /// # Panics + /// + /// Panics if `f` returns a slice that is not a prefix of the slice that + /// was passed to it. + fn extend_init]) -> &[Self::Elem]>(&mut self, f: F); +} + +impl ExtendInit for Vec { + type Elem = T; + + fn extend_init]) -> &[Self::Elem]>(&mut self, f: F) { + let cap = self.spare_capacity_mut(); + let cap_ptr = cap.as_ptr(); + let cap_len = cap.len(); + + let initialized = f(cap); + assert_eq!( + initialized.as_ptr(), + cap_ptr as *const T, + "returned slice must be a prefix of the input" + ); + assert!( + initialized.len() <= cap_len, + "initialized slice length {} is longer than input {}", + initialized.len(), + cap_len + ); + let n_init = initialized.len(); + + // Safety: `n_init` elements from the spare capacity have been initialized. + unsafe { self.set_len(self.len() + n_init) } + } +} + +#[cfg(test)] +mod tests { + use std::mem::MaybeUninit; + + use super::ExtendInit; + + // Implementation of `MaybeUninit::fill` from nightly Rust. + fn fill(xs: &mut [MaybeUninit], value: T) -> &mut [T] { + for x in xs.iter_mut() { + x.write(value); + } + unsafe { std::mem::transmute::<&mut [MaybeUninit], &mut [T]>(xs) } + } + + #[test] + fn test_extend_init() { + let mut vec = Vec::with_capacity(7); + + vec.extend_init(|uninit| { + assert_eq!(uninit.len(), 7); + fill(&mut uninit[..3], 1.) + }); + assert_eq!(vec.len(), 3); + assert_eq!(vec, &[1., 1., 1.]); + + vec.extend_init(|uninit| { + assert_eq!(uninit.len(), 4); + fill(uninit, 2.) + }); + assert_eq!(vec.len(), 7); + assert_eq!(vec, &[1., 1., 1., 2., 2., 2., 2.]); + } +} diff --git a/ocr/vendor/rten-vecmath/src/lib.rs b/ocr/vendor/rten-vecmath/src/lib.rs new file mode 100644 index 0000000..432e4c2 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/lib.rs @@ -0,0 +1,135 @@ +//! SIMD-vectorized implementations of operations used in neural networks. +//! +//! These implementations are used as kernels for operations in the +//! [rten](https://crates.io/crates/rten) crate. +//! +//! ## Constructing and dispatching operations +//! +//! The operations are implemented by structs which implement the SIMD operation +//! traits from [rten-simd](rten_simd). To apply an operation to data, first +//! construct the operation using the struct from this crate, then use a +//! dispatch method from the [`SimdOp`](rten_simd::SimdOp) or +//! [`SimdUnaryOp`](rten_simd::SimdUnaryOp) traits to execute +//! the operation. +//! +//! ## In-place and non in-place operations +//! +//! Some operations support both updating data in place or reading input from +//! one slice and writing to another. For unary operations this is controlled by +//! dispatching with either [`map`](rten_simd::SimdUnaryOp::map) or +//! [`map_mut`](rten_simd::SimdUnaryOp::map_mut). For other operations +//! this is handled by exposing different constructors for the in-place and +//! mutating cases, such as [`Softmax::new`] and [`Softmax::new_mut`]. +//! +//! For operations which use a separate source and destination, the destination +//! is expected to be an uninitialized slice (`[MaybeUninit]`). This allows +//! the caller to control allocation of the buffer and avoid the overhead of +//! initializing elements which the operation will overwrite. The [`ExtendInit`] +//! trait provides a safe API for the common task of filling a new `Vec` with +//! the result of the operation. +//! +//! ## Examples +//! +//! ### Applying a vectorized unary function +//! +//! ``` +//! use std::mem::MaybeUninit; +//! +//! use rten_simd::SimdUnaryOp; +//! use rten_vecmath::Erf; +//! +//! // Apply the error function to each element of `data`. +//! let mut data = [1., 0.5, 2.0]; +//! let erf_op = Erf {}; +//! erf_op.map_mut(&mut data); +//! +//! // Apply the error function to each element of `src`, writing to `dest`. +//! let src = [1., 0.5, 2.0]; +//! let mut dest = [MaybeUninit::uninit(); 3]; +//! erf_op.map(&src, &mut dest); +//! ``` +//! +//! ### Applying softmax in place +//! +//! This example applies the softmax function in-place to a mutable slice. +//! +//! ``` +//! use rten_simd::SimdOp; +//! use rten_vecmath::Softmax; +//! +//! let mut data = [1., 0.5, 2.0]; +//! Softmax::new_mut(&mut data).dispatch(); +//! ``` +//! +//! ### Applying softmax with separate input and output buffers +//! +//! This example reads data from an input and writes to an uninitialized output +//! buffer (`&mut [MaybeUninit]`), obtained from the uninitialized portion +//! of a `Vec`. To update the length of the `Vec` after it is +//! initialized, the helper `ExtendInit` trait is used. +//! +//! ``` +//! use rten_simd::SimdOp; +//! use rten_vecmath::{Softmax, ExtendInit}; +//! +//! let data = [1., 0.5, 2.0]; +//! let mut output = Vec::with_capacity(data.len()); +//! output.extend_init(|output_uninit| { +//! // `output_uninit` is the uninitialized part of `output`, as returned by +//! // `output.spare_capacity_mut()`. +//! // +//! // The `dispatch` call initializes it and returns the initialized slice. +//! Softmax::new(&data, output_uninit).dispatch() +//! }); +//! assert_eq!(output.len(), 3); +//! ``` +//! +//! ### Computing the sum of a list of floats +//! +//! ``` +//! use rten_simd::SimdOp; +//! use rten_vecmath::Sum; +//! +//! let data = [1., 0.5, 2.0]; +//! let sum = Sum::new(&data).dispatch(); +//! ``` + +mod convert; +mod erf; +mod exp; +mod min_max; +mod normalize; +mod quantize; +mod relu; +mod sin_cos; +mod softmax; +mod sum; +mod tanh; + +#[cfg(test)] +mod ulp; + +#[cfg(test)] +mod testing; + +mod extend_init; + +// Unary functions. +pub use erf::{ApproxGelu, Erf, Gelu}; +pub use exp::{Elu, Exp, Sigmoid, Silu, Swish}; +pub use quantize::Quantize; +pub use relu::LeakyRelu; +pub use sin_cos::{Cos, Sin}; +pub use tanh::Tanh; + +// Normalization and reduction functions. +pub use min_max::{MaxNum, MinMax, MinNum}; +pub use normalize::{Normalize, NormalizeOptions}; +pub use softmax::{LogSoftmax, Softmax}; +pub use sum::{Sum, SumAbs, SumExpSub, SumSquare, SumSquareSub}; + +// Conversion functions. +pub use convert::{F16ToF32, F32ToF16}; + +// Utilities +pub use extend_init::ExtendInit; diff --git a/ocr/vendor/rten-vecmath/src/min_max.rs b/ocr/vendor/rten-vecmath/src/min_max.rs new file mode 100644 index 0000000..eecf99c --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/min_max.rs @@ -0,0 +1,204 @@ +use rten_simd::ops::{BitOps, NumOps}; +use rten_simd::{Isa, Simd, SimdIterable, SimdOp}; + +/// Compute the minimum and maximum values in a slice of floats. +/// +/// For an empty slice, returns `(+infinity, -infinity)` — the identity values +/// for min and max on the extended reals. +pub struct MinMax<'a> { + input: &'a [f32], +} + +impl<'a> MinMax<'a> { + pub fn new(input: &'a [f32]) -> Self { + MinMax { input } + } +} + +impl SimdOp for MinMax<'_> { + type Output = (f32, f32); + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + let [vec_min, vec_max] = self.input.simd_iter(ops).fold_n_unroll::<2, 4>( + [ops.splat(f32::INFINITY), ops.splat(f32::NEG_INFINITY)], + #[inline(always)] + |[min, max], x| [ops.min(x, min), ops.max(x, max)], + #[inline(always)] + |[min_a, max_a], [min_b, max_b]| [ops.min(min_a, min_b), ops.max(max_a, max_b)], + ); + let min = vec_min + .to_array() + .as_ref() + .iter() + .fold(f32::INFINITY, |min, x| x.min(min)); + let max = vec_max + .to_array() + .as_ref() + .iter() + .fold(f32::NEG_INFINITY, |max, x| x.max(max)); + (min, max) + } +} + +/// Compute the maximum value in a slice, propagating NaNs. +/// +/// For an empty slice, returns `-infinity` — the identity value for max on +/// the extended reals. +pub struct MaxNum<'a, T> { + input: &'a [T], +} + +impl<'a, T> MaxNum<'a, T> { + pub fn new(input: &'a [T]) -> Self { + MaxNum { input } + } +} + +impl<'a> SimdOp for MaxNum<'a, f32> { + type Output = f32; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + + let max_num = |max, x| { + let not_nan = ops.eq(x, x); + let new_max = ops.max(max, x); + ops.select(new_max, x, not_nan) + }; + + let vec_max = self.input.simd_iter(ops).fold_unroll::<2>( + ops.splat(f32::NEG_INFINITY), + max_num, + max_num, + ); + + vec_max + .to_array() + .as_ref() + .iter() + .copied() + .fold(f32::NEG_INFINITY, |max, x| { + if x.is_nan() { + x + } else if max.is_nan() { + max + } else { + x.max(max) + } + }) + } +} + +/// Compute the minimum value in a slice, propagating NaNs. +/// +/// For an empty slice, returns `+infinity` — the identity value for min on +/// the extended reals. +pub struct MinNum<'a, T> { + input: &'a [T], +} + +impl<'a, T> MinNum<'a, T> { + pub fn new(input: &'a [T]) -> Self { + MinNum { input } + } +} + +impl<'a> SimdOp for MinNum<'a, f32> { + type Output = f32; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + + let min_num = |min, x| { + let not_nan = ops.eq(x, x); + let new_min = ops.min(min, x); + ops.select(new_min, x, not_nan) + }; + + let vec_min = self + .input + .simd_iter(ops) + .fold(ops.splat(f32::INFINITY), min_num); + + vec_min + .to_array() + .as_ref() + .iter() + .copied() + .fold(f32::INFINITY, |min, x| { + if x.is_nan() { + x + } else if min.is_nan() { + min + } else { + x.min(min) + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::{MaxNum, MinMax, MinNum}; + use rten_simd::SimdOp; + + // Chosen to not be a multiple of vector size, so that tail handling is + // exercised. + const LEN: usize = 100; + + fn reference_min_max(xs: &[f32]) -> (f32, f32) { + let min = xs.iter().fold(f32::MAX, |min, x| x.min(min)); + let max = xs.iter().fold(f32::MIN, |max, x| x.max(max)); + (min, max) + } + + #[test] + fn test_min_max() { + let xs: Vec = (0..LEN).map(|i| i as f32 * 0.1).collect(); + let expected = reference_min_max(&xs); + let min_max = MinMax::new(&xs).dispatch(); + assert_eq!(min_max, expected); + } + + #[test] + fn test_max_num() { + let xs: Vec = (0..LEN).map(|i| i as f32 * 0.1).collect(); + let (_, expected_max) = reference_min_max(&xs); + let max = MaxNum::new(&xs).dispatch(); + assert_eq!(max, expected_max); + + let xs = [0.1, 1.0, 0.2, f32::NAN, 0.4, 0.5, 0.6]; + let max = MaxNum::new(&xs).dispatch(); + assert!(max.is_nan()); + } + + #[test] + fn test_min_num() { + let xs: Vec = (0..LEN).map(|i| i as f32 * 0.1).collect(); + let (expected_min, _) = reference_min_max(&xs); + let min = MinNum::new(&xs).dispatch(); + assert_eq!(min, expected_min); + + let xs = [0.1, 1.0, 0.2, f32::NAN, 0.4, 0.5, 0.6]; + let min = MinNum::new(&xs).dispatch(); + assert!(min.is_nan()); + } + + // For an empty slice, min and max return their identity values on the + // extended reals (+/-infinity). This matches the ONNX spec for ReduceMin + // and ReduceMax. + #[test] + fn test_empty() { + let xs: [f32; 0] = []; + assert_eq!(MinNum::new(&xs).dispatch(), f32::INFINITY); + assert_eq!(MaxNum::new(&xs).dispatch(), f32::NEG_INFINITY); + assert_eq!( + MinMax::new(&xs).dispatch(), + (f32::INFINITY, f32::NEG_INFINITY) + ); + } +} diff --git a/ocr/vendor/rten-vecmath/src/normalize.rs b/ocr/vendor/rten-vecmath/src/normalize.rs new file mode 100644 index 0000000..7709efe --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/normalize.rs @@ -0,0 +1,271 @@ +use std::mem::MaybeUninit; + +use rten_simd::functional::simd_map; +use rten_simd::ops::{BitOps, NumOps}; +use rten_simd::span::SrcDest; +use rten_simd::{Isa, SimdIterable, SimdOp}; + +/// Normalize the mean and variance of elements in a slice. +/// +/// This normalizes elements according to the formula: +/// +/// ```text +/// output[i] = (input[i] - pre_scale_bias) * scale * element_scale[i] + bias + element_bias[i] +/// ``` +/// +/// # Panics +/// +/// Dispatching the operation panics if any of the slices have different lengths. +pub struct Normalize<'src, 'dst> { + src_dest: SrcDest<'src, 'dst, f32>, + opts: NormalizeOptions<'src>, +} + +impl<'src, 'dst> Normalize<'src, 'dst> { + /// Create a normalize operation which reads `input` and writes the normalized + /// output to `output`. + pub fn new( + input: &'src [f32], + output: &'dst mut [MaybeUninit], + opts: NormalizeOptions<'src>, + ) -> Self { + Normalize { + src_dest: (input, output).into(), + opts, + } + } + + /// Create a normalize operation which normalizes `input` in-place. + pub fn new_mut(input: &'dst mut [f32], opts: NormalizeOptions<'src>) -> Self + where + 'dst: 'src, + { + Normalize { + src_dest: input.into(), + opts, + } + } +} + +/// Configuration for the [`Normalize`] operation. +pub struct NormalizeOptions<'a> { + /// Bias to subtract before scaling. This is usually the mean of the data. + pub pre_scale_bias: f32, + + /// Constant scale to multiply each element by. This is usually the inverse + /// standard deviation of the data. + pub scale: f32, + + /// Per-element scale to multiply each element by. + pub element_scale: Option<&'a [f32]>, + + /// Constant bias to add after scaling + pub bias: f32, + + /// Per-element bias to add after scaling + pub element_bias: Option<&'a [f32]>, +} + +impl Default for NormalizeOptions<'_> { + fn default() -> Self { + NormalizeOptions { + pre_scale_bias: 0., + scale: 1., + element_scale: None, + bias: 0., + element_bias: None, + } + } +} + +impl<'dst> SimdOp for Normalize<'_, 'dst> { + /// The normalized elements. + type Output = &'dst mut [f32]; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + + let Self { + src_dest, + opts: + NormalizeOptions { + pre_scale_bias, + scale, + element_scale, + bias, + element_bias, + }, + } = self; + + if let Some(scale) = element_scale { + assert_eq!(scale.len(), src_dest.len()); + } + if let Some(bias) = element_bias { + assert_eq!(bias.len(), src_dest.len()); + } + + let one = ops.one(); + let zero = ops.zero(); + let pre_scale_bias_vec = ops.splat(pre_scale_bias); + + match (element_scale, element_bias, scale, bias) { + (None, None, scale, bias) => { + // Per channel scale and bias only. Used for BatchNormalization. + let const_scale_vec = ops.splat(scale); + let const_bias_vec = ops.splat(bias); + + simd_map( + ops, + src_dest, + #[inline(always)] + |x| { + let y = ops.sub(x, pre_scale_bias_vec); + ops.mul_add(y, const_scale_vec, const_bias_vec) + }, + ) + } + (Some(scale), None, const_scale, 0.) => { + // Scale only. Used by eg. LayerNormalization when there is no + // bias and RMS normalization. + let const_scale_vec = ops.splat(const_scale); + let mut scale_iter = scale.simd_iter_pad(ops); + + simd_map( + ops, + src_dest, + #[inline(always)] + |x| { + let scale_vec = scale_iter.next().unwrap(); + let scale_vec = ops.mul(scale_vec, const_scale_vec); + + let y = ops.sub(x, pre_scale_bias_vec); + ops.mul(y, scale_vec) + }, + ) + } + (element_scale, element_bias, const_scale, const_bias) => { + let const_scale_vec = ops.splat(const_scale); + let const_bias_vec = ops.splat(const_bias); + let mut scale_iter = element_scale.map(|s| s.simd_iter_pad(ops)); + let mut bias_iter = element_bias.map(|b| b.simd_iter_pad(ops)); + + simd_map( + ops, + src_dest, + #[inline(always)] + |x| { + let scale_vec = scale_iter.as_mut().and_then(|s| s.next()).unwrap_or(one); + let scale_vec = ops.mul(scale_vec, const_scale_vec); + + let bias_vec = bias_iter.as_mut().and_then(|b| b.next()).unwrap_or(zero); + let bias_vec = ops.add(bias_vec, const_bias_vec); + + let y = ops.sub(x, pre_scale_bias_vec); + ops.mul_add(y, scale_vec, bias_vec) + }, + ) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::{Normalize, NormalizeOptions}; + use rten_simd::SimdOp; + + fn reference_normalize_mut( + data: &mut [f32], + pre_scale_bias: f32, + scale: f32, + element_scale: Option<&[f32]>, + bias: f32, + element_bias: Option<&[f32]>, + ) { + for i in 0..data.len() { + let x_scale = scale * element_scale.map(|es| es[i]).unwrap_or(1.); + let x_bias = bias + element_bias.map(|eb| eb[i]).unwrap_or(0.); + data[i] = (data[i] - pre_scale_bias).mul_add(x_scale, x_bias) + } + } + + #[test] + fn test_normalize_mut() { + let data: Vec<_> = (0..10).map(|i| i as f32 * 0.1).collect(); + let pre_scale_bias = 0.5; + let scale = 0.123; + let element_scale: Vec<_> = (0..data.len()).map(|i| 1.0 + i as f32 * 0.1).collect(); + let bias = 0.3; + let element_bias: Vec<_> = (0..data.len()).map(|i| -0.5 + i as f32 * 0.2).collect(); + + // Per-element scale and bias + let mut expected = data.clone(); + reference_normalize_mut( + &mut expected[..], + pre_scale_bias, + scale, + Some(&element_scale), + bias, + Some(&element_bias), + ); + + let mut actual = data.clone(); + Normalize::new_mut( + &mut actual[..], + NormalizeOptions { + pre_scale_bias, + scale, + element_scale: Some(&element_scale), + bias, + element_bias: Some(&element_bias), + }, + ) + .dispatch(); + assert_eq!(actual, expected); + + // Per-element scale, but no bias + let mut expected = data.clone(); + reference_normalize_mut( + &mut expected[..], + pre_scale_bias, + scale, + Some(&element_scale), + 0., + None, + ); + + let mut actual = data.clone(); + Normalize::new_mut( + &mut actual[..], + NormalizeOptions { + pre_scale_bias, + scale, + element_scale: Some(&element_scale), + bias: 0., + element_bias: None, + }, + ) + .dispatch(); + assert_eq!(actual, expected); + + // Per-channel (ie. constant) scale and bias only + let mut expected = data.clone(); + reference_normalize_mut(&mut expected[..], pre_scale_bias, scale, None, bias, None); + + let mut actual = data.clone(); + Normalize::new_mut( + &mut actual[..], + NormalizeOptions { + pre_scale_bias, + scale, + element_scale: None, + bias, + element_bias: None, + }, + ) + .dispatch(); + + assert_eq!(actual, expected); + } +} diff --git a/ocr/vendor/rten-vecmath/src/quantize.rs b/ocr/vendor/rten-vecmath/src/quantize.rs new file mode 100644 index 0000000..0aa7801 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/quantize.rs @@ -0,0 +1,125 @@ +use std::mem::MaybeUninit; + +use rten_simd::ops::{BitOps, FloatOps, NarrowSaturate, NumOps}; +use rten_simd::{Isa, SimdOp, SliceWriter}; + +/// Quantize a slice of `f32` elements to 8-bit integers using the formula: +/// +/// ```text +/// y = saturate(round(x * inv_scale) + zero_point) +/// ``` +/// +/// Where `round` rounds to the nearest `i32` value with ties to even and +/// `saturate` converts `i32` to the small integer type `To` with saturation. +pub struct Quantize<'s, 'd, To> { + src: &'s [f32], + dest: &'d mut [MaybeUninit], + inv_scale: f32, + zero_point: To, +} + +impl<'s, 'd, To> Quantize<'s, 'd, To> { + pub fn new( + src: &'s [f32], + dest: &'d mut [MaybeUninit], + inv_scale: f32, + zero_point: To, + ) -> Self { + assert_eq!(src.len(), dest.len()); + Quantize { + src, + dest, + inv_scale, + zero_point, + } + } +} + +impl<'d> SimdOp for Quantize<'_, 'd, u8> { + type Output = &'d mut [u8]; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let src_ops = isa.f32(); + let i32_ops = isa.i32(); + + let zp_vec = i32_ops.splat(self.zero_point as i32); + let scale_vec = src_ops.splat(self.inv_scale); + let f32_v_len = src_ops.len(); + + // Generate one vector of u8 elements in each iteration by quantizing + // 4 vectors of f32 elements. + let mut src_chunks = self.src.chunks_exact(f32_v_len * 4); + let mut dest_writer = SliceWriter::new(self.dest); + + for src_chunk in src_chunks.by_ref() { + let src = src_ops.load_many::<4>(src_chunk); + let quant_i32 = src.map(|x| { + let y = src_ops.mul(x, scale_vec); + let y = src_ops.to_int_round(y); + i32_ops.add(y, zp_vec) + }); + let quant_i16_low = i32_ops.narrow_saturate(quant_i32[0], quant_i32[1]); + let quant_i16_high = i32_ops.narrow_saturate(quant_i32[2], quant_i32[3]); + let quant_u8 = isa.i16().narrow_saturate(quant_i16_low, quant_i16_high); + dest_writer.write_vec(isa.u8(), quant_u8); + } + + // Quantize tail elements. + for src in src_chunks.remainder() { + let y = (src * self.inv_scale).round_ties_even() as i32; + let y = (y + self.zero_point as i32).clamp(0, u8::MAX as i32); + dest_writer.write_scalar(y as u8); + } + + dest_writer.into_mut_slice() + } +} + +#[cfg(test)] +mod tests { + use rten_simd::ops::BitOps; + use rten_simd::{Isa, SimdOp}; + + use super::Quantize; + + fn reference_quantize(src: &[f32], inv_scale: f32, zero_point: u8) -> Vec { + src.iter() + .map(|x| { + let tmp = (x * inv_scale).round_ties_even() + zero_point as f32; + tmp as u8 // Saturating cast + }) + .collect() + } + + /// Return number of u8 lanes supported in a SIMD vector. + fn u8_vec_len() -> usize { + struct U8VecLen {} + impl SimdOp for U8VecLen { + type Output = usize; + fn eval(self, isa: I) -> usize { + isa.u8().len() + } + } + U8VecLen {}.dispatch() + } + + #[test] + fn test_quantize() { + let mut rng = fastrand::Rng::with_seed(1234); + + // Larger than max u8 SIMD vector length, and not an exact multiple, so + // we have a tail. + let len = u8_vec_len() + 1; + let src: Vec = std::iter::from_fn(|| Some(rng.f32())).take(len).collect(); + let inv_scale = 5.2; + let zero_point = 10; + let expected = reference_quantize(&src, inv_scale, zero_point); + + let mut buf = Vec::with_capacity(src.len()); + let actual = &mut buf.spare_capacity_mut(); + let actual = Quantize::new(&src, actual, inv_scale, zero_point).dispatch(); + + assert_eq!(actual, expected); + } +} diff --git a/ocr/vendor/rten-vecmath/src/relu.rs b/ocr/vendor/rten-vecmath/src/relu.rs new file mode 100644 index 0000000..557f811 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/relu.rs @@ -0,0 +1,48 @@ +//! Activations related to the ReLU activation. +//! +//! Vanilla ReLU doesn't really need an explicitly vectorized kernel because it +//! is just `x.max(0)` which is easy for compilers to auto-vectorize. Variants +//! such as leaky ReLU however do benefit. + +use rten_simd::ops::{BitOps, NumOps}; +use rten_simd::{Isa, SimdUnaryOp}; + +/// Computes the leaky ReLU activation function. +/// +/// This evaluates `if x < 0. { alpha * x } else { x }`. +pub struct LeakyRelu { + pub alpha: f32, +} + +impl SimdUnaryOp for LeakyRelu { + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + let alpha = ops.splat(self.alpha); + let x_neg = ops.lt(x, ops.zero()); + let x_mul_alpha = ops.mul(x, alpha); + ops.select(x_mul_alpha, x, x_neg) + } +} + +#[cfg(test)] +mod tests { + use crate::testing::{Tolerance, UnaryOpTester}; + + use super::LeakyRelu; + + fn reference_leaky_relu(x: f32, alpha: f32) -> f32 { + if x < 0. { alpha * x } else { x } + } + + #[test] + fn test_leaky_relu() { + let alpha = 0.5; + let test = UnaryOpTester { + reference: |x: f32| reference_leaky_relu(x, alpha), + simd: LeakyRelu { alpha }, + range: [-2., -1., 0., 1., 2.].iter().copied(), + tolerance: Tolerance::Ulp(1.0), + }; + test.run(); + } +} diff --git a/ocr/vendor/rten-vecmath/src/sin_cos.rs b/ocr/vendor/rten-vecmath/src/sin_cos.rs new file mode 100644 index 0000000..0b98742 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/sin_cos.rs @@ -0,0 +1,198 @@ +// We reuse constants exactly from the XNNPACK implementation. +#![allow(clippy::excessive_precision)] +#![allow(clippy::approx_constant)] + +use rten_base::hint::unlikely; +use rten_simd::ops::{BitOps, FloatOps, MaskOps, NumOps}; +use rten_simd::{Isa, Simd, SimdUnaryOp}; + +// Values taken from the XNNPACK vsin implementation that was used as a +// reference. +const PI: f32 = 3.1415927; +const INV_2_PI: f32 = 0.15915494; +const HALF_PI: f32 = 1.5707964; + +// Threshold for large inputs. If abs(x) exceeds this value, the implementation +// falls back to the standard library. +const LARGE_THRESHOLD: f32 = 48_000.0; + +/// Computes the sine function. +/// +/// The implementation has a maximum absolute error of 2.98e-7 (2.5 * f32::EPSILON). +#[derive(Default)] +pub struct Sin(SinCos); + +impl Sin { + pub fn new() -> Self { + Self::default() + } +} + +impl SimdUnaryOp for Sin { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + self.0.eval(isa, x) + } +} + +/// Computes the cosine function. +/// +/// The implementation has a maximum absolute error of 4.17e-7 (3.5 * f32::EPSILON). +#[derive(Default)] +pub struct Cos(SinCos); + +impl Cos { + pub fn new() -> Self { + Self::default() + } +} + +impl SimdUnaryOp for Cos { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + self.0.eval(isa, x) + } +} + +/// Computes the sine or cosine function. +#[derive(Default)] +struct SinCos {} + +impl SimdUnaryOp for SinCos { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + let mask_ops = isa.m32(); + + // For large inputs the vectorized algorithm can produce results + // outside the [-1, 1] range. Fall back to scalar evaluation for such + // inputs to avoid this. + let large = ops.ge(ops.abs(x), ops.splat(LARGE_THRESHOLD)); + if unlikely(mask_ops.any(large)) { + let mut y = x.to_array(); + for i in 0..ops.len() { + y[i] = if COS { y[i].cos() } else { y[i].sin() }; + } + return ops.load(y.as_ref()); + } + + // The implementation here is based on XNNPACK. + // See https://github.com/google/XNNPACK/blob/master/src/f32-vsin/rational-5-4.c.in. + + // Range reduction constants. + let inv_2_pi = ops.splat(INV_2_PI); + let two_pi_hi = ops.splat(6.28125); + let two_pi_lo = ops.splat(1.9353072e-3); + let pi = ops.splat(PI); + + // Rational approximation numerator constants. + let a3 = ops.splat(-1.3314664364e-01); + let a5 = ops.splat(3.2340581529e-03); + let one = ops.splat(1.0); + + // Rational approximation denominator constants. + let b2 = ops.splat(3.3519912511e-02); + let b4 = ops.splat(4.8770775902e-04); + + // Compute range-reduced `x_rr` such that `x_rr ∈ [−π, π]`. + let k = ops.round_ties_even(ops.mul(x, inv_2_pi)); + let x_rr = ops.mul_sub_from(k, two_pi_hi, x); + let mut x_rr = ops.mul_sub_from(k, two_pi_lo, x_rr); + + if COS { + let pi_half = ops.splat(HALF_PI); + x_rr = ops.sub(pi_half, x_rr); + } + + // Further reduce range to [-π/2, π/2]. + let x_rr = ops.min(x_rr, ops.sub(pi, x_rr)); + let x_rr = ops.max(x_rr, ops.sub(ops.neg(pi), x_rr)); + let x_rr = ops.min(x_rr, ops.sub(pi, x_rr)); + + // Approximate sin via a rational approximation. + let x_rr_sq = ops.mul(x_rr, x_rr); + + // Numerator polynomial + let p = ops.mul_add(x_rr_sq, a5, a3); + let p = ops.mul_add(x_rr_sq, p, one); + let p = ops.mul(p, x_rr); + + // Denominator polynomial + let q = ops.mul_add(x_rr_sq, b4, b2); + let q = ops.mul_add(x_rr_sq, q, one); + + ops.div(p, q) + } +} + +#[cfg(test)] +mod tests { + use super::LARGE_THRESHOLD; + use crate::testing::{ARange, Tolerance, UnaryOpTester, arange}; + use crate::{Cos, Sin}; + + // Maximum error of `SinCos` compared to `f32::sin` and `f32::cos` in the + // `SMALL_X` range. + const MAX_ERROR_FOR_SMALL_X: f32 = 2.0 * std::f32::EPSILON; // 2.38e-7 + + // Range of small/medium X values which we expect most inputs will be in. + const SMALL_X: ARange = arange(-10., 10., 0.1f32); + + // Multiples of π are the worst case for range reduction. + fn multiples_of_pi() -> impl Iterator + Clone { + (-5..5).map(|n| (n as f32) * super::PI) + } + + // Generate all float values in the range [min, max]. + fn all_floats_in_range(min: f32, max: f32) -> impl Iterator + Clone { + std::iter::successors(Some(min), |f| Some(f.next_up())).take_while(move |x| *x <= max) + } + + #[test] + fn test_sin() { + let test = UnaryOpTester { + reference: f32::sin, + simd: Sin::new(), + range: SMALL_X.chain(multiples_of_pi()), + tolerance: Tolerance::Absolute(MAX_ERROR_FOR_SMALL_X), + }; + test.run(); + } + + #[test] + #[ignore] // Ignored by default due to long runtime + fn test_sin_exhaustive() { + let test = UnaryOpTester { + reference: f32::sin, + simd: Sin::new(), + range: all_floats_in_range(-LARGE_THRESHOLD, LARGE_THRESHOLD), + tolerance: Tolerance::Absolute(3e-7), + }; + test.run_with_progress(); + } + + #[test] + fn test_cos() { + let test = UnaryOpTester { + reference: f32::cos, + simd: Cos::new(), + range: SMALL_X.chain(multiples_of_pi()), + tolerance: Tolerance::Absolute(MAX_ERROR_FOR_SMALL_X), + }; + test.run(); + } + + #[test] + #[ignore] // Ignored by default due to long runtime + fn test_cos_exhaustive() { + let test = UnaryOpTester { + reference: f32::cos, + simd: Cos::new(), + range: all_floats_in_range(-LARGE_THRESHOLD, LARGE_THRESHOLD), + // Maximum error for cos is larger than for sin because cos has an + // extra subtraction. + tolerance: Tolerance::Absolute(5e-7), + }; + test.run_with_progress(); + } +} diff --git a/ocr/vendor/rten-vecmath/src/softmax.rs b/ocr/vendor/rten-vecmath/src/softmax.rs new file mode 100644 index 0000000..a41aead --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/softmax.rs @@ -0,0 +1,347 @@ +use std::mem::MaybeUninit; + +use rten_simd::functional::{simd_apply, simd_map}; +use rten_simd::ops::{BitOps, FloatOps, NumOps}; +use rten_simd::span::SrcDest; +use rten_simd::{Isa, Simd, SimdIterable, SimdOp, SimdUnaryOp}; + +use crate::exp::ReducedRangeExp; +use crate::sum::fixed_order_sum; + +/// Computes the [softmax][softmax] function over a slice of floats. +/// +/// The implementation uses a three-pass approach for numerical stability. +/// See . +/// and . +/// +/// [softmax]: +pub struct Softmax<'src, 'dst> { + src_dest: SrcDest<'src, 'dst, f32>, + flush_nans_to_zero: bool, +} + +impl<'src, 'dst> Softmax<'src, 'dst> { + /// Construct a softmax operation which reads `input` and writes to to + /// `output`. + #[track_caller] + pub fn new(input: &'src [f32], output: &'dst mut [MaybeUninit]) -> Self { + Softmax { + src_dest: (input, output).into(), + flush_nans_to_zero: false, + } + } + + /// Construct a softmax operation which updates `input` in place. + pub fn new_mut(input: &'dst mut [f32]) -> Self + where + 'dst: 'src, + { + Softmax { + src_dest: input.into(), + flush_nans_to_zero: false, + } + } + + /// Replace NaN values in the output with zeros. + /// + /// This option exists to changing handling of the case where the input + /// values are all negative infinity. In that case the normal output would + /// be NaN. + /// + /// In the context of attention operations which use negative infinity to + /// represent masked token positions, it is preferable to produce zeros as + /// the output if _all_ input positions are masked. See + /// . + pub fn flush_nans_to_zero(mut self, flush: bool) -> Self { + self.flush_nans_to_zero = flush; + self + } +} + +impl<'dst> SimdOp for Softmax<'_, 'dst> { + /// The normalized elements. + type Output = &'dst mut [f32]; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + + let max_val = max(ops, self.src_dest.src()); + + // Compute `y = exp(x - max(x))` and `sum(y)`. + let (dest, exp_sum) = exp_sum_minus_max(isa, self.src_dest, max_val); + + // Divide by `exp_sum`. + let exp_sum = ops.splat(exp_sum); + let inv_exp_sum = ops.reciprocal(exp_sum); + const UNROLL: usize = 2; + let zero = ops.zero(); + + if self.flush_nans_to_zero { + simd_apply::<_, _, _, UNROLL>( + ops, + dest, + #[inline(always)] + |x| { + let y = ops.mul(x, inv_exp_sum); + let not_nan = ops.eq(y, y); + ops.select(y, zero, not_nan) + }, + ); + } else { + simd_apply::<_, _, _, UNROLL>( + ops, + dest, + #[inline(always)] + |x| ops.mul(x, inv_exp_sum), + ); + } + + dest + } +} + +/// Computes the log softmax function over a slice of floats. +/// +/// This is conceptually `log(softmax(x))` which can be rewritten as +/// `log(exp(x) / sum(exp(x)))`. +pub struct LogSoftmax<'src, 'dst> { + src_dest: SrcDest<'src, 'dst, f32>, +} + +impl<'src, 'dst> LogSoftmax<'src, 'dst> { + /// Construct a log softmax operation which reads `input` and writes to + /// `output`. + pub fn new(input: &'src [f32], output: &'dst mut [MaybeUninit]) -> Self { + LogSoftmax { + src_dest: (input, output).into(), + } + } + + /// Construct a log softmax operation which updates `input` in place. + pub fn new_mut(input: &'dst mut [f32]) -> Self + where + 'dst: 'src, + { + LogSoftmax { + src_dest: input.into(), + } + } +} + +impl<'dst> SimdOp for LogSoftmax<'_, 'dst> { + /// The normalized elements. + type Output = &'dst mut [f32]; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + + // The maximum is subtracted from the input for numerical stability. + // Log identities are used to simplify the result: + // + // log(exp(xi - xmax) / sum(exp(x - xmax))) + // = log(exp(xi - xmax)) - log(sum(exp(x - xmax))) + // = xi - xmax - log(sum(exp(x - xmax))) + let max_val = max(ops, self.src_dest.src()); + + // Compute `sum(exp(x - max(x)))`. + let max_vec = ops.splat(max_val); + let exp_sum = fixed_order_sum( + ops, + self.src_dest.src(), + #[inline(always)] + |exp_sum, x| { + // Use faster `exp(x)` since input is known to be <= 0. + let y = ReducedRangeExp::apply(isa, ops.sub(x, max_vec)); + ops.add(exp_sum, y) + }, + ); + + // Compute `y = (x - x_max) - log(exp_sum)`. + // + // We use two separate subtractions inside the loop instead of computing + // `x_max + log(exp_sum)` once and then using a single subtraction. This + // reduces rounding error when `|x_max|` is large and `log(exp_sum)` is + // small. + let log_exp_sum = ops.splat(exp_sum.ln()); + simd_map( + ops, + self.src_dest, + #[inline(always)] + |x| ops.sub(ops.sub(x, max_vec), log_exp_sum), + ) + } +} + +/// Returns the maximum value in `xs`, or [`f32::MIN`] if `xs` is empty. +#[inline(always)] +fn max>(ops: O, xs: &[f32]) -> f32 { + let max_val = xs.simd_iter(ops).fold_unroll::<4>( + ops.splat(f32::MIN), + #[inline(always)] + |max, x| ops.max(max, x), + #[inline(always)] + |max, x| ops.max(max, x), + ); + max_val + .to_array() + .into_iter() + .fold(f32::MIN, |max, x| max.max(x)) +} + +/// Computes `y = exp(x - max(x))` and `sum(y)` in a single pass. +#[inline(always)] +fn exp_sum_minus_max<'dst, I: Isa>( + isa: I, + src_dest: SrcDest<'_, 'dst, f32>, + max_val: f32, +) -> (&'dst mut [f32], f32) { + let ops = isa.f32(); + + let max_val = ops.splat(max_val); + + // *x = (*x - max_val).exp() + let dest = simd_map( + ops, + src_dest, + #[inline(always)] + |x| { + // Use faster `exp(x)` since input is known to be <= 0. + ReducedRangeExp::apply(isa, ops.sub(x, max_val)) + }, + ); + + // Ethos fork: sum in the fixed order, so the bits do not depend on the vector width. + let exp_sum = fixed_order_sum(ops, dest, |sum, y| ops.add(sum, y)); + + (dest, exp_sum) +} + +#[cfg(test)] +mod tests { + use rten_simd::SimdOp; + + use super::{LogSoftmax, Softmax}; + use crate::testing::{AsUninit, benchmark_op, check_f32s_are_equal_ulps, triples}; + + fn reference_log_softmax(xs: &[f32], ys: &mut [f32]) { + let max = xs.iter().copied().fold(f32::MIN, |max, x| max.max(x)); + let log_exp_sum = xs + .iter() + .fold(0., |exp_sum: f32, x| exp_sum + (x - max).exp()) + .ln(); + for (x, y) in xs.iter().zip(ys.iter_mut()) { + *y = (*x - max) - log_exp_sum; + } + } + + fn reference_softmax(xs: &[f32], ys: &mut [f32]) { + let max = xs.iter().copied().fold(f32::MIN, |max, x| max.max(x)); + + let mut exp_sum = 0.; + for (x, y) in xs.iter().zip(ys.iter_mut()) { + *y = (*x - max).exp(); + exp_sum += *y; + } + + for el in ys.iter_mut() { + *el /= exp_sum; + } + } + + #[test] + fn test_softmax() { + // Test against reference values. + let input = vec![0.1634, 0.8647, 0.6401, 0.8265, 0.0560, 0.2304]; + let expected = &([ + 0.11715934, 0.23623686, 0.18871443, 0.2273828, 0.10522857, 0.12527795, + ]); + let mut actual = vec![0.; input.len()]; + + Softmax::new(&input, actual.as_mut_slice().as_uninit()).dispatch(); + check_f32s_are_equal_ulps(triples(&input, &actual, expected), 1. /* max ULPs */); + + // Test against reference implementation for various lengths. + for len in 1..20 { + let input: Vec = (0..len).map(|x| x as f32 + 0.1).collect(); + let mut expected = vec![0.; input.len()]; + reference_softmax(&input, &mut expected); + + let mut actual = vec![0.; input.len()]; + Softmax::new(&input, actual.as_mut_slice().as_uninit()).dispatch(); + + check_f32s_are_equal_ulps(triples(&input, &actual, &expected), 3. /* max ULPs */); + } + } + + #[test] + fn test_softmax_flush_nans_to_zero() { + let mut input = [f32::NEG_INFINITY; 3]; + Softmax::new_mut(&mut input).dispatch(); + assert!(input.iter().all(|x| x.is_nan())); + + let mut input = [f32::NEG_INFINITY; 3]; + Softmax::new_mut(&mut input) + .flush_nans_to_zero(true) + .dispatch(); + assert_eq!(input, [0.; 3]); + } + + #[test] + fn test_log_softmax() { + // Test against reference implementation for various lengths. + for len in 1..20 { + let input: Vec = (0..len).map(|x| x as f32 + 0.1).collect(); + let mut expected = vec![0.; input.len()]; + reference_log_softmax(&input, &mut expected); + + let mut actual = vec![0.; input.len()]; + LogSoftmax::new(&input, actual.as_mut_slice().as_uninit()).dispatch(); + + check_f32s_are_equal_ulps(triples(&input, &actual, &expected), 3. /* max ULPs */); + } + } + + #[test] + fn test_log_softmax_in_place() { + let input: Vec = (0..32).map(|x| x as f32 * 0.5 - 8.).collect(); + let mut expected = vec![0.; input.len()]; + reference_log_softmax(&input, &mut expected); + + let mut actual = input.clone(); + LogSoftmax::new_mut(&mut actual).dispatch(); + + check_f32s_are_equal_ulps(triples(&input, &actual, &expected), 3. /* max ULPs */); + } + + #[test] + fn test_log_softmax_sums_to_one() { + // `exp(log_softmax(x))` is a probability distribution, so it must sum + // to one regardless of the input scale. + for scale in [1e-3, 1., 10., 100.] { + let input: Vec = (0..64).map(|x| (x as f32 - 32.) * scale).collect(); + let mut actual = input.clone(); + LogSoftmax::new_mut(&mut actual).dispatch(); + + let sum: f32 = actual.iter().map(|x| x.exp()).sum(); + assert!((sum - 1.).abs() < 1e-4, "scale {scale} sum {sum}"); + } + } + + #[test] + #[ignore] + fn bench_softmax() { + benchmark_op(reference_softmax, |src, dest| { + Softmax::new(src, dest).dispatch(); + }); + } + + #[test] + #[ignore] + fn bench_log_softmax() { + benchmark_op(reference_log_softmax, |src, dest| { + LogSoftmax::new(src, dest).dispatch(); + }); + } +} diff --git a/ocr/vendor/rten-vecmath/src/sum.rs b/ocr/vendor/rten-vecmath/src/sum.rs new file mode 100644 index 0000000..64d3dde --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/sum.rs @@ -0,0 +1,249 @@ +use rten_simd::ops::{BitOps, FloatOps, NumOps}; +use rten_simd::{Isa, Simd, SimdOp, SimdUnaryOp}; + +use crate::Exp; + +/// Partial sums every reduction keeps, whatever the CPU's vector width. +/// +/// Ethos fork: element `i` adds into partial `i % SUM_LANES`, in input order, and the +/// partials then add in one fixed tree. Every instruction set's lane count (4, 8 or 16) +/// divides `SUM_LANES`, so the additions, and the result's bits, are the same on every CPU. +/// Upstream folds per hardware lane, so its bits change with the vector width. +pub const SUM_LANES: usize = 16; + +/// Folds `xs` into [`SUM_LANES`] partials with `fold(partial, x)`, then adds the partials +/// in a fixed tree. +#[inline(always)] +pub(crate) fn fixed_order_sum>( + ops: O, + xs: &[f32], + mut fold: impl FnMut(O::Simd, O::Simd) -> O::Simd, +) -> f32 { + let n = ops.len(); + assert!(SUM_LANES % n == 0 && SUM_LANES / n <= 4); + let regs = SUM_LANES / n; + let mut acc = [ops.zero(); 4]; + let mut rest = xs; + while let Some((block, tail)) = rest.split_at_checked(SUM_LANES) { + for (k, a) in acc.iter_mut().enumerate().take(regs) { + // Safety: `block` holds `SUM_LANES = regs * n` elements. + let x = unsafe { ops.load_ptr(block.as_ptr().add(k * n)) }; + *a = fold(*a, x); + } + rest = tail; + } + // Element `j` of the tail still adds into partial `j`; padding lanes keep their partial. + for (k, a) in acc.iter_mut().enumerate().take(regs) { + if k * n >= rest.len() { + break; + } + let (x, mask) = ops.load_pad(&rest[k * n..]); + *a = ops.select(fold(*a, x), *a, mask); + } + let mut parts = [0f32; SUM_LANES]; + for (k, a) in acc.iter().enumerate().take(regs) { + parts[k * n..(k + 1) * n].copy_from_slice(a.to_array().as_ref()); + } + let mut width = SUM_LANES / 2; + while width > 0 { + for i in 0..width { + parts[i] += parts[i + width]; + } + width /= 2; + } + parts[0] +} + +/// Computes the sum of a sequence of numbers. +/// +/// This is more efficient than `slice.iter().sum()` as it computes multiple +/// partial sums in parallel using SIMD and then sums across the SIMD lanes at +/// the end. This will produce very slightly different results because the +/// additions are happening in a different order. +pub struct Sum<'a> { + input: &'a [f32], +} + +impl<'a> Sum<'a> { + pub fn new(input: &'a [f32]) -> Self { + Sum { input } + } +} + +impl SimdOp for Sum<'_> { + type Output = f32; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + fixed_order_sum(ops, self.input, |sum, x| ops.add(sum, x)) + } +} + +/// Computes the sum of squares of a sequence of numbers. +/// +/// This is conceptually equivalent to `slice.iter().map(|&x| x * x).sum()` but +/// more efficient as it computes multiple partial sums in parallel using SIMD +/// and then sums across the SIMD lanes at the end. This will produce very +/// slightly different results because the additions are happening in a +/// different order. +pub struct SumSquare<'a> { + input: &'a [f32], +} + +impl<'a> SumSquare<'a> { + pub fn new(input: &'a [f32]) -> Self { + SumSquare { input } + } +} + +impl SimdOp for SumSquare<'_> { + type Output = f32; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + fixed_order_sum(ops, self.input, |sum, x| ops.mul_add(x, x, sum)) + } +} + +/// Computes the sum of absolute values of a sequence of numbers. +pub struct SumAbs<'a> { + input: &'a [f32], +} + +impl<'a> SumAbs<'a> { + pub fn new(input: &'a [f32]) -> Self { + SumAbs { input } + } +} + +impl SimdOp for SumAbs<'_> { + type Output = f32; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + fixed_order_sum(ops, self.input, |sum, x| ops.add(sum, ops.abs(x))) + } +} + +/// Compute the sum of squares of input with a bias subtracted. +/// +/// This is a variant of [`SumSquare`] which subtracts a constant value from each +/// element before squaring it. A typical use case is to compute the variance of +/// a sequence, which is defined as `mean((X - x_mean)^2)`. +pub struct SumSquareSub<'a> { + input: &'a [f32], + offset: f32, +} + +impl<'a> SumSquareSub<'a> { + pub fn new(input: &'a [f32], offset: f32) -> Self { + SumSquareSub { input, offset } + } +} + +impl SimdOp for SumSquareSub<'_> { + type Output = f32; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + let offset_vec = ops.splat(self.offset); + + fixed_order_sum(ops, self.input, |sum, x| { + let x_offset = ops.sub(x, offset_vec); + ops.mul_add(x_offset, x_offset, sum) + }) + } +} + +/// Compute the sum of `exp(x - offset)` over the input. +pub struct SumExpSub<'a> { + input: &'a [f32], + offset: f32, +} + +impl<'a> SumExpSub<'a> { + pub fn new(input: &'a [f32], offset: f32) -> Self { + SumExpSub { input, offset } + } +} + +impl SimdOp for SumExpSub<'_> { + type Output = f32; + + #[inline(always)] + fn eval(self, isa: I) -> Self::Output { + let ops = isa.f32(); + let offset_vec = ops.splat(self.offset); + + fixed_order_sum(ops, self.input, |sum, x| { + let exp = Exp::apply(isa, ops.sub(x, offset_vec)); + ops.add(sum, exp) + }) + } +} + +#[cfg(test)] +mod tests { + use crate::ulp::assert_ulp_diff_le; + + use super::{Sum, SumAbs, SumExpSub, SumSquare, SumSquareSub}; + use rten_simd::SimdOp; + + // Chosen to not be a multiple of vector size, so that tail handling is + // exercised. + const LEN: usize = 100; + + #[test] + fn test_sum() { + let xs: Vec = (0..LEN).map(|i| i as f32 * 0.1).collect(); + let expected_sum: f64 = xs.iter().map(|x| *x as f64).sum(); + let sum = Sum::new(&xs).dispatch(); + assert_ulp_diff_le!(sum, expected_sum as f32, 1.0); + } + + #[test] + fn test_sum_square() { + let xs: Vec = (0..LEN).map(|i| i as f32 * 0.1).collect(); + let expected_sum: f64 = xs.iter().copied().map(|x| x as f64 * x as f64).sum(); + let sum = SumSquare::new(&xs).dispatch(); + assert_ulp_diff_le!(sum, expected_sum as f32, 2.0); + } + + #[test] + fn test_sum_abs() { + let xs: Vec = (0..LEN).map(|i| (i as f32 * 0.1) - 5.0).collect(); + let expected_sum: f64 = xs.iter().map(|x| (*x as f64).abs()).sum(); + let sum = SumAbs::new(&xs).dispatch(); + assert_ulp_diff_le!(sum, expected_sum as f32, 2.0); + } + + #[test] + fn test_sum_square_sub() { + let xs: Vec = (0..LEN).map(|i| i as f32 * 0.1).collect(); + let mean = xs.iter().sum::() / xs.len() as f32; + let expected_sum: f64 = xs + .iter() + .copied() + .map(|x| (x as f64 - mean as f64) * (x as f64 - mean as f64)) + .sum(); + let sum = SumSquareSub::new(&xs, mean).dispatch(); + assert_ulp_diff_le!(sum, expected_sum as f32, 2.0); + } + + #[test] + fn test_sum_exp_sub() { + let xs: Vec = (0..LEN).map(|i| i as f32 * 0.1).collect(); + let max = xs.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let expected_sum: f64 = xs + .iter() + .copied() + .map(|x| (x as f64 - max as f64).exp()) + .sum(); + let sum = SumExpSub::new(&xs, max).dispatch(); + assert_ulp_diff_le!(sum, expected_sum as f32, 2.0); + } +} diff --git a/ocr/vendor/rten-vecmath/src/tanh.rs b/ocr/vendor/rten-vecmath/src/tanh.rs new file mode 100644 index 0000000..aad9366 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/tanh.rs @@ -0,0 +1,115 @@ +#![allow(clippy::excessive_precision)] + +use rten_simd::ops::{BitOps, FloatOps, NumOps}; +use rten_simd::{Isa, SimdUnaryOp}; + +use crate::Exp; + +/// Computes the hyperbolic tangent function. +#[derive(Default)] +pub struct Tanh {} + +impl SimdUnaryOp for Tanh { + #[inline(always)] + fn eval(&self, isa: I, x: I::F32) -> I::F32 { + let ops = isa.f32(); + + let x_negative = ops.le(x, ops.zero()); + let abs_x = ops.abs(x); + + // Cutoff beyond which `f32::tanh(x)` saturates at +/- 1.0. + let x_cutoff = ops.ge(abs_x, ops.splat(9.02)); + + // tanh(x) ~ x when |x| is very small. + let x_tiny = ops.le(abs_x, ops.splat(0.0004)); + + // Threshold below which `tanh(x) = (exp(2x) - 1) / (exp(2x) + 1)` method + // produces errors >= 2 ULP. + let x_small = ops.le(abs_x, ops.splat(0.55)); + + // For small x, use polynomial approximation. Computed using Sollya with + // `P = fpminimax(f, [|1, 3, 5, 7, 9|], [|SG...|], [0, 0.6])`. + const P1: f32 = 0.999999940395355224609375; + const P3: f32 = -0.33332359790802001953125; + const P5: f32 = 0.13310669362545013427734375; + const P7: f32 = -5.21197654306888580322265625e-2; + const P9: f32 = 1.5497927553951740264892578125e-2; + + let p1 = ops.splat(P1); + let p3 = ops.splat(P3); + let p5 = ops.splat(P5); + let p7 = ops.splat(P7); + let p9 = ops.splat(P9); + + let x_sqr = ops.mul(x, x); + let y_small = ops.mul_add(p9, x_sqr, p7); + let y_small = ops.mul_add(y_small, x_sqr, p5); + let y_small = ops.mul_add(y_small, x_sqr, p3); + let y_small = ops.mul_add(y_small, x_sqr, p1); + let y_small = ops.mul(y_small, abs_x); + + // For medium x, compute `tanh(x) = (exp(2x) - 1) / (exp(2x) + 1)`. + let x2 = ops.mul(abs_x, ops.splat(2.0)); + let exp_2x = Exp::apply(isa, x2); + let exp_2x_m1 = ops.sub(exp_2x, ops.one()); + let exp_2x_p1 = ops.add(exp_2x, ops.one()); + let y_medium = ops.div(exp_2x_m1, exp_2x_p1); + + // Select output to use depending on |x|. + let y = ops.select(ops.one(), y_medium, x_cutoff); + let y = ops.select(y_small, y, x_small); + let y = ops.select(abs_x, y, x_tiny); + + // Flip sign if input was negative. + ops.select(ops.neg(y), y, x_negative) + } +} + +#[cfg(test)] +mod tests { + use rten_simd::SimdUnaryOp; + + use crate::Tanh; + use crate::testing::{AllF32s, Tolerance, UnaryOpTester, arange, benchmark_op}; + + // Maximum error of `Tanh` compared to `f32::tanh`. + const MAX_TANH_ERROR_ULPS: f32 = 3.0; + + #[test] + #[ignore] // Ignored by default due to long runtime + fn test_tanh_exhaustive() { + let test = UnaryOpTester { + reference: f32::tanh, + simd: Tanh {}, + range: AllF32s::new(), + tolerance: Tolerance::Ulp(MAX_TANH_ERROR_ULPS), + }; + test.run_with_progress(); + } + + #[test] + fn test_tanh() { + let test = UnaryOpTester { + reference: f32::tanh, + simd: Tanh {}, + range: arange(-8., 8., 0.001), + tolerance: Tolerance::Ulp(MAX_TANH_ERROR_ULPS), + }; + test.run(); + } + + #[test] + #[ignore] + fn bench_tanh() { + benchmark_op( + |xs, ys| { + xs.iter() + .zip(ys.iter_mut()) + .for_each(|(x, y)| *y = x.tanh()) + }, + |xs, ys| { + Tanh {}.map(xs, ys); + }, + ); + } +} diff --git a/ocr/vendor/rten-vecmath/src/testing.rs b/ocr/vendor/rten-vecmath/src/testing.rs new file mode 100644 index 0000000..d70b42a --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/testing.rs @@ -0,0 +1,329 @@ +use std::mem::MaybeUninit; + +use std::io::prelude::*; +use std::iter::repeat_with; +use std::time::Instant; + +use crate::ulp::Ulp; +use rten_simd::SimdUnaryOp; + +/// Trait for converting containers of initialized values into uninitialized +/// ones. +pub trait AsUninit { + type Output; + + /// Convert all elements from `T` to `MaybeUninit` + fn as_uninit(self) -> Self::Output; +} + +impl<'a, T: Copy> AsUninit for &'a mut [T] { + type Output = &'a mut [MaybeUninit]; + + fn as_uninit(self) -> Self::Output { + unsafe { std::mem::transmute(self) } + } +} + +/// Iterator over all possible f32 values. +#[derive(Clone)] +pub struct AllF32s { + next: Option, +} + +impl AllF32s { + pub fn new() -> AllF32s { + AllF32s { next: Some(0) } + } +} + +impl Iterator for AllF32s { + type Item = f32; + + fn next(&mut self) -> Option { + let next = self.next?; + self.next = next.checked_add(1); + Some(f32::from_bits(next)) + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self + .next + .map(|next| (u32::MAX - next) as usize + 1) + .unwrap_or(0); + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for AllF32s {} + +/// Iterator over an arithmetic range. See [`arange`]. +#[derive(Copy, Clone, Debug)] +pub struct ARange> { + next: T, + end: T, + step: T, +} + +/// Return an iterator over an arithmetic range `[start, end)` in steps of `step`. +/// +/// Iteration stops if the next value in the series cannot be compared against +/// the end value (ie. if `next.partial_cmp(end)` yields `None`). +pub const fn arange>( + start: T, + end: T, + step: T, +) -> ARange { + ARange { + next: start, + end, + step, + } +} + +impl> Iterator for ARange { + type Item = T; + + fn next(&mut self) -> Option { + use std::cmp::Ordering; + let next = self.next; + match next.partial_cmp(&self.end) { + Some(Ordering::Less) => { + self.next = self.next + self.step; + Some(next) + } + _ => None, + } + } +} + +/// Take three slices of equal length and return an iterator over triples of +/// corresponding elements. +/// +/// This is a helper for use with other testing methods. +pub fn triples<'a, T: Copy>( + input: &'a [T], + actual: &'a [T], + expected: &'a [T], +) -> impl Iterator + 'a { + assert!(input.len() == actual.len() && actual.len() == expected.len()); + input + .iter() + .zip(actual.iter().zip(expected.iter())) + .map(|(x, (actual, expected))| (*x, *actual, *expected)) +} + +/// Compare results of an operation on floats against expected results. +/// +/// `results` is an iterator yielding tuples of `(input, actual, expected)` +/// values. +/// +/// `max_diff` specifies the maximum allowed difference between `actual` and +/// `expected`. +pub fn check_f32s_are_equal_atol>(results: I, max_diff: f32) { + for (x, actual, expected) in results { + if actual.is_nan() && expected.is_nan() { + continue; + } + let diff = (actual - expected).abs(); + assert!( + diff <= max_diff, + "diff {} exceeds expected {} at x = {}. actual = {}, expected = {}", + diff, + max_diff, + x, + actual, + expected + ); + } +} + +/// Compare results of an operation on floats against expected results. +/// +/// `results` is an iterator yielding tuples of `(input, actual, expected)` +/// values. +/// +/// `ulp_threshold` specifies the maximum allowed difference between +/// `actual` and `expected` in ULPs. +pub fn check_f32s_are_equal_ulps>( + results: I, + ulp_threshold: f32, +) { + let mut max_diff_ulps = 0.0f32; + let mut max_diff_x = 0.0f32; + let mut max_diff_actual = 0.0f32; + let mut max_diff_expected = 0.0f32; + + for (x, actual, expected) in results { + if actual == expected { + // Fast path for expected common case where results are exactly + // equal. + continue; + } + + assert_eq!( + actual.is_nan(), + expected.is_nan(), + "NaN mismatch at {x}. Actual {actual} Expected {expected}", + ); + assert_eq!( + expected.is_infinite(), + actual.is_infinite(), + "Infinite mismatch at {x}. Actual {actual} Expected {expected}" + ); + + if !expected.is_infinite() && !expected.is_nan() { + let diff = (actual - expected).abs(); + let diff_ulps = diff / expected.ulp(); + if diff_ulps > max_diff_ulps { + max_diff_ulps = max_diff_ulps.max(diff_ulps); + max_diff_x = x; + max_diff_actual = actual; + max_diff_expected = expected; + } + } + } + assert!( + max_diff_ulps <= ulp_threshold, + "max diff against reference is {} ULPs for x = {}, actual = {}, expected = {}, ULP = {}. Above ULP threshold {}", + max_diff_ulps, + max_diff_x, + max_diff_actual, + max_diff_expected, + max_diff_expected.ulp(), + ulp_threshold + ); +} + +/// Benchmark a vectorized implementation of a function against a reference +/// implementation. +/// +/// `reference` and `vectorized` are functions which take an input slice of +/// elements as a first argument and write the results of applying the operation +/// to the second argument. +pub fn benchmark_op])>( + reference: RF, + vectorized: VF, +) { + // Generate values in [-0.5, 0.5]. + // + // This range is chosen because a) it is a common range for inputs to NN + // functions and b) some of the vectorized functions are activations which + // change behavior above/below zero. If we used the default `fastrand::f32` + // range of [0, 1] this would give misleading results for reference + // implementations as they would always take the same branch. + let input: Vec<_> = repeat_with(|| -0.5 + fastrand::f32()) + .take(1_000_000) + .collect(); + + let mut output = vec![0.; input.len()]; + let iters = 100; + + let reference_start = Instant::now(); + for _ in 0..iters { + reference(&input, &mut output); + } + let reference_elapsed = reference_start.elapsed().as_micros(); + + let vecmath_vec_start = Instant::now(); + for _ in 0..iters { + vectorized(&input, output.as_mut_slice().as_uninit()); + } + let vecmath_vec_elapsed = vecmath_vec_start.elapsed().as_micros(); + + let ratio = reference_elapsed as f32 / vecmath_vec_elapsed as f32; + + println!( + "reference {} us vectorized {} us. reference / vectorized ratio {:.3}", + reference_elapsed, vecmath_vec_elapsed, ratio + ); +} + +pub enum Tolerance { + /// Number of Units of Least Precision. + Ulp(f32), + /// Maximum absolute difference. + Absolute(f32), +} + +/// Tests a vectorized implementation of a unary function against a reference +/// implementation. +pub struct UnaryOpTester f32, S: SimdUnaryOp, R: Iterator + Clone> { + /// Reference implementation of the operation. + pub reference: F, + + /// Vectorized implementation of the operation. + pub simd: S, + + /// Iterator yielding values to test. + pub range: R, + + /// Tolerance for comparisons between reference and actual results. + pub tolerance: Tolerance, +} + +impl f32, S: SimdUnaryOp, R: Iterator + Clone> + UnaryOpTester +{ + /// Run an evaluation of a vectorized operation against a reference and + /// panic if the difference exceeds the tolerance for any input. + pub fn run(&self) { + self.run_impl(false) + } + + /// Variant of [`run`](Self::run) which reports progress as it runs. + /// + /// This is intended for use with ranges like [`AllF32s`] with large numbers + /// of elements. + pub fn run_with_progress(&self) { + self.run_impl(true) + } + + fn run_impl(&self, with_progress: bool) { + let mut total = 0; + let mut remaining = self.range.clone(); + + // Progress reporting uses the upper bound, so it can report a lower + // value than the actual progress. For example a range with a filter + // applied will jump when values are skipped. + let (_lower_bound, upper_bound) = remaining.size_hint(); + + let chunk_size = 16 * 1024; + let mut input = Vec::with_capacity(chunk_size); + let mut actual = Vec::with_capacity(chunk_size); + let mut expected = Vec::with_capacity(chunk_size); + + if with_progress && upper_bound.is_none() { + println!("Testing: ..."); + } + + loop { + if with_progress && upper_bound.is_some() { + let progress = total as f32 / upper_bound.unwrap() as f32; + print!("\rTesting: {:.2}%", progress * 100.); + let _ = std::io::stdout().flush(); + } + + input.clear(); + actual.clear(); + expected.clear(); + + input.extend((&mut remaining).take(chunk_size)); + if input.is_empty() { + break; + } + + total += input.len(); + + expected.extend(input.iter().copied().map(&self.reference)); + actual.extend(&input); + self.simd.map_mut(&mut actual); + + let results = triples(&input, &actual, &expected); + match self.tolerance { + Tolerance::Ulp(max_ulps) => check_f32s_are_equal_ulps(results, max_ulps), + Tolerance::Absolute(max_error) => check_f32s_are_equal_atol(results, max_error), + } + } + assert!(total > 0, "input range was empty"); + } +} diff --git a/ocr/vendor/rten-vecmath/src/ulp.rs b/ocr/vendor/rten-vecmath/src/ulp.rs new file mode 100644 index 0000000..f364ac2 --- /dev/null +++ b/ocr/vendor/rten-vecmath/src/ulp.rs @@ -0,0 +1,85 @@ +/// Trait for obtaining the size of the Unit in the Last Place (ULP) for floats. +pub trait Ulp { + /// Return the size of the ULP for a given value. + fn ulp(self) -> Self; + + /// Return the difference between this value and `other` in units of + /// `other.ulp()`. + fn diff_ulps(self, other: Self) -> Self; +} + +impl Ulp for f32 { + /// Return the size of the Unit in the Last Place for a given value. + /// + /// Handling of special cases (NaN, infinity, zero, min/max) follows `Math.ulp` + /// in Java [1]. + /// + /// [1] Java SE 21, `java.lang.Math.ulp(float)`. + fn ulp(self: f32) -> f32 { + if self.is_nan() { + self + } else if self.is_infinite() { + f32::INFINITY + } else if self == 0. { + f32::MIN + } else if self == f32::MIN || self == f32::MAX { + f32::from_bits((127 + 104) << 23) // 2^104 + } else { + let bits = self.to_bits(); + let next_up = f32::from_bits(bits + 1); + (next_up - self).abs() + } + } + + fn diff_ulps(self: f32, other: f32) -> f32 { + (self - other).abs() / other.ulp() + } +} + +/// Assert that the difference between two values is less than or equal to a +/// given number of [ULPs](Ulp). +macro_rules! assert_ulp_diff_le { + ($actual:expr, $expected:expr, $max_diff:expr) => {{ + use crate::ulp::Ulp; + + let ulp_diff = ($actual).diff_ulps($expected); + assert!( + ulp_diff <= $max_diff, + "difference between {} and {} is {} ULPs which exceeds {}", + $actual, + $expected, + ulp_diff, + $max_diff + ); + }}; +} + +pub(crate) use assert_ulp_diff_le; + +#[cfg(test)] +mod tests { + use super::Ulp; + + #[test] + fn test_f32_ulp() { + assert_eq!((1.0f32).ulp(), f32::EPSILON); + + // Special cases. See the Java `Math.ulp` docs. + assert!(f32::NAN.ulp().is_nan()); + assert_eq!(f32::INFINITY.ulp(), f32::INFINITY); + assert_eq!(f32::NEG_INFINITY.ulp(), f32::INFINITY); + assert_eq!((0.0f32).ulp(), f32::MIN); + assert_eq!((-0.0f32).ulp(), f32::MIN); + assert_eq!(f32::MAX.ulp(), (104f32).exp2()); + assert_eq!(f32::MIN.ulp(), (104f32).exp2()); + } + + #[test] + fn test_f32_diff_ulps() { + let x = 1.0f32; + let y = 1.001f32; + + let diff_bits = y.to_bits() - x.to_bits(); + assert_eq!(y.diff_ulps(x), diff_bits as f32); + } +}