diff --git a/.github/actions/install-test/action.yml b/.github/actions/install-test/action.yml new file mode 100644 index 0000000..063022d --- /dev/null +++ b/.github/actions/install-test/action.yml @@ -0,0 +1,47 @@ +name: Install the wheel and run the tests +description: > + Installs the built wheel into a clean virtual environment for one Python and + runs the installed-wheel checks, the binding tests and the conformance corpus + from a copy of tests/ outside the checkout, with python -I. +inputs: + python: + description: Python version + required: true +runs: + using: composite + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python }} + - name: Install the wheel into a clean environment + shell: bash + run: | + env="$RUNNER_TEMP/clean-${{ inputs.python }}" + python -m venv "$env" + if [ "$RUNNER_OS" = Windows ]; then py="$env/Scripts/python.exe"; else py="$env/bin/python"; fi + echo "PY=$py" >> "$GITHUB_ENV" + "$py" -m pip install --no-index --no-deps "$GITHUB_WORKSPACE"/dist/*.whl + "$py" -m pip list + # Tests run from a copy outside the checkout so the source tree cannot be imported. + rm -rf "$RUNNER_TEMP/run" && mkdir "$RUNNER_TEMP/run" + cp -r tests "$RUNNER_TEMP/run/tests" + - name: Installed-wheel checks + shell: bash + working-directory: ${{ runner.temp }}/run + env: + UKIRYU_WHEEL_EVIDENCE: ${{ github.workspace }}/evidence/wheel.json + run: '"$PY" -I -m unittest discover -s tests/wheel -v' + - name: Binding tests against the installed wheel + if: ${{ !cancelled() }} + shell: bash + working-directory: ${{ runner.temp }}/run + run: '"$PY" -I -m unittest discover -s tests -p "test_[vae]*.py" -v' + - name: Conformance corpus against the installed wheel + if: ${{ !cancelled() }} + shell: bash + working-directory: ${{ runner.temp }}/run + env: + CI: 'true' + UKIRYU_CONFORMANCE_DIR: ${{ github.workspace }}/ukiryu-rs/corpus/conformance + UKIRYU_CORPUS_REPORT: ${{ github.workspace }}/corpus-report-${{ inputs.python }}.json + run: '"$PY" -I -m unittest discover -s tests -p "test_corpus.py" -v' diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..01320f4 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,149 @@ +name: Wheels + +# Per-platform wheels that carry the pinned ukiryu-rs C library, installed +# into clean environments (in the same job, so one slow runner type does not +# hold the others back) and run against the conformance corpus. Nothing is +# published: the wheels are workflow artifacts only. + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: wheels-${{ github.ref }} + cancel-in-progress: true + +env: + RUST_TOOLCHAIN: 1.95.0 + +jobs: + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64 + runner: ubuntu-24.04 + plat: manylinux_2_28_x86_64 + container: quay.io/pypa/manylinux_2_28_x86_64 + - name: linux-aarch64 + runner: ubuntu-24.04-arm + plat: manylinux_2_28_aarch64 + container: quay.io/pypa/manylinux_2_28_aarch64 + - name: macos-arm64 + runner: macos-15 + plat: macosx_11_0_arm64 + deployment: '11.0' + - name: macos-x86_64 + runner: macos-15-intel + plat: macosx_10_15_x86_64 + deployment: '10.15' + - name: windows-x86_64 + runner: windows-2022 + plat: win_amd64 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Build the core library in the manylinux image + if: matrix.container + run: | + docker run --rm -v "$GITHUB_WORKSPACE:/w" -w /w/ukiryu-rs "${{ matrix.container }}" bash -euxo pipefail -c ' + curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain "'"$RUST_TOOLCHAIN"'" + . "$HOME/.cargo/env" + cargo build -p ukiryu-ffi --release --locked + chown -R '"$(id -u):$(id -g)"' /w/ukiryu-rs/target + ' + echo "LIBRARY=$GITHUB_WORKSPACE/ukiryu-rs/target/release/libukiryu_ffi.so" >> "$GITHUB_ENV" + - uses: dtolnay/rust-toolchain@stable + if: ${{ !matrix.container }} + with: + toolchain: 1.95.0 + - uses: Swatinem/rust-cache@v2 + if: ${{ !matrix.container }} + with: + workspaces: ukiryu-rs + key: wheel-${{ matrix.name }} + - name: Build the core library + if: ${{ !matrix.container }} + shell: bash + working-directory: ukiryu-rs + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment }} + # Static CRT: the wheel must not need the Visual C++ redistributable. + RUSTFLAGS: ${{ runner.os == 'Windows' && '-C target-feature=+crt-static' || '' }} + run: | + cargo build -p ukiryu-ffi --release --locked + case "$RUNNER_OS" in + macOS) lib=libukiryu_ffi.dylib ;; + Windows) lib=ukiryu_ffi.dll ;; + esac + echo "LIBRARY=$GITHUB_WORKSPACE/ukiryu-rs/target/release/$lib" >> "$GITHUB_ENV" + - name: Build tools + shell: bash + run: | + python -m pip install "setuptools>=70.1" wheel + case "$RUNNER_OS" in + Linux) python -m pip install auditwheel ;; + Windows) python -m pip install pefile ;; + esac + - name: Build the wheel + shell: bash + run: | + test -f "$LIBRARY" + python scripts/build_wheel.py "$LIBRARY" "${{ matrix.plat }}" dist + ls -l dist + - name: Check the wheel and record hashes + shell: bash + run: | + mkdir -p evidence + python scripts/check_wheel.py dist/*.whl "${{ matrix.plat }}" evidence/wheel.json + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.name }} + path: | + dist/*.whl + evidence/wheel.json + if-no-files-found: error + - name: Install and test on Python 3.9 + uses: ./.github/actions/install-test + with: + python: '3.9' + - name: Install and test on Python 3.13 + if: ${{ !cancelled() }} + uses: ./.github/actions/install-test + with: + python: '3.13' + - name: Corpus on the manylinux glibc baseline + if: ${{ !cancelled() && matrix.container }} + shell: bash + run: | + image="${{ matrix.container }}" + docker run --rm -v "$GITHUB_WORKSPACE:/w" -e CI=true \ + -e UKIRYU_WHEEL_EVIDENCE=/w/evidence/wheel.json \ + -e UKIRYU_CONFORMANCE_DIR=/w/ukiryu-rs/corpus/conformance "$image" bash -euxo pipefail -c ' + /opt/python/cp39-cp39/bin/python -m venv /tmp/clean + /tmp/clean/bin/python -m pip install --no-index --no-deps /w/dist/*.whl + mkdir /tmp/run && cp -r /w/tests /tmp/run/tests && cd /tmp/run + ldd --version | head -1 + /tmp/clean/bin/python -I -m unittest discover -s tests/wheel -v + /tmp/clean/bin/python -I -m unittest discover -s tests -p "test_corpus.py" -v + ' + - uses: actions/upload-artifact@v4 + if: always() + with: + name: corpus-report-${{ matrix.name }} + path: corpus-report-*.json + if-no-files-found: warn + diff --git a/README.md b/README.md index b27866b..0eb1b7a 100644 --- a/README.md +++ b/README.md @@ -58,3 +58,18 @@ PYTHONPATH=src python -m unittest discover -s tests The corpus test runs the shared conformance corpus through the public Python API and pins how many rows each operation runs. + +## Platform wheels + +`.github/workflows/wheels.yml` builds one wheel per platform, each carrying +the pinned core's C library at `ukiryu/lib/`: Linux x86_64 and aarch64 (built +in a manylinux_2_28 image), macOS arm64 and x86_64, and Windows x86_64. The +wheels are tagged `py3-none-`; there is no sdist and nothing is +published. `scripts/check_wheel.py` checks the library's glibc, macOS +deployment or Windows import baseline from the binary and records the wheel, +library and core hashes. Each wheel is then installed into a clean +environment, outside the source tree, and runs the binding tests and the +conformance corpus against the installed package. + +The package name, version and Python floor in `pyproject.toml` are the values +the repository already uses; they are not a ratified release contract. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..091d214 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=70.1", "wheel"] +build-backend = "setuptools.build_meta" + +# The name, version and Python floor below are what the repository already +# uses (src/ukiryu/__init__.py and the CI matrix). They are build inputs for +# the wheel gate and are not a ratified release contract: the package +# metadata is an owner decision and nothing here is published. +[project] +name = "ukiryu" +dynamic = ["version"] +requires-python = ">=3.9" +description = "Python binding for the Ukiryu core, over its C ABI." +readme = "README.md" + +[tool.setuptools] +package-dir = {"" = "src"} +packages = ["ukiryu"] +include-package-data = false + +[tool.setuptools.package-data] +ukiryu = ["lib/*"] + +[tool.setuptools.dynamic] +version = {attr = "ukiryu.__version__"} diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py new file mode 100644 index 0000000..b4e7e19 --- /dev/null +++ b/scripts/build_wheel.py @@ -0,0 +1,47 @@ +"""Build one platform wheel around a built ukiryu-ffi library. + +Usage: python scripts/build_wheel.py LIBRARY PLATFORM_TAG OUT_DIR + +Copies LIBRARY to src/ukiryu/lib/ under the name the loader expects, builds +a wheel tagged py3-none-PLATFORM_TAG with pip (no isolation, no sdist), and +removes the copy again. Needs setuptools>=70.1 and wheel installed. +""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from ukiryu._ffi import library_file_name # noqa: E402 + + +def main() -> int: + if len(sys.argv) != 4: + sys.stderr.write(__doc__) + return 2 + library, plat, out = Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3]) + if not library.is_file(): + sys.stderr.write("library not found: %s\n" % library) + return 2 + lib_dir = ROOT / "src" / "ukiryu" / "lib" + shutil.rmtree(lib_dir, ignore_errors=True) + shutil.rmtree(ROOT / "build", ignore_errors=True) + lib_dir.mkdir(parents=True) + shutil.copy2(library, lib_dir / library_file_name()) + out.mkdir(parents=True, exist_ok=True) + try: + subprocess.run([sys.executable, "-m", "pip", "wheel", str(ROOT), "--no-deps", + "--no-build-isolation", "-w", str(out)], + check=True, env=dict(os.environ, UKIRYU_WHEEL_PLAT=plat)) + finally: + shutil.rmtree(lib_dir, ignore_errors=True) + shutil.rmtree(ROOT / "build", ignore_errors=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_wheel.py b/scripts/check_wheel.py new file mode 100644 index 0000000..d8a31ba --- /dev/null +++ b/scripts/check_wheel.py @@ -0,0 +1,120 @@ +"""Inspect a built wheel and record what it carries. + +Usage: python scripts/check_wheel.py WHEEL PLATFORM_TAG EVIDENCE.json + +Fails unless the wheel is a platform wheel (never py3-none-any) with exactly +one core library for that platform and the expected tag. It then checks the +library's own baseline from the binary, not from the runner image: + +* Linux: the highest GLIBC symbol version required is within the manylinux + tag (``objdump -T``), and ``auditwheel show`` is logged. +* macOS: the Mach-O minimum OS version equals the tag's deployment target. +* Windows: no import of the Visual C++ runtime; every imported DLL is an + operating-system DLL. + +The record holds the wheel and library hashes and the pinned core commit. +This script never imports ukiryu, so it cannot see the source tree. +""" + +import hashlib +import json +import re +import subprocess +import sys +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LIBRARIES = {"ukiryu_ffi.dll", "libukiryu_ffi.so", "libukiryu_ffi.dylib"} +WINDOWS_RUNTIME = re.compile(r"^(vcruntime|msvcp|vcomp|concrt)", re.I) + + +def run(*cmd: str) -> str: + return subprocess.run(cmd, check=True, capture_output=True, text=True).stdout + + +def glibc_floor(lib: Path) -> tuple: + versions = [tuple(int(p) for p in m.split(".")) + for m in re.findall(r"GLIBC_(\d+(?:\.\d+)+)", run("objdump", "-T", str(lib)))] + return max(versions) if versions else (0, 0) + + +def macho_minos(lib: Path) -> str: + text = run("otool", "-l", str(lib)) + found = re.findall(r"LC_BUILD_VERSION.*?minos (\S+)", text, re.S) + found += re.findall(r"LC_VERSION_MIN_MACOSX.*?version (\S+)", text, re.S) + if not found: + raise SystemExit("no minimum macOS version in %s" % lib) + return found[0] + + +def pe_imports(lib: Path) -> list: + import pefile # installed by the workflow + + pe = pefile.PE(str(lib)) + return sorted({e.dll.decode().lower() for e in getattr(pe, "DIRECTORY_ENTRY_IMPORT", [])}) + + +def main() -> int: + wheel, plat, evidence = Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3]) + tag = plat.replace("-", "_").replace(".", "_") + problems = [] + name = wheel.name + if not name.endswith("-py3-none-%s.whl" % tag): + problems.append("wheel %s is not tagged py3-none-%s" % (name, tag)) + with zipfile.ZipFile(wheel) as z: + names = z.namelist() + libs = [n for n in names if Path(n).name in LIBRARIES] + binaries = [n for n in names if re.search(r"\.(so|dylib|dll|pyd|exe)(\.\d+)*$", n)] + wheel_meta = z.read(next(n for n in names if n.endswith(".dist-info/WHEEL"))).decode() + if len(libs) != 1 or not libs[0].startswith("ukiryu/lib/") or binaries != libs: + problems.append("expected exactly one ukiryu/lib library, found libs=%s binaries=%s" + % (libs, binaries)) + if "Root-Is-Purelib: false" not in wheel_meta: + problems.append("WHEEL does not say Root-Is-Purelib: false") + extract = evidence.parent / "extracted" + z.extract(libs[0], extract) if libs else None + detail = {} + if libs: + lib = extract / libs[0] + detail["library_sha256"] = hashlib.sha256(lib.read_bytes()).hexdigest() + detail["library_bytes"] = lib.stat().st_size + if tag.startswith("manylinux"): + m = re.match(r"manylinux_(\d+)_(\d+)_", tag) + limit = (int(m.group(1)), int(m.group(2))) + floor = glibc_floor(lib) + detail["max_glibc_symbol"] = "%d.%d" % floor + detail["auditwheel"] = (lambda r: r.stdout + r.stderr)(subprocess.run( + ["auditwheel", "show", str(wheel)], capture_output=True, text=True)) + if floor > limit: + problems.append("library needs GLIBC %d.%d, tag allows %d.%d" % (floor + limit)) + elif tag.startswith("macosx"): + m = re.match(r"macosx_(\d+)_(\d+)_", tag) + want = "%s.%s" % (m.group(1), m.group(2)) + detail["macho_minos"] = macho_minos(lib) + have = tuple(int(x) for x in detail["macho_minos"].split(".")[:2]) + if have > tuple(int(x) for x in want.split(".")): + problems.append("library minos %s exceeds the tag's %s" % (detail["macho_minos"], want)) + elif tag.startswith("win"): + imports = pe_imports(lib) + detail["pe_imports"] = imports + bad = [i for i in imports if WINDOWS_RUNTIME.match(i)] + if bad: + problems.append("library imports the VC++ runtime: %s" % bad) + core = ROOT / "ukiryu-rs" + record = { + "wheel": name, + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "platform_tag": tag, + "core_commit": run("git", "-C", str(core), "rev-parse", "HEAD").strip(), + "cargo_lock_sha256": hashlib.sha256((core / "Cargo.lock").read_bytes()).hexdigest(), + **detail, + "problems": problems, + } + evidence.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + sys.stdout.write(json.dumps(record, indent=2) + "\n") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..64700ff --- /dev/null +++ b/setup.py @@ -0,0 +1,31 @@ +"""Build a platform wheel that carries the core's C library. + +The package is pure ctypes, so the wheel is tagged ``py3-none-``: +one wheel per OS and architecture, usable by every CPython 3 that can load +the library. The platform tag is given by UKIRYU_WHEEL_PLAT (for example +``manylinux_2_28_x86_64``); it is never guessed from the build host. Only a +wheel is built; there is no sdist. +""" + +import os + +from setuptools import Distribution, setup +from setuptools.command.bdist_wheel import bdist_wheel + + +class BinaryDistribution(Distribution): + """Declares platform content so the package lands in platlib, not purelib.""" + + def has_ext_modules(self): + return True + + +class PlatformWheel(bdist_wheel): + def get_tag(self): + plat = os.environ.get("UKIRYU_WHEEL_PLAT") + if not plat: + raise SystemExit("UKIRYU_WHEEL_PLAT must name the wheel platform tag") + return "py3", "none", plat.replace("-", "_").replace(".", "_") + + +setup(distclass=BinaryDistribution, cmdclass={"bdist_wheel": PlatformWheel}) diff --git a/tests/wheel/test_installed_wheel.py b/tests/wheel/test_installed_wheel.py new file mode 100644 index 0000000..a9b422c --- /dev/null +++ b/tests/wheel/test_installed_wheel.py @@ -0,0 +1,49 @@ +"""Checks that only make sense against an installed wheel. + +Run with ``python -I`` from a clean virtual environment that has the wheel +installed and no source tree, no UKIRYU_CDYLIB and no PYTHONPATH. They prove +the package that loaded is the installed one, the library came from inside +it, and that library is byte-for-byte the one the build recorded. + +UKIRYU_WHEEL_EVIDENCE names the JSON written by scripts/check_wheel.py. +""" + +import hashlib +import json +import os +import sysconfig +import unittest +from pathlib import Path + +import ukiryu + + +class InstalledWheel(unittest.TestCase): + def test_no_override_is_in_play(self): + self.assertNotIn("UKIRYU_CDYLIB", os.environ) + + def test_package_is_the_installed_one(self): + site = Path(sysconfig.get_paths()["purelib"]).resolve() + package = Path(ukiryu.__file__).resolve() + self.assertIn(site, package.parents, "%s is not under %s" % (package, site)) + + def test_library_is_the_packaged_one(self): + info = ukiryu.library_info() + self.assertEqual(info.source, "packaged") + self.assertEqual(info.abi_version, 1) + self.assertIn(Path(ukiryu.__file__).resolve().parent, Path(info.path).resolve().parents) + + def test_library_matches_the_recorded_build(self): + path = os.environ.get("UKIRYU_WHEEL_EVIDENCE") + if not path: + raise AssertionError("UKIRYU_WHEEL_EVIDENCE must be set") + record = json.loads(Path(path).read_text(encoding="utf-8")) + loaded = hashlib.sha256(Path(ukiryu.library_info().path).read_bytes()).hexdigest() + self.assertEqual(loaded, record["library_sha256"]) + + def test_core_answers_over_the_wire(self): + self.assertTrue(ukiryu.core_version().version) + + +if __name__ == "__main__": + unittest.main()