From c0d0f5400b85c2d60d5f9646f3e50359e6ba3a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Hidalgo?= <47150974+IvanHCenalmor@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:34:44 +0200 Subject: [PATCH] Template 0.1.30: no silent change of a notebook's package versions, no lost work when a notebook is updated - #48: merge_requirements.py keeps requirements as written and combines them per package (names compared as pip compares them). Different exact pins, a pin outside another notebook's range, or ranges no version meets stop the merge before any file is written, naming every notebook and requirement. requirements_overrides.yaml decides explicitly, with a required reason, written at the top of the generated files. Different markers stay separate lines; OS-specific lists are checked against the common ones. --check only looks for conflicts. packaging==25.0 in .tools/requirements.txt. - The release runs merge_requirements.py --check after check_placeholders.py; a push of requirements_overrides.yaml runs the notebook workflow's merge again. - #49: the Welcome notebook's Update asks first, keeps the current notebook as .backup-.ipynb, writes the new one atomically and records the new version only after that. Cancel or any failure changes nothing; a private repository without a token no longer crashes the button. - Migration v0_1_29_to_v0_1_30.py: replaces the 0.1.29 merge script, adds packaging, creates requirements_overrides.yaml where notebooks already pin different versions (keeping the version used until now), updates the two workflows and the Welcome template's update handler; SKIPPED for customised files. - Design note migration_0_1_30.md, docs, tests (tests/test_template_sync_0_1_30.py; tests/_v0_1_30.py keeps the older frozen-resource tests comparable). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release_constructor.yml | 5 + .../workflows/update_on_notebook_change.yml | 4 +- .template_sync/manifest.yaml | 5 +- .../migrations/v0_1_29_to_v0_1_30.py | 505 ++++++++++++ .../.tools/python/merge_requirements.py | 755 ++++++++++++++++++ .../.tools/python/merge_requirements.py | 543 +++++++++++++ .tools/docs/migration_0_1_30.md | 54 ++ .tools/docs/notebook_requirements.md | 22 + .tools/docs/notebook_upload.md | 3 +- .tools/docs/notebook_usage.md | 4 + .tools/docs/troubleshooting.md | 6 + .tools/docs/troubleshooting_installed_app.md | 4 + .tools/python/merge_requirements.py | 500 ++++++++---- .tools/requirements.txt | 2 + .tools/templates/Welcome_template.ipynb | 147 +++- tests/_v0_1_25.py | 3 + tests/_v0_1_29.py | 3 + tests/_v0_1_30.py | 53 ++ .../template_0_1_29/Welcome_template.ipynb | 658 +++++++++++++++ tests/test_template_sync_0_1_20.py | 2 +- tests/test_template_sync_0_1_24.py | 12 +- tests/test_template_sync_0_1_29.py | 2 +- tests/test_template_sync_0_1_30.py | 468 +++++++++++ 23 files changed, 3564 insertions(+), 196 deletions(-) create mode 100644 .template_sync/migrations/v0_1_29_to_v0_1_30.py create mode 100644 .template_sync/resources/v0_1_29_to_v0_1_30/.tools/python/merge_requirements.py create mode 100644 .template_sync/resources/v0_1_29_to_v0_1_30/previous/.tools/python/merge_requirements.py create mode 100644 .tools/docs/migration_0_1_30.md create mode 100644 tests/_v0_1_30.py create mode 100644 tests/fixtures/template_0_1_29/Welcome_template.ipynb create mode 100644 tests/test_template_sync_0_1_30.py diff --git a/.github/workflows/release_constructor.yml b/.github/workflows/release_constructor.yml index fe5ec41..fffa59f 100644 --- a/.github/workflows/release_constructor.yml +++ b/.github/workflows/release_constructor.yml @@ -68,6 +68,11 @@ jobs: "check_placeholders.py" \ .tools/python/check_placeholders.py \ python .tools/python/check_placeholders.py + bash .tools/bash/run_with_context.sh \ + artifacts/error_log_document.log \ + "merge_requirements.py --check" \ + .tools/python/merge_requirements.py \ + python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt --check - name: Commit updated versioned files diff --git a/.github/workflows/update_on_notebook_change.yml b/.github/workflows/update_on_notebook_change.yml index 4a8c03b..9c779a5 100644 --- a/.github/workflows/update_on_notebook_change.yml +++ b/.github/workflows/update_on_notebook_change.yml @@ -7,6 +7,7 @@ on: - 'notebooks/**/*.ipynb' - 'notebooks/notebook_latest_versions.yaml' - 'notebooks/**/requirements.yaml' + - 'requirements_overrides.yaml' workflow_dispatch: # One run of this workflow at a time per branch. The src workflow has its own group: the two can run at the same time, @@ -45,6 +46,7 @@ jobs: notebooks/**/*.ipynb notebooks/notebook_latest_versions.yaml notebooks/**/requirements.yaml + requirements_overrides.yaml - name: Syncronize notebook versions if: contains(steps.changed-files.outputs.all_changed_files, '.ipynb') || contains(steps.changed-files.outputs.all_changed_files, 'notebook_latest_versions.yaml') @@ -92,7 +94,7 @@ jobs: python .tools/python/bump_notebook_README.py - name: Run merge script - if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') + if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml') || contains(steps.changed-files.outputs.all_changed_files, 'requirements_overrides.yaml') run: | bash .tools/bash/run_with_context.sh \ artifacts/error_log_document.log \ diff --git a/.template_sync/manifest.yaml b/.template_sync/manifest.yaml index f25214e..7e2792c 100644 --- a/.template_sync/manifest.yaml +++ b/.template_sync/manifest.yaml @@ -1,4 +1,4 @@ -template_version: 0.1.29 +template_version: 0.1.30 migrations: - from: 0.0.0 to: 0.0.1 @@ -93,3 +93,6 @@ migrations: - from: 0.1.28 to: 0.1.29 script: migrations/v0_1_28_to_v0_1_29.py + - from: 0.1.29 + to: 0.1.30 + script: migrations/v0_1_29_to_v0_1_30.py diff --git a/.template_sync/migrations/v0_1_29_to_v0_1_30.py b/.template_sync/migrations/v0_1_29_to_v0_1_30.py new file mode 100644 index 0000000..5bb496b --- /dev/null +++ b/.template_sync/migrations/v0_1_29_to_v0_1_30.py @@ -0,0 +1,505 @@ +"""Template 0.1.29 -> 0.1.30: no silent change of a notebook's package versions (#48), no lost work when a notebook is updated (#49). + +Design note: https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_30.md + +What this changes in an app repository (every step is independent, idempotent and never deletes anything): + +1. #48, the requirements merge replaced one notebook's pin with another notebook's higher one (`numpy==1.26.4` and `numpy==2.0.0` gave + `numpy==2.0.0`), and wrote every requirement as `==` (`numpy<2.0` became `numpy==2.0`), without a word. + * .tools/python/merge_requirements.py is replaced by the 0.1.30 file when it is still the 0.1.29 template file: requirements are kept + as written and combined, and requirements that disagree stop the merge, naming every notebook, unless requirements_overrides.yaml + decides, with the reason. `--check` only looks for conflicts. + * .tools/requirements.txt installs `packaging` (the merge compares versions as pip does). + * requirements_overrides.yaml is created when the notebooks already pin a package to different versions: it records the version the + merge chose until now (the newest), so the app installs what it installed before, but now as a visible decision to check. + * .github/workflows/update_on_notebook_change.yml runs the merge again when requirements_overrides.yaml changes. + * .github/workflows/release_constructor.yml runs `merge_requirements.py --check` before a release, after check_placeholders.py. +2. #49, the Update button of the Welcome notebook wrote the new notebook over the user's copy, with their changes and outputs, without a + word. .tools/templates/Welcome_template.ipynb (the Welcome notebook of the next release is made from it): the button asks first, keeps + the current copy as .backup-.ipynb, writes the new notebook atomically, and records the new version only after that. + +Rules this migration keeps: surgical (whole files only when they are the 0.1.29 template files, otherwise known blocks), idempotent (a +second run reports no changes), never raises for a customised file (the sync pull request must still be created), the line endings of +each file are kept, nothing is deleted. +""" +from __future__ import annotations + +import importlib.util +import json +import re +from functools import partial +from pathlib import Path +from types import ModuleType +from typing import Any, Callable + +DOCS_URL = "https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_30.md" +TAG = "[migration 0.1.30]" +RESOURCE_ROOT = Path(__file__).resolve().parents[1] / "resources" / "v0_1_29_to_v0_1_30" +PREVIOUS_ROOT = RESOURCE_ROOT / "previous" + +MERGE = Path(".tools/python/merge_requirements.py") +TOOLS_REQUIREMENTS = Path(".tools/requirements.txt") +OVERRIDES = Path("requirements_overrides.yaml") +NOTEBOOKS = Path("notebooks") +NOTEBOOK_WORKFLOW = Path(".github/workflows/update_on_notebook_change.yml") +RELEASE_WORKFLOW = Path(".github/workflows/release_constructor.yml") +WELCOME_TEMPLATE = Path(".tools/templates/Welcome_template.ipynb") + +PACKAGING_PIN = "packaging==25.0" +PACKAGING_LINES = "# merge_requirements.py (versions compared as pip compares them)\n" + PACKAGING_PIN + "\n" +PACKAGING_LINE = re.compile(r"^\s*packaging\s*([=<>!~;\[]|$)", re.IGNORECASE) + +# -------------------------------------------------------------------------------------------------------------------------------- +# update_on_notebook_change.yml: a change of requirements_overrides.yaml runs the merge again. (0.1.29 text, 0.1.30 text), each once. +# -------------------------------------------------------------------------------------------------------------------------------- +NOTEBOOK_WORKFLOW_BLOCKS: tuple[tuple[str, str], ...] = ( + (" - 'notebooks/**/requirements.yaml'\n", " - 'notebooks/**/requirements.yaml'\n - 'requirements_overrides.yaml'\n"), + (" notebooks/**/requirements.yaml\n", " notebooks/**/requirements.yaml\n requirements_overrides.yaml\n"), + ( + " if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml')\n", + " if: github.event_name == 'workflow_dispatch' || contains(steps.changed-files.outputs.all_changed_files, 'requirements.yaml')" + " || contains(steps.changed-files.outputs.all_changed_files, 'requirements_overrides.yaml')\n", + ), +) + +# -------------------------------------------------------------------------------------------------------------------------------- +# release_constructor.yml: `merge_requirements.py --check` right after check_placeholders.py in the bump-version job (the first one). +# -------------------------------------------------------------------------------------------------------------------------------- +PLACEHOLDER_STEP = re.compile( + r"(?P[ ]+)bash \.tools/bash/run_with_context\.sh \\\n" + r"(?P=indent) (?P\S+) \\\n" + r"(?P=indent) \"check_placeholders\.py\" \\\n" + r"(?P=indent) \.tools/python/check_placeholders\.py \\\n" + r"(?P=indent) python \.tools/python/check_placeholders\.py\n" +) +REQUIREMENTS_CHECK_STEP = ( + "{indent}bash .tools/bash/run_with_context.sh \\\n" + "{indent} {log} \\\n" + '{indent} "merge_requirements.py --check" \\\n' + "{indent} .tools/python/merge_requirements.py \\\n" + "{indent} python .tools/python/merge_requirements.py --source-dir notebooks --output requirements.txt --check\n" +) + +# -------------------------------------------------------------------------------------------------------------------------------- +# Welcome_template.ipynb (#49), in the code cell that defines the notebook table. Blocks are compared line by line without trailing +# spaces (the template's lines carry some); the new text replaces the lines of the old block. +# -------------------------------------------------------------------------------------------------------------------------------- +HELPER_ANCHOR = "def apply_grid_layout(grid):\n" +HELPER = '''def replace_notebook_with_backup(notebook_path: Path, new_content: str): + """Replace a notebook by its update, keeping the current file (with the user's changes and outputs) as a backup. + + The new content must be a notebook. It is written to a temporary file next to the notebook and then moved into place, so a failed + download or write never leaves a broken notebook. Returns the path of the backup, or None when there was no notebook yet. + """ + import os + from datetime import datetime + + if not isinstance(json.loads(new_content), dict): + raise ValueError("the downloaded file is not a notebook") + notebook_path.parent.mkdir(parents=True, exist_ok=True) + backup_path = None + if notebook_path.exists(): + stamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + backup_path = notebook_path.with_name(f"{notebook_path.stem}.backup-{stamp}{notebook_path.suffix}") + number = 1 + while backup_path.exists(): + number += 1 + backup_path = notebook_path.with_name(f"{notebook_path.stem}.backup-{stamp}-{number}{notebook_path.suffix}") + shutil.copy2(notebook_path, backup_path) + temporary_path = notebook_path.with_name(f".{notebook_path.name}.update") + try: + temporary_path.write_text(new_content, encoding="utf-8") + os.replace(temporary_path, notebook_path) + finally: + if temporary_path.exists(): + temporary_path.unlink() + return backup_path + + +''' +HANDLER_OLD = ''' def button_update(main_folder, subfolder, row_idx): + def show(button): + button.disabled = True + button.description = "Updating..." + + # Download the notebook + notebook_url = f"https://api.github.com/repos/{github_owner}/{github_repo_name}/contents/notebooks/{main_folder}/{subfolder}.ipynb?ref={github_branch}" + if private_repository: + if github_token.value: + notebook_response = requests.get(notebook_url, headers={"Authorization": f"token {github_token.value}"}) + else: + grip_output.value = "❌ Please check your Presonal Access Token" + else: + notebook_response = requests.get(notebook_url) + + if notebook_response.status_code == 200: + try: + # Decode the notebook content + notebook_content = base64.b64decode( + notebook_response.json()['content'] + ).decode('utf-8') + + # Save to local path + local_notebook_path = Path(".") / main_folder / f"{subfolder}.ipynb" + local_notebook_path.parent.mkdir(parents=True, exist_ok=True) + local_notebook_path.write_text(notebook_content, encoding='utf-8') + + # Update the local version YAML + local_latest_versions[main_folder][subfolder] = online_latest_versions[main_folder][subfolder] + local_latest_version_path.write_text(yaml.dump(local_latest_versions, default_flow_style=False)) + + grip_output.value = f"✅ Updated {subfolder} to version {online_latest_versions[main_folder][subfolder]}" + + # Update the grid to show up-to-date + grid[row_idx, 3] = table_cell(online_latest_versions[main_folder][subfolder]) + grid[row_idx, 4] = table_cell("✅ Up-to-date") + grid[row_idx, 5] = table_cell("-") + apply_grid_layout(grid) + + except Exception as e: + grip_output.value = f"❌ Error saving notebook: {e}" + button.disabled = False + button.description = "Update" + else: + grip_output.value = f"❌ Failed to download notebook (HTTP {notebook_response.status_code})" + display(notebook_response.text[:300]) + button.disabled = False + button.description = "Update" + + return show +''' +HANDLER_NEW = ''' def button_update(main_folder, subfolder, row_idx): + # Updating replaces the user's copy of the notebook: ask first, keep that copy as a backup, and change the + # recorded version only once the new notebook is in place (CellMigrationLab/LabConstrictor#49). + local_notebook_path = Path(".") / main_folder / f"{subfolder}.ipynb" + new_version = online_latest_versions[main_folder][subfolder] + + def reset(button): + button.disabled = False + button.description = "Update" + + def do_update(button): + button.description = "Updating..." + notebook_url = f"https://api.github.com/repos/{github_owner}/{github_repo_name}/contents/notebooks/{main_folder}/{subfolder}.ipynb?ref={github_branch}" + headers = {} + if private_repository: + if not github_token.value: + grip_output.value = "❌ Please check your Personal Access Token. Your notebook was not changed." + reset(button) + return + headers = {"Authorization": f"token {github_token.value}"} + try: + notebook_response = requests.get(notebook_url, headers=headers, timeout=60) + except requests.RequestException as e: + grip_output.value = f"❌ Failed to download notebook: {e}. Your notebook was not changed." + reset(button) + return + if notebook_response.status_code != 200: + grip_output.value = f"❌ Failed to download notebook (HTTP {notebook_response.status_code}). Your notebook was not changed." + display(notebook_response.text[:300]) + reset(button) + return + try: + notebook_content = base64.b64decode(notebook_response.json()['content']).decode('utf-8') + backup_path = replace_notebook_with_backup(local_notebook_path, notebook_content) + except Exception as e: + grip_output.value = f"❌ Error saving notebook: {e}. Your notebook was not changed." + reset(button) + return + + # The new notebook is in place: only now record its version + local_latest_versions[main_folder][subfolder] = new_version + local_latest_version_path.write_text(yaml.dump(local_latest_versions, default_flow_style=False)) + saved = f" Your previous copy is saved as {backup_path.name}, next to it." if backup_path else "" + grip_output.value = f"✅ Updated {subfolder} to version {new_version}.{saved}" + + # Update the grid to show up-to-date + grid[row_idx, 3] = table_cell(new_version) + grid[row_idx, 4] = table_cell("✅ Up-to-date") + grid[row_idx, 5] = table_cell("-") + apply_grid_layout(grid) + + def show(button): + button.disabled = True + button.description = "Confirm below" + question = widgets.HTML( + f"Update {subfolder} to version {new_version}?
" + f"The new version replaces {local_notebook_path.as_posix()}. Your current copy, with your changes " + f"and outputs, is first saved as {subfolder}.backup-<date>.ipynb in the same folder.
" + "If the notebook is open in a tab, close that tab first: otherwise JupyterLab can save the old copy over the update." + ) + confirm = widgets.Button(description="Back up and update", button_style="warning", layout=widgets.Layout(width="auto")) + cancel = widgets.Button(description="Cancel") + dialog = widgets.VBox([question, widgets.HBox([confirm, cancel])]) + + def on_confirm(_): + dialog.close() + do_update(button) + + def on_cancel(_): + dialog.close() + grip_output.value = f"Update of {subfolder} cancelled: nothing was changed." + reset(button) + + confirm.on_click(_widget_callback_output_cell_1.capture(clear_output=True, wait=True)(on_confirm)) + cancel.on_click(_widget_callback_output_cell_1.capture(clear_output=True, wait=True)(on_cancel)) + display(dialog) + + return show +''' +WELCOME_DONE = "def replace_notebook_with_backup(" + + +def log(message: str) -> None: + print(f"{TAG} {message}") + + +def _load(name: str, alias: str) -> ModuleType: + path = Path(__file__).resolve().parent / name + spec = importlib.util.spec_from_file_location(alias, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _lf(text: str) -> tuple[str, str]: + return text.replace("\r\n", "\n"), ("\r\n" if "\r\n" in text else "\n") + + +def apply_blocks(text_lf: str, blocks: tuple[tuple[str, str], ...]) -> str | None: + """The text with every block changed, or None when a 0.1.29 block is not there exactly once (a customised file).""" + for old, new in blocks: + if text_lf.count(old) != 1: + return None + text_lf = text_lf.replace(old, new, 1) + return text_lf + + +# -------------------------------------------------------------------------------------------------------------------------------- +# #48 +# -------------------------------------------------------------------------------------------------------------------------------- +def _replace_merge(helpers: Any, repo_root: Path) -> str: + path = repo_root / MERGE + current, newline = _lf(helpers.read_text(path)) + new = (RESOURCE_ROOT / MERGE).read_text(encoding="utf-8") + if current == new: + return "already the 0.1.30 template file: unchanged" + if current != (PREVIOUS_ROOT / MERGE).read_text(encoding="utf-8"): + return f"SKIPPED: customised (not the 0.1.29 template file), so it was left unchanged; apply the 0.1.30 change by hand, see {DOCS_URL}" + helpers.write_text(path, new.replace("\n", newline)) + return "updated: replaced by the 0.1.30 template file (requirements that disagree stop the merge)" + + +def _add_packaging(helpers: Any, repo_root: Path) -> str: + path = repo_root / TOOLS_REQUIREMENTS + current = helpers.read_text(path) + if any(PACKAGING_LINE.match(line) for line in current.splitlines()): + return "already installs packaging: unchanged" + newline = "\r\n" if "\r\n" in current else "\n" + text = current if not current or current.endswith(("\n", "\r")) else current + newline + helpers.write_text(path, text + PACKAGING_LINES.replace("\n", newline)) + return f"updated: added {PACKAGING_PIN}" + + +def _canonical(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def _version_key(version: str) -> tuple[tuple[int, Any], ...]: + """The order the merge used until 0.1.29 for plain versions (numbers compared as numbers).""" + parts = re.split(r"[.\-+_]|(?=[a-zA-Z])", version) + return tuple((0, int(p)) if p.isdigit() else (1, p) for p in parts if p) + + +def notebook_dependencies(text: str) -> list[str]: + """The `dependencies` list of a requirements.yaml (PyYAML when it is installed, else the simple list forms the template uses).""" + try: + import yaml + except ImportError: + pass + else: + data = yaml.safe_load(text) or {} + found = data.get("dependencies") if isinstance(data, dict) else None + return [str(item) for item in found or []] + items: list[str] = [] + inside = False + for raw in text.splitlines(): + line = raw.split("#", 1)[0].rstrip() + key = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line) + if key: + inside = key.group(1) == "dependencies" + value = key.group(2).strip() + if inside and value.startswith("[") and value.endswith("]"): + items += [part.strip().strip("'\"") for part in value[1:-1].split(",") if part.strip()] + inside = False + continue + item = re.match(r"^\s*-\s+(.*)$", line) + if inside and item: + items.append(item.group(1).strip().strip("'\"")) + return items + + +EXACT_PIN = re.compile(r"^(?P[A-Za-z0-9][A-Za-z0-9_.-]*)(\[[^\]]*\])?\s*==\s*(?P[A-Za-z0-9_.+!-]+)$") + + +def pin_conflicts(repo_root: Path, helpers: Any) -> dict[str, list[tuple[str, str]]]: + """Packages that the notebooks' `dependencies` pin (==, without a marker) to different versions: {name: [(file, version)]}.""" + pins: dict[str, list[tuple[str, str]]] = {} + root = repo_root / NOTEBOOKS + files = sorted({*root.rglob("requirements.yaml"), *root.rglob("requirements.yml")}) if root.is_dir() else [] + for path in files: + for dependency in notebook_dependencies(helpers.read_text(path)): + match = EXACT_PIN.match(dependency.split("#", 1)[0].strip()) + if match: + pins.setdefault(_canonical(match.group("name")), []).append((path.relative_to(repo_root).as_posix(), match.group("version"))) + return {name: found for name, found in sorted(pins.items()) if len({version for _, version in found}) > 1} + + +def overrides_text(conflicts: dict[str, list[tuple[str, str]]]) -> str: + def quoted(text: str) -> str: + return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' + + lines = [ + "# Decisions for packages that the notebooks pin to different versions (template 0.1.30, CellMigrationLab/LabConstrictor#48).", + "# The requirements merge (.tools/python/merge_requirements.py) stops when notebooks disagree and no entry here decides. Each entry", + "# gives the requirement every notebook gets (use) and why (reason); both are written at the top of the generated requirements files.", + f"# See {DOCS_URL}", + "overrides:", + ] + for name, found in conflicts.items(): + newest = max((version for _, version in found), key=_version_key) + asked = ", ".join(f"{file} =={version}" for file, version in found) + reason = ( + "Recorded by the template 0.1.30 migration: the version the requirements merge chose until then (the newest pin), so the app " + f"installs what it installed before. The notebooks ask for: {asked}. Check every notebook with this version, then write here why it is right." + ) + lines += [f" {name}:", f" use: {quoted('==' + newest)}", f" reason: {quoted(reason)}"] + return "\n".join(lines) + "\n" + + +def _seed_overrides(helpers: Any, repo_root: Path) -> str: + conflicts = pin_conflicts(repo_root, helpers) + path = repo_root / OVERRIDES + if path.is_file(): + return "already exists: unchanged (the merge says which conflicts it does not decide yet)" + if not conflicts: + return "not needed: no package is pinned to different versions by the notebooks (nothing was created)" + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(overrides_text(conflicts)) + return f"updated: created; it records the version used until now for {', '.join(conflicts)}. Check each one (see {DOCS_URL})" + + +def add_requirements_check(text_lf: str) -> str | None: + """The release workflow with `merge_requirements.py --check` after the first check_placeholders.py step; None without that step.""" + match = PLACEHOLDER_STEP.search(text_lf) + if match is None: + return None + step = REQUIREMENTS_CHECK_STEP.format(indent=match.group("indent"), log=match.group("log")) + return text_lf[: match.end()] + step + text_lf[match.end() :] + + +def _update_release_workflow(helpers: Any, repo_root: Path, path: Path) -> str: + current, newline = _lf(helpers.read_text(path)) + if "merge_requirements.py --source-dir notebooks --output requirements.txt --check" in current: + return "unchanged: already checks the requirements" + merge = repo_root / MERGE + if not merge.is_file() or '"--check"' not in helpers.read_text(merge): + return f"SKIPPED: the repository's merge_requirements.py has no --check (it was not replaced), so the release does not run it. See {DOCS_URL}" + new = add_requirements_check(current) + if new is None: + return f"SKIPPED: the check_placeholders.py command was not found (customised workflow); add the requirements check by hand. See {DOCS_URL}" + helpers.write_text(path, new.replace("\n", newline)) + return "updated: the release stops when the notebooks' requirements conflict (merge_requirements.py --check)" + + +def _blocks_step(helpers: Any, path: Path, blocks: tuple[tuple[str, str], ...], what: str, skipped_hint: str) -> str: + current, newline = _lf(helpers.read_text(path)) + if all(new in current for _, new in blocks): + return f"unchanged: already {what}" + new = apply_blocks(current, blocks) + if new is None: + return f"SKIPPED: customised (a 0.1.29 block was not found), so it was left unchanged; {skipped_hint} See {DOCS_URL}" + helpers.write_text(path, new.replace("\n", newline)) + return f"updated: {what}" + + +# -------------------------------------------------------------------------------------------------------------------------------- +# #49 +# -------------------------------------------------------------------------------------------------------------------------------- +def _replace_lines(source: str, old: str, new: str) -> str | None: + """`source` with the lines of `old` (compared without trailing spaces) replaced by `new`; None when they are not there exactly once.""" + lines = source.splitlines(keepends=True) + wanted = [line.rstrip() for line in old.splitlines()] + stripped = [line.rstrip() for line in lines] + starts = [i for i in range(len(lines) - len(wanted) + 1) if stripped[i : i + len(wanted)] == wanted] + if len(starts) != 1: + return None + start = starts[0] + return "".join(lines[:start]) + new + "".join(lines[start + len(wanted) :]) + + +def update_welcome_source(source: str) -> str | None: + """The table cell with the safe update (#49), or None when the 0.1.29 handler or the helper's anchor is not found.""" + if source.count(HELPER_ANCHOR) != 1: + return None + replaced = _replace_lines(source, HANDLER_OLD, HANDLER_NEW) + if replaced is None: + return None + return replaced.replace(HELPER_ANCHOR, HELPER + HELPER_ANCHOR, 1) + + +def _update_welcome(helpers: Any, path: Path) -> str: + text = helpers.read_text(path) + newline = "\r\n" if "\r\n" in text else "\n" + notebook = json.loads(text) + cells = [cell for cell in notebook.get("cells", []) if cell.get("cell_type") == "code"] + sources = ["".join(cell["source"]) if isinstance(cell.get("source"), list) else cell.get("source", "") for cell in cells] + if any(WELCOME_DONE in source for source in sources): + return "unchanged: the Update button already keeps a backup" + for cell, source in zip(cells, sources): + new = update_welcome_source(source.replace("\r\n", "\n")) + if new is None: + continue + cell["source"] = new.splitlines(keepends=True) if isinstance(cell.get("source"), list) else new + rendered = json.dumps(notebook, ensure_ascii=False, indent=1) + if text.endswith(("\n", "\r\n")): + rendered += "\n" + helpers.write_text(path, rendered.replace("\n", newline)) + return "updated: the Update button asks first and keeps the current notebook as a backup" + return f"SKIPPED: the update code of the notebook table was not found (customised); apply the 0.1.30 change by hand. See {DOCS_URL}" + + +def migrate(repo_root: Path, context: dict[str, Any]) -> None: + """Requirements that disagree stop the merge (#48); the notebook update asks first and keeps a backup (#49). Idempotent, surgical.""" + _ = context + helpers = _load("v0_1_9_to_v0_1_10.py", "template_sync_helpers_0_1_30") + changed = 0 + + def report(name: Path, step: Callable[[], str], must_exist: bool = True) -> None: + nonlocal changed + if must_exist and not (repo_root / name).is_file(): + log(f"{name}: not present in this repository (nothing was created)") + return + try: + status = step() + except (OSError, ValueError, UnicodeDecodeError) as exc: + status = f"SKIPPED: {exc}; left unchanged. See {DOCS_URL}" + if status.startswith("updated"): + changed += 1 + log(f"{name}: {status}") + + report(MERGE, partial(_replace_merge, helpers, repo_root)) + report(TOOLS_REQUIREMENTS, partial(_add_packaging, helpers, repo_root)) + report(OVERRIDES, partial(_seed_overrides, helpers, repo_root), must_exist=False) + report(NOTEBOOK_WORKFLOW, partial(_blocks_step, helpers, repo_root / NOTEBOOK_WORKFLOW, NOTEBOOK_WORKFLOW_BLOCKS, + "runs the merge when requirements_overrides.yaml changes", + "after a change of requirements_overrides.yaml, run Updates on notebook change by hand.")) + report(RELEASE_WORKFLOW, partial(_update_release_workflow, helpers, repo_root, repo_root / RELEASE_WORKFLOW)) + report(WELCOME_TEMPLATE, partial(_update_welcome, helpers, repo_root / WELCOME_TEMPLATE)) + + if changed: + log(f"done: {changed} file(s) changed. Review the pull request before merging; the fixes reach users with the next release of the app.") + else: + log("done: no repository changes were required.") diff --git a/.template_sync/resources/v0_1_29_to_v0_1_30/.tools/python/merge_requirements.py b/.template_sync/resources/v0_1_29_to_v0_1_30/.tools/python/merge_requirements.py new file mode 100644 index 0000000..37fc5fe --- /dev/null +++ b/.template_sync/resources/v0_1_29_to_v0_1_30/.tools/python/merge_requirements.py @@ -0,0 +1,755 @@ +#!/usr/bin/env python3 +"""Merge requirements files under a directory (e.g. notebooks/*) into a single +root requirements file. + +The script supports both classic requirements.txt files and the +requirements.yaml format used by notebooks. For YAML files, it merges the +`python_version` (every notebook must agree) and the `dependencies` list while +discarding per-notebook descriptions. + +It also recognizes optional per-OS fields (`windows_dependencies`, +`linux_dependencies`, `macos_dependencies`) and emits dedicated requirements +outputs for each platform containing only the entries from their OS-specific +lists, in addition to the common file. + +Alongside the common requirements file, it emits a GPU-targeted variant. The +GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` on Linux only (TensorFlow's CUDA +wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the +PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present. + +Version requirements are never changed silently (template 0.1.30, CellMigrationLab/LabConstrictor#48): + * the requirements of a package are kept as written: `numpy>=1.24` stays `numpy>=1.24`, and the + requirements of several notebooks are combined (`numpy<2,>=1.24`); the lock step then finds the versions; + * when notebooks pin a package to different versions (`numpy==1.26.4` and `numpy==2.0.0`), or a pin does not + meet another notebook's requirement, nothing is written and the script fails, naming every notebook and its + requirement. The developer makes the notebooks agree, or decides explicitly in requirements_overrides.yaml + (next to the output), with the reason; the decision is written at the top of every generated file; + * requirements with different environment markers (`; sys_platform == "win32"`) are kept as separate lines; + * package names are compared as pip compares them (`NumPy`, `numpy` and `num_py` are one package). +`python merge_requirements.py --check` only looks for conflicts: it writes nothing (the release runs it). + +It preserves non-package lines (pip options, VCS/URLs) and supports simple `-r` includes. + +It also automatically adds ipywidgets if missing, with a version determined by +Python, JupyterLab, and matplotlib versions from environment.yaml. +""" +import argparse +import re +import sys +from collections import OrderedDict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +import yaml +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version + +DOCS_URL = "https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_30.md" +OVERRIDES_FILE = "requirements_overrides.yaml" + +# Regex to capture name, optional extras, operator and version, and optional markers (used for the GPU rewrite of final lines) +pkg_re = re.compile(r"^(?P[A-Za-z0-9_.+-]+)(?P\[[^\]]+\])?\s*(?P==|>=|<=|~=|!=|>|<)\s*(?P[^;\s]+)(?P\s*;.*)?$") + +OS_FIELD_MAP = { + "windows": "windows_dependencies", + "linux": "linux_dependencies", + "macos": "macos_dependencies", +} +OS_DISPLAY_NAMES = { + "windows": "Windows", + "linux": "Linux", + "macos": "macOS", +} +GPU_DISPLAY_NAME = "NVIDIA GPU" +GPU_TORCH_EXTRA_INDEX_URL = "--extra-index-url https://download.pytorch.org/whl/cu128" +TORCH_GPU_PACKAGES = {"torch", "torchvision", "torchaudio"} +PLAIN_PKG_RE = re.compile( + r"^(?P[A-Za-z0-9_.+-]+)(?P\[[^\]]+\])?(?P\s*;.*)?$" +) +ADDED_BY_SCRIPT = "added by merge_requirements.py" + + +@dataclass +class Entry: + """One requirement line of one file, as written.""" + + source: str + line: str + name: str + extras: tuple[str, ...] + spec: str + marker: str + + +@dataclass +class Override: + """A decision of requirements_overrides.yaml: every notebook gets `use` for this package.""" + + name: str + use: str + reason: str + used: bool = False + + +@dataclass +class Resolved: + """The line written for one package (and marker), and how it was decided.""" + + entries: list[Entry] + spec: str + extras: list[str] + override: Override | None = None + + @property + def name(self) -> str: + return self.entries[0].name + + @property + def marker(self) -> str: + return self.entries[0].marker + + def line(self) -> str: + extras = f"[{','.join(self.extras)}]" if self.extras else "" + marker = f" ; {self.marker}" if self.marker else "" + return f"{self.name}{extras}{self.spec}{marker}" + + +@dataclass +class RequirementContext: + groups: OrderedDict[tuple[str, str], list[Entry]] = field(default_factory=OrderedDict) + other_lines: OrderedDict[str, None] = field(default_factory=OrderedDict) + errors: list[str] = field(default_factory=list) + + +class Conflict(Exception): + """Requirements of one package that no single version meets, or that the script will not choose between.""" + + def __init__(self, package: str, reason: str, entries: list[Entry], where: str = ""): + super().__init__(reason) + self.package = package + self.reason = reason + self.entries = entries + self.where = where + + +def normalize_line(line: str) -> str: + # Remove inline comments and surrounding whitespace. + if "#" in line: + line = line.split("#", 1)[0] + return line.strip() + + +def display_path(path: Path) -> str: + try: + return path.resolve().relative_to(Path.cwd().resolve()).as_posix() + except ValueError: + return path.as_posix() + + +def merge_python_version(filepath: str | Path | None, current: str | None, candidate: str | None): + """Every notebook that names a python_version must name the same one.""" + if not candidate: + return current + if current is None: + return candidate + if current == candidate: + return current + else: + raise ValueError(f"Conflicting python_version found {current} vs {candidate} when processing {filepath}") + + +def process_requirement_line(line: str, ctx: RequirementContext, source: str, base_dir: Path | None = None): + """Handle a single requirement line (shared between txt and yaml inputs).""" + if not line: + return + + parts = line.split(maxsplit=1) + if parts[0] in ("-r", "--requirement") and len(parts) == 2: + include_path = Path(parts[1]) if base_dir is None else (base_dir / parts[1]) + include = include_path.resolve() + if include.exists(): + read_requirements_file(include, ctx) + return + + # Keep pip option lines (indexes, find-links, etc.) as-is + if line.startswith("--") or line.startswith("-f ") or line.startswith("-i ") or line.startswith("-e "): + ctx.other_lines.setdefault(line, None) + return + + # VCS or URL lines (git+, http(s)://, file:) keep as-is + if any(line.startswith(prefix) for prefix in ("git+", "http://", "https://", "file:", "ssh://")): + ctx.other_lines.setdefault(line, None) + return + + try: + requirement = Requirement(line) + except InvalidRequirement as exc: + ctx.errors.append(f"{source}: '{line}' is not a valid requirement ({exc})") + return + if requirement.url: + # name @ url: installed from that address, as written + ctx.other_lines.setdefault(line, None) + return + marker = str(requirement.marker) if requirement.marker else "" + entry = Entry(source, line, requirement.name, tuple(sorted(requirement.extras)), str(requirement.specifier), marker) + ctx.groups.setdefault((canonicalize_name(requirement.name), marker), []).append(entry) + + +def read_requirements_file(path: Path, ctx: RequirementContext): + try: + text = path.read_text(encoding="utf-8") + except Exception: + return + + for raw in text.splitlines(): + line = normalize_line(raw) + process_requirement_line(line, ctx, display_path(path), base_dir=path.parent) + + +def read_requirements_yaml(path: Path, ctx: RequirementContext, os_contexts: dict[str, RequirementContext]) -> str | None: + """Read requirements.yaml and return its python_version if present.""" + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except Exception: + return None + + python_version = data.get("python_version") + dependencies = data.get("dependencies", []) or [] + source = display_path(path) + + for dep in dependencies: + line = normalize_line(str(dep)) + process_requirement_line(line, ctx, source, base_dir=path.parent) + + for os_name, field_name in OS_FIELD_MAP.items(): + os_deps = data.get(field_name) or [] + if not os_deps: + continue + os_ctx = os_contexts.get(os_name) + if os_ctx is None: + continue + for dep in os_deps: + line = normalize_line(str(dep)) + process_requirement_line(line, os_ctx, f"{source} ({field_name})", base_dir=path.parent) + + return python_version + + +def find_requirements_files(source_dir): + if not source_dir.exists(): + return [] + + patterns = ["requirements.yaml", "requirements.yml", "requirements*.txt"] + + files = [] + for pattern in patterns: + files.extend(source_dir.rglob(pattern)) + return sorted({f.resolve() for f in files}) + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Deciding the requirement of each package +# -------------------------------------------------------------------------------------------------------------------------------- +def load_overrides(path: Path) -> dict[str, Override]: + """The decisions of requirements_overrides.yaml, by package name as pip compares them (none when the file is absent).""" + if not path.is_file(): + return {} + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + raise ValueError(f"{display_path(path)} is not valid YAML: {exc}") from exc + entries = data.get("overrides") if isinstance(data, dict) else None + if entries is None: + return {} + if not isinstance(entries, dict): + raise ValueError(f"{display_path(path)}: `overrides` must map package names to {{use: ..., reason: ...}}") + overrides: dict[str, Override] = {} + for name, value in entries.items(): + use = value.get("use") if isinstance(value, dict) else None + reason = value.get("reason") if isinstance(value, dict) else None + if not isinstance(use, str) or not use.strip(): + raise ValueError(f"{display_path(path)}: {name} needs `use`, the requirement every notebook gets, for example \"==2.0.0\"") + if not isinstance(reason, str) or not reason.strip(): + raise ValueError(f"{display_path(path)}: {name} needs `reason`, why this requirement is right for every notebook that uses {name}") + try: + SpecifierSet(use.strip()) + except InvalidSpecifier as exc: + raise ValueError(f"{display_path(path)}: {name}: `use: {use}` is not a version requirement such as \"==2.0.0\" or \">=1.24,<2\" ({exc})") from exc + overrides[canonicalize_name(str(name))] = Override(str(name), use.strip(), " ".join(reason.split())) + return overrides + + +def _exact_versions(entry: Entry) -> list[str]: + return [s.version for s in SpecifierSet(entry.spec) if s.operator in ("==", "===") and "*" not in s.version] + + +def _same_version(a: str, b: str) -> bool: + try: + return Version(a) == Version(b) + except InvalidVersion: + return a == b + + +def _union_extras(entries: list[Entry]) -> list[str]: + extras: list[str] = [] + for entry in entries: + extras.extend(extra for extra in entry.extras if extra not in extras) + return extras + + +def resolve_group(package: str, entries: list[Entry], checked_with: list[Entry], override: Override | None, where: str = "") -> Resolved: + """The requirement written for `entries` (one package, one marker); `checked_with` are other requirements that apply too. + + An override decides. Otherwise the requirements are combined: several different exact pins, or a pin that does not meet another + requirement, raise Conflict (whatever the order of the files). Without a pin, the combined ranges are written as they are. + """ + extras = _union_extras(entries) + if override is not None: + override.used = True + return Resolved(entries, override.use, extras, override) + everything = entries + checked_with + pins: list[tuple[str, Entry]] = [(version, entry) for entry in everything for version in _exact_versions(entry)] + distinct: list[str] = [] + for version, _ in pins: + if not any(_same_version(version, seen) for seen in distinct): + distinct.append(version) + if len(distinct) > 1: + raise Conflict(package, f"pinned to different versions ({', '.join(sorted(distinct))})", everything, where) + if distinct: + pin = distinct[0] + for entry in everything: + if entry.spec and not SpecifierSet(entry.spec).contains(pin, prereleases=True): + raise Conflict(package, f"the pin =={pin} does not meet {entry.name}{entry.spec} of {entry.source}", everything, where) + elif not satisfiable(SpecifierSet(",".join(entry.spec for entry in everything if entry.spec))): + raise Conflict(package, "no version meets all of these requirements", everything, where) + # The line keeps the requirements of its own entries; `checked_with` (the common requirements of an OS file) only had to agree. + own_pins = [version for entry in entries for version in _exact_versions(entry)] + if own_pins: + return Resolved(entries, f"=={own_pins[0]}", extras) + return Resolved(entries, str(SpecifierSet(",".join(entry.spec for entry in entries if entry.spec))), extras) + + +def satisfiable(specifiers: SpecifierSet) -> bool: + """Whether some version meets every specifier (`<2,>=2.1` does not). + + Every version a specifier names is tried, and versions just above it (2.1, 2.1.1, 2.1.0.1, 2.1.0.0.1): a range that only versions + closer than that could meet is reported as unsatisfiable, which stops the merge (an override then decides) instead of guessing. + """ + if not len(specifiers): + return True + candidates: list[str] = [] + for specifier in specifiers: + version = specifier.version.removesuffix(".*") + candidates += [version, f"{version}.1", f"{version}.0.1", f"{version}.0.0.1"] + for candidate in candidates: + try: + if specifiers.contains(candidate, prereleases=True): + return True + except InvalidVersion: + continue + return False + + +def resolve_context(ctx: RequirementContext, overrides: dict[str, Override], conflicts: list[Conflict], + base: RequirementContext | None = None, where: str = "") -> list[Resolved]: + """Resolve every package of a context; an OS context is also checked against the common requirements that apply on that OS.""" + resolved: list[Resolved] = [] + for (package, marker), entries in ctx.groups.items(): + checked_with: list[Entry] = [] + if base is not None: + for (base_package, base_marker), base_entries in base.groups.items(): + if base_package == package and base_marker in ("", marker): + checked_with.extend(base_entries) + try: + resolved.append(resolve_group(package, entries, checked_with, overrides.get(package), where)) + except Conflict as conflict: + conflicts.append(conflict) + return resolved + + +def conflict_report(conflicts: list[Conflict], errors: list[str], overrides_path: Path) -> str: + lines = ["The notebooks ask for incompatible package versions, so no requirements file was written.", ""] + for error in errors: + lines.append(f" {error}") + for conflict in sorted(conflicts, key=lambda c: (c.package, c.where)): + where = f" ({conflict.where})" if conflict.where else "" + lines.append(f" {conflict.package}{where}: {conflict.reason}") + for entry in sorted(conflict.entries, key=lambda e: (e.source, e.line)): + lines.append(f" {entry.source}: {entry.line}") + if conflicts: + example = sorted(conflicts, key=lambda c: c.package)[0].package + lines += [ + "", + f"Make the notebooks agree (edit their requirements.yaml), or decide in {display_path(overrides_path)} which requirement every", + "notebook gets, and why. It is written at the top of the generated requirements files. For example:", + "", + "overrides:", + f" {example}:", + ' use: "=="', + f' reason: ""', + ] + lines += ["", f"See {DOCS_URL}"] + return "\n".join(lines) + + +def override_header(resolved: list[Resolved]) -> list[str]: + lines = [] + for item in resolved: + if item.override is None: + continue + asked = "; ".join(f"{entry.source}: {entry.line}" for entry in item.entries if entry.source != ADDED_BY_SCRIPT) + lines.append(f"# Decided in {OVERRIDES_FILE}: {item.line()} (the notebooks ask for: {asked}). Reason: {item.override.reason}") + return lines + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Packages the app always needs +# -------------------------------------------------------------------------------------------------------------------------------- +def resolve_ipywidgets_version(py_version, jl_version): + """Resolve ipywidgets version based on Python, JupyterLab versions. + + Returns a version spec string like ">=7.0.0,<8.0.0" or a pinned version. + """ + + # Simple heuristic: ipywidgets 8.x for modern stacks, 7.x for older + try: + py_major, py_minor = map(int, py_version.split(".")[:2]) + jl_major = int(jl_version.split(".")[0]) + + if jl_major >= 4 and py_major >= 3 and py_minor >= 9: + return "8.1.7" # Latest stable for modern Python + JupyterLab 4 + elif py_major >= 3 and py_minor >= 7: + return "8.1.6" # Good for Python 3.7+ + else: + return "7.7.2" # Fallback for older environments + except Exception: + return ">=7.0.0" # Safe fallback + + +def _has_package(ctx: RequirementContext, package: str) -> bool: + return any(name == package for name, _ in ctx.groups) + + +def _add_package(ctx: RequirementContext, name: str, spec: str) -> None: + ctx.groups[(canonicalize_name(name), "")] = [Entry(ADDED_BY_SCRIPT, f"{name}{spec}", name, (), spec, "")] + + +def determine_jupyterlab_version(resolved: list[Resolved], default: str = "4.4.0") -> str: + for item in resolved: + if canonicalize_name(item.name) == "jupyterlab" and not item.marker and item.spec.startswith("==") and "," not in item.spec: + return item.spec[2:] + return default + + +def ensure_jl_hidecode(ctx: RequirementContext): + if _has_package(ctx, "jl-hidecode"): + return + _add_package(ctx, "jl-hidecode", "==0.0.1") + print("Added jl-hidecode==0.0.1 (enforced)") + + +def ensure_ipywidgets(ctx: RequirementContext, python_version: str | None, jl_ver: str) -> bool: + if _has_package(ctx, "ipywidgets"): + return False + iw_version = resolve_ipywidgets_version(python_version, jl_ver) + spec = iw_version if iw_version[:1] in "<>=!~" else f"=={iw_version}" + _add_package(ctx, "ipywidgets", spec) + print(f"Added ipywidgets{spec} (resolved)") + return True + + +def build_final_lines(ctx: RequirementContext, resolved: list[Resolved], sort: bool) -> list[str]: + final_lines: list[str] = list(ctx.other_lines.keys()) + items = sorted(resolved, key=lambda item: item.name.casefold()) if sort else resolved + final_lines.extend(item.line() for item in items) + return final_lines + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Output files +# -------------------------------------------------------------------------------------------------------------------------------- +def get_os_output_path(output: Path, os_name: str) -> Path: + suffix = output.suffix + stem = output.stem + return output.with_name(f"{stem}-{os_name}{suffix}") + + +def get_gpu_output_path(output: Path) -> Path: + suffix = output.suffix + stem = output.stem + return output.with_name(f"{stem}_gpu{suffix}") + + +def merge_requirement_extras(extras: str, extra_name: str) -> str: + if not extras: + return f"[{extra_name}]" + + extras_content = extras[1:-1] + parts = [part.strip() for part in extras_content.split(",") if part.strip()] + if extra_name not in parts: + parts.append(extra_name) + return "[" + ",".join(parts) + "]" + + +def rewrite_tensorflow_for_gpu(line: str) -> str: + match = pkg_re.match(line) + if match: + name = match.group("name") + if name.lower() != "tensorflow": + return line + extras = merge_requirement_extras(match.group("extras") or "", "and-cuda") + marker = match.group("marker") or "" + return f"{name}{extras}{match.group('op')}{match.group('ver')}{marker}" + + match = PLAIN_PKG_RE.match(line) + if match and match.group("name").lower() == "tensorflow": + extras = merge_requirement_extras(match.group("extras") or "", "and-cuda") + marker = match.group("marker") or "" + return f"{match.group('name')}{extras}{marker}" + + return line + + +# TensorFlow's `and-cuda` extra pulls nvidia-* wheels that exist only for Linux (nvidia-nccl-cu12 has never had a Windows or macOS +# wheel), and the extra itself carries no platform marker. Without a marker the universal lock of requirements_gpu.txt pins them for +# every system, so `pip install` fails on a Windows computer with an NVIDIA driver (the installer picks the GPU file there). TensorFlow +# has had no native Windows GPU build since 2.10, so Windows and macOS keep plain `tensorflow` in the GPU file. +GPU_TENSORFLOW_PLATFORM = 'sys_platform == "linux"' +NON_GPU_TENSORFLOW_PLATFORM = 'sys_platform != "linux"' + + +def _with_platform(marker: str, platform: str) -> str: + """Return ` ; `, combined with the line's own marker (`; `) when it has one.""" + existing = marker.split(";", 1)[1].strip() if ";" in marker else "" + return f" ; ({existing}) and {platform}" if existing else f" ; {platform}" + + +def tensorflow_gpu_lines(line: str) -> list[str]: + """Return the GPU-file lines for a tensorflow requirement: `[and-cuda]` on Linux, the plain requirement elsewhere. + + A line that is not a tensorflow requirement is returned unchanged (as a one-item list). + """ + match = pkg_re.match(line) + if match: + if match.group("name").lower() != "tensorflow": + return [line] + name, extras = match.group("name"), match.group("extras") or "" + spec = f"{match.group('op')}{match.group('ver')}" + else: + match = PLAIN_PKG_RE.match(line) + if not match or match.group("name").lower() != "tensorflow": + return [line] + name, extras, spec = match.group("name"), match.group("extras") or "", "" + marker = match.group("marker") or "" + cuda_extras = merge_requirement_extras(extras, "and-cuda") + return [ + f"{name}{cuda_extras}{spec}{_with_platform(marker, GPU_TENSORFLOW_PLATFORM)}", + f"{name}{extras}{spec}{_with_platform(marker, NON_GPU_TENSORFLOW_PLATFORM)}", + ] + + +def build_gpu_lines(final_lines: list[str]) -> list[str]: + gpu_lines: list[str] = [] + has_torch_package = False + has_torch_index = False + + for line in final_lines: + if "download.pytorch.org/whl" in line: + has_torch_index = True + + match = pkg_re.match(line) or PLAIN_PKG_RE.match(line) + if match: + package_name = match.group("name").lower() + if package_name in TORCH_GPU_PACKAGES: + has_torch_package = True + if package_name == "tensorflow": + gpu_lines.extend(tensorflow_gpu_lines(line)) + continue + + gpu_lines.append(line) + + if has_torch_package and not has_torch_index: + insertion_index = 0 + while insertion_index < len(gpu_lines) and gpu_lines[insertion_index].startswith("-"): + insertion_index += 1 + gpu_lines.insert(insertion_index, GPU_TORCH_EXTRA_INDEX_URL) + + return gpu_lines + + +def write_requirements_output( + path: Path, + files: list[Path], + final_lines: list[str], + target_os: str | None = None, + target_accelerator: str | None = None, + decisions: list[str] | None = None, +): + generated = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + header_lines = [ + f"# Generated by .tools/python/merge_requirements.py on {generated}Z", + f"# Source files: {', '.join(str(p) for p in files)}", + ] + if target_os: + header_lines.append(f"# Target OS: {OS_DISPLAY_NAMES.get(target_os, target_os.capitalize())}") + if target_accelerator: + header_lines.append(f"# Target accelerator: {target_accelerator}") + header_lines.extend(decisions or []) + header_lines.append("# ---") + + content = "\n".join(header_lines) + "\n" + if final_lines: + content += "\n".join(final_lines) + if not content.endswith("\n"): + content += "\n" + + path.write_text(content, encoding="utf-8") + + +def main(argv: list[str] | None = None): + parser = argparse.ArgumentParser(description="Merge multiple requirements files (txt or yaml) into one.") + parser.add_argument("--source-dir", default="notebooks", help="Directory to scan for requirements files") + parser.add_argument("--output", default="requirements.txt", help="Output requirements file to write") + parser.add_argument("--sort", action="store_true", help="Sort final requirements alphabetically") + parser.add_argument("--overrides", help=f"Decisions for conflicting requirements (default: {OVERRIDES_FILE} next to the output)") + parser.add_argument("--check", action="store_true", help="Only check that the notebooks' requirements do not conflict; write nothing") + args = parser.parse_args(argv) + + source = Path(args.source_dir) + output = Path(args.output) + overrides_path = Path(args.overrides) if args.overrides else output.with_name(OVERRIDES_FILE) + + discovered_files = find_requirements_files(source) + + skip_outputs = {output.resolve()} + skip_outputs.add(get_gpu_output_path(output).resolve()) + for os_name in OS_FIELD_MAP: + skip_outputs.add(get_os_output_path(output, os_name).resolve()) + + files = [f for f in discovered_files if f.resolve() not in skip_outputs] + if not files: + if args.check: + print(f"No requirements files found under {source}: nothing to check.") + return 0 + print(f"No requirements files found under {source}, writing empty {output}") + output.write_text("# Generated requirements (none found)\n", encoding="utf-8") + gpu_output = get_gpu_output_path(output) + gpu_output.write_text("# Generated GPU requirements (none found)\n", encoding="utf-8") + return 0 + + base_ctx = RequirementContext() + os_contexts = {name: RequirementContext() for name in OS_FIELD_MAP} + python_version = None + + for f in files: + if f.suffix.lower() in {".yaml", ".yml"}: + candidate_py = read_requirements_yaml(f, base_ctx, os_contexts) + python_version = merge_python_version(f, python_version, candidate_py) + else: + read_requirements_file(f, base_ctx) + + try: + overrides = load_overrides(overrides_path) + except ValueError as exc: + print(f"{exc}\nSee {DOCS_URL}", file=sys.stderr) + return 1 + + ensure_jl_hidecode(base_ctx) + conflicts: list[Conflict] = [] + base_resolved = resolve_context(base_ctx, overrides, conflicts) + jl_ver = determine_jupyterlab_version(base_resolved) + if ensure_ipywidgets(base_ctx, python_version, jl_ver): + key = next(reversed(base_ctx.groups)) + base_resolved.append(resolve_group(key[0], base_ctx.groups[key], [], overrides.get(key[0]))) + os_resolved = { + os_name: resolve_context(ctx, overrides, conflicts, base=base_ctx, where=f"{OS_FIELD_MAP[os_name]}, with the common dependencies") + for os_name, ctx in os_contexts.items() + } + errors = base_ctx.errors + [error for ctx in os_contexts.values() for error in ctx.errors] + if conflicts or errors: + print(conflict_report(conflicts, errors, overrides_path), file=sys.stderr) + return 1 + for override in overrides.values(): + if not override.used: + print(f"Note: {display_path(overrides_path)} decides {override.name}, which no notebook requires any more; the entry can be removed.") + if args.check: + print(f"The requirements of {len(files)} file(s) do not conflict" + + (f" ({sum(o.used for o in overrides.values())} decision(s) from {display_path(overrides_path)})." if overrides else ".")) + return 0 + + decisions = override_header(base_resolved) + base_final_lines = build_final_lines(base_ctx, base_resolved, args.sort) + write_requirements_output(output, files, base_final_lines, decisions=decisions) + print(f"Wrote {output} with {len(base_final_lines)} entries (from {len(files)} files)") + + gpu_output = get_gpu_output_path(output) + gpu_final_lines = build_gpu_lines(base_final_lines) + write_requirements_output( + gpu_output, + files, + gpu_final_lines, + target_accelerator=GPU_DISPLAY_NAME, + decisions=decisions, + ) + print(f"Wrote {gpu_output} with {len(gpu_final_lines)} entries (from {len(files)} files, target {GPU_DISPLAY_NAME})") + + for os_name, ctx in os_contexts.items(): + if not ctx.groups and not ctx.other_lines: + continue + os_final_lines = build_final_lines(ctx, os_resolved[os_name], args.sort) + os_output = get_os_output_path(output, os_name) + write_requirements_output(os_output, files, os_final_lines, target_os=os_name, decisions=override_header(os_resolved[os_name])) + print(f"Wrote {os_output} with {len(os_final_lines)} entries (from {len(files)} files, target {os_name})") + + # Create the environment.yaml if not present + env_path = Path("environment.yaml") + if not env_path.exists(): + print(f"{env_path} not found, please be sure to create one.") + raise FileNotFoundError(f"{env_path} not found.") + else: + data = yaml.safe_load(env_path.read_text(encoding="utf-8")) + dependencies = data.get("dependencies", []) + python_updated = False + jl_updated = False + for i, dep in enumerate(dependencies): + if not isinstance(dep, str): + continue + if python_version and dep.startswith("python="): + dependencies[i] = f"python={python_version}" + python_updated = True + if dep.startswith("jupyterlab="): + dependencies[i] = f"jupyterlab={jl_ver}" + jl_updated = True + data["dependencies"] = dependencies + env_path.write_text(yaml.dump(data, sort_keys=False), encoding="utf-8") + if python_updated or jl_updated: + parts = [] + if python_updated: + parts.append(f"python={python_version}") + if jl_updated: + parts.append(f"jupyterlab={jl_ver}") + print(f"Updated {env_path} with {', '.join(parts)}") + else: + print(f"No python/jupyterlab entries updated in {env_path}") + + # Replace on setup.py the Python version + setup_path = Path("setup.py") + if setup_path.exists() and python_version: + setup_text = setup_path.read_text(encoding="utf-8") + # Require only the major.minor series: environment.yaml pins the exact + # patch release for the installer, but a source install should accept + # any patch release of that series (e.g. 3.12.3 for 3.12.13). + python_series = ".".join(str(python_version).split(".")[:2]) + pattern = r'python_requires\s*=\s*["\'][><=\.0-9 ]+(PYTHON\_VERSION)?["\']' + new_setup_text = re.sub(pattern, f'python_requires=">={python_series}"', setup_text) + setup_path.write_text(new_setup_text, encoding="utf-8") + print(f"Updated {setup_path} with python_requires>={python_series}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.template_sync/resources/v0_1_29_to_v0_1_30/previous/.tools/python/merge_requirements.py b/.template_sync/resources/v0_1_29_to_v0_1_30/previous/.tools/python/merge_requirements.py new file mode 100644 index 0000000..1fccecb --- /dev/null +++ b/.template_sync/resources/v0_1_29_to_v0_1_30/previous/.tools/python/merge_requirements.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +"""Merge requirements files under a directory (e.g. notebooks/*) into a single +root requirements file. + +The script now supports both classic requirements.txt files and the new +requirements.yaml format used by notebooks. For YAML files, it merges the +`python_version` (picking the newest) and the `dependencies` list while +discarding per-notebook descriptions. + +It also recognizes optional per-OS fields (`windows_dependencies`, +`linux_dependencies`, `macos_dependencies`) and emits dedicated requirements +outputs for each platform containing only the entries from their OS-specific +lists, in addition to the common file. + +Alongside the common requirements file, it emits a GPU-targeted variant. The +GPU file rewrites `tensorflow` to `tensorflow[and-cuda]` on Linux only (TensorFlow's CUDA +wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the +PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present. + +It resolves simple version conflicts by choosing the newest version seen for a +given package (and emits a pinned `pkg==`). It still preserves +non-package lines (pip options, VCS/URLs) and supports simple `-r` includes. + +It also automatically adds ipywidgets if missing, with a version determined by +Python, JupyterLab, and matplotlib versions from environment.yaml. +""" +from pathlib import Path +import argparse +import sys +from collections import OrderedDict +from datetime import datetime +import re +import yaml + + +# Regex to capture name, optional extras, operator and version, and optional markers +pkg_re = re.compile(r"^(?P[A-Za-z0-9_.+-]+)(?P\[[^\]]+\])?\s*(?P==|>=|<=|~=|!=|>|<)\s*(?P[^;\s]+)(?P\s*;.*)?$") + +OS_FIELD_MAP = { + "windows": "windows_dependencies", + "linux": "linux_dependencies", + "macos": "macos_dependencies", +} +OS_DISPLAY_NAMES = { + "windows": "Windows", + "linux": "Linux", + "macos": "macOS", +} +GPU_DISPLAY_NAME = "NVIDIA GPU" +GPU_TORCH_EXTRA_INDEX_URL = "--extra-index-url https://download.pytorch.org/whl/cu128" +TORCH_GPU_PACKAGES = {"torch", "torchvision", "torchaudio"} +PLAIN_PKG_RE = re.compile( + r"^(?P[A-Za-z0-9_.+-]+)(?P\[[^\]]+\])?(?P\s*;.*)?$" +) + + +class RequirementContext: + def __init__(self): + self.pkg_order: list[str] = [] + self.pkgs: dict[str, dict] = {} + self.other_lines: OrderedDict[str, None] = OrderedDict() + + +def normalize_line(line: str) -> str: + # Remove inline comments and surrounding whitespace. + if "#" in line: + line = line.split("#", 1)[0] + return line.strip() + + +try: + # Prefer packaging.version for robust version comparison + from packaging.version import Version, InvalidVersion + + def parse_version(v: str): + try: + return Version(v) + except InvalidVersion: + return v + + def is_version_greater(a, b): + try: + return Version(a) > Version(b) + except Exception: + return str(a) > str(b) + +except Exception: + # Fallback: simple numeric-aware comparator + def _split_parts(v: str): + parts = re.split(r"[.\-+_]|(?=[a-zA-Z])", v) + key = [] + for p in parts: + if not p: + continue + if p.isdigit(): + key.append((0, int(p))) + else: + key.append((1, p)) + return tuple(key) + + def parse_version(v: str): + return _split_parts(v) + + def is_version_greater(a, b): + return parse_version(a) > parse_version(b) + + +def merge_python_version(filepath: str | None, current: str | None, candidate: str | None): + """Pick the newest python_version string, if provided.""" + if not candidate: + return current + if current is None: + return candidate + if current == candidate: + return current + else: + raise ValueError(f"Conflicting python_version found {current} vs {candidate} when processing {filepath}") + + +def process_requirement_line(line: str, pkg_order: list, pkgs: dict, other_lines: OrderedDict, base_dir: Path | None = None): + """Handle a single requirement line (shared between txt and yaml inputs).""" + if not line: + return + + parts = line.split(maxsplit=1) + if parts[0] in ("-r", "--requirement") and len(parts) == 2: + include_path = Path(parts[1]) if base_dir is None else (base_dir / parts[1]) + include = include_path.resolve() + if include.exists(): + read_requirements_file(include, pkg_order, pkgs, other_lines) + return + + # Keep pip option lines (indexes, find-links, etc.) as-is + if line.startswith("--") or line.startswith("-f ") or line.startswith("-i ") or line.startswith("-e "): + other_lines.setdefault(line, None) + return + + # VCS or URL lines (git+, http(s)://, file:) keep as-is + if any(line.startswith(prefix) for prefix in ("git+", "http://", "https://", "file:", "ssh://")): + other_lines.setdefault(line, None) + return + + m = pkg_re.match(line) + if not m: + # plain package (unpinned) or marker-only spec + key = line.split(";", 1)[0].strip().lower() + if key not in pkgs: + pkgs[key] = {"name": line, "pinned": False, "ver": None} + pkg_order.append(key) + return + + name = m.group("name") + extras = m.group("extras") or "" + ver = m.group("ver") + marker = m.group("marker") or "" + + base = name.lower() + if base not in pkgs: + pkgs[base] = {"name": name + (extras or ""), "pinned": True, "ver": ver, "marker": marker} + pkg_order.append(base) + else: + existing = pkgs[base] + if not existing.get("pinned"): + pkgs[base] = {"name": name + (extras or ""), "pinned": True, "ver": ver, "marker": marker} + else: + try: + if is_version_greater(ver, existing.get("ver")): + pkgs[base] = {"name": name + (extras or ""), "pinned": True, "ver": ver, "marker": marker} + except Exception: + if str(ver) > str(existing.get("ver")): + pkgs[base] = {"name": name + (extras or ""), "pinned": True, "ver": ver, "marker": marker} + + +def read_requirements_file(path: Path, pkg_order: list, pkgs: dict, other_lines: OrderedDict): + try: + text = path.read_text(encoding="utf-8") + except Exception: + return + + for raw in text.splitlines(): + line = normalize_line(raw) + process_requirement_line(line, pkg_order, pkgs, other_lines, base_dir=path.parent) + + +def read_requirements_yaml(path: Path, pkg_order: list, pkgs: dict, other_lines: OrderedDict, os_contexts: dict[str, RequirementContext]) -> str | None: + """Read requirements.yaml and return its python_version if present.""" + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except Exception: + return None + + python_version = data.get("python_version") + dependencies = data.get("dependencies", []) or [] + + for dep in dependencies: + line = normalize_line(str(dep)) + process_requirement_line(line, pkg_order, pkgs, other_lines, base_dir=path.parent) + + for os_name, field in OS_FIELD_MAP.items(): + os_deps = data.get(field) or [] + if not os_deps: + continue + ctx = os_contexts.get(os_name) + if ctx is None: + continue + for dep in os_deps: + line = normalize_line(str(dep)) + process_requirement_line(line, ctx.pkg_order, ctx.pkgs, ctx.other_lines, base_dir=path.parent) + + return python_version + + +def find_requirements_files(source_dir): + if not source_dir.exists(): + return [] + + patterns = ["requirements.yaml", "requirements.yml", "requirements*.txt"] + + files = [] + for pattern in patterns: + files.extend(source_dir.rglob(pattern)) + return sorted({f.resolve() for f in files}) + + +def resolve_ipywidgets_version(py_version, jl_version): + """Resolve ipywidgets version based on Python, JupyterLab versions. + + Returns a version spec string like ">=7.0.0,<8.0.0" or a pinned version. + """ + + # Simple heuristic: ipywidgets 8.x for modern stacks, 7.x for older + try: + py_major, py_minor = map(int, py_version.split(".")[:2]) + jl_major = int(jl_version.split(".")[0]) + + if jl_major >= 4 and py_major >= 3 and py_minor >= 9: + return "8.1.7" # Latest stable for modern Python + JupyterLab 4 + elif py_major >= 3 and py_minor >= 7: + return "8.1.6" # Good for Python 3.7+ + else: + return "7.7.2" # Fallback for older environments + except Exception: + return ">=7.0.0" # Safe fallback + + +def determine_jupyterlab_version(pkgs: dict, default: str = "4.4.0") -> str: + jl_info = pkgs.get("jupyterlab") + if jl_info and jl_info.get("pinned") and jl_info.get("ver"): + return jl_info["ver"] + return default + + +def ensure_jl_hidecode(ctx: RequirementContext): + if "jl-hidecode" in ctx.pkgs: + return + ctx.pkgs["jl-hidecode"] = { + "name": "jl-hidecode", + "pinned": True, + "ver": "0.0.1", + "marker": "", + } + ctx.pkg_order.append("jl-hidecode") + print("Added jl-hidecode==0.0.1 (enforced)") + + +def ensure_ipywidgets(ctx: RequirementContext, python_version: str | None, jl_ver: str): + if "ipywidgets" in ctx.pkgs: + return + iw_version = resolve_ipywidgets_version(python_version, jl_ver) + ctx.pkgs["ipywidgets"] = { + "name": "ipywidgets", + "pinned": True, + "ver": iw_version, + "marker": "", + } + ctx.pkg_order.append("ipywidgets") + print(f"Added ipywidgets=={iw_version} (resolved)") + + +def build_final_lines(ctx: RequirementContext, sort: bool) -> list[str]: + final_lines: list[str] = [] + for ol in ctx.other_lines.keys(): + final_lines.append(ol) + + pkg_keys = sorted(ctx.pkg_order, key=str.casefold) if sort else ctx.pkg_order + + for key in pkg_keys: + info = ctx.pkgs.get(key) + if not info: + continue + if info.get("pinned") and info.get("ver"): + final_lines.append(f"{info['name']}=={info['ver']}{info.get('marker','')}") + else: + final_lines.append(info["name"]) + return final_lines + + +def get_os_output_path(output: Path, os_name: str) -> Path: + suffix = output.suffix + stem = output.stem + return output.with_name(f"{stem}-{os_name}{suffix}") + + +def get_gpu_output_path(output: Path) -> Path: + suffix = output.suffix + stem = output.stem + return output.with_name(f"{stem}_gpu{suffix}") + + +def merge_requirement_extras(extras: str, extra_name: str) -> str: + if not extras: + return f"[{extra_name}]" + + extras_content = extras[1:-1] + parts = [part.strip() for part in extras_content.split(",") if part.strip()] + if extra_name not in parts: + parts.append(extra_name) + return "[" + ",".join(parts) + "]" + + +def rewrite_tensorflow_for_gpu(line: str) -> str: + match = pkg_re.match(line) + if match: + name = match.group("name") + if name.lower() != "tensorflow": + return line + extras = merge_requirement_extras(match.group("extras") or "", "and-cuda") + marker = match.group("marker") or "" + return f"{name}{extras}{match.group('op')}{match.group('ver')}{marker}" + + match = PLAIN_PKG_RE.match(line) + if match and match.group("name").lower() == "tensorflow": + extras = merge_requirement_extras(match.group("extras") or "", "and-cuda") + marker = match.group("marker") or "" + return f"{match.group('name')}{extras}{marker}" + + return line + + +# TensorFlow's `and-cuda` extra pulls nvidia-* wheels that exist only for Linux (nvidia-nccl-cu12 has never had a Windows or macOS +# wheel), and the extra itself carries no platform marker. Without a marker the universal lock of requirements_gpu.txt pins them for +# every system, so `pip install` fails on a Windows computer with an NVIDIA driver (the installer picks the GPU file there). TensorFlow +# has had no native Windows GPU build since 2.10, so Windows and macOS keep plain `tensorflow` in the GPU file. +GPU_TENSORFLOW_PLATFORM = 'sys_platform == "linux"' +NON_GPU_TENSORFLOW_PLATFORM = 'sys_platform != "linux"' + + +def _with_platform(marker: str, platform: str) -> str: + """Return ` ; `, combined with the line's own marker (`; `) when it has one.""" + existing = marker.split(";", 1)[1].strip() if ";" in marker else "" + return f" ; ({existing}) and {platform}" if existing else f" ; {platform}" + + +def tensorflow_gpu_lines(line: str) -> list[str]: + """Return the GPU-file lines for a tensorflow requirement: `[and-cuda]` on Linux, the plain requirement elsewhere. + + A line that is not a tensorflow requirement is returned unchanged (as a one-item list). + """ + match = pkg_re.match(line) + if match: + if match.group("name").lower() != "tensorflow": + return [line] + name, extras = match.group("name"), match.group("extras") or "" + spec = f"{match.group('op')}{match.group('ver')}" + else: + match = PLAIN_PKG_RE.match(line) + if not match or match.group("name").lower() != "tensorflow": + return [line] + name, extras, spec = match.group("name"), match.group("extras") or "", "" + marker = match.group("marker") or "" + cuda_extras = merge_requirement_extras(extras, "and-cuda") + return [ + f"{name}{cuda_extras}{spec}{_with_platform(marker, GPU_TENSORFLOW_PLATFORM)}", + f"{name}{extras}{spec}{_with_platform(marker, NON_GPU_TENSORFLOW_PLATFORM)}", + ] + + +def build_gpu_lines(final_lines: list[str]) -> list[str]: + gpu_lines: list[str] = [] + has_torch_package = False + has_torch_index = False + + for line in final_lines: + if "download.pytorch.org/whl" in line: + has_torch_index = True + + match = pkg_re.match(line) or PLAIN_PKG_RE.match(line) + if match: + package_name = match.group("name").lower() + if package_name in TORCH_GPU_PACKAGES: + has_torch_package = True + if package_name == "tensorflow": + gpu_lines.extend(tensorflow_gpu_lines(line)) + continue + + gpu_lines.append(line) + + if has_torch_package and not has_torch_index: + insertion_index = 0 + while insertion_index < len(gpu_lines) and gpu_lines[insertion_index].startswith("-"): + insertion_index += 1 + gpu_lines.insert(insertion_index, GPU_TORCH_EXTRA_INDEX_URL) + + return gpu_lines + + +def write_requirements_output( + path: Path, + files: list[Path], + final_lines: list[str], + target_os: str | None = None, + target_accelerator: str | None = None, +): + header_lines = [ + f"# Generated by .tools/python/merge_requirements.py on {datetime.utcnow().isoformat()}Z", + f"# Source files: {', '.join(str(p) for p in files)}", + ] + if target_os: + header_lines.append(f"# Target OS: {OS_DISPLAY_NAMES.get(target_os, target_os.capitalize())}") + if target_accelerator: + header_lines.append(f"# Target accelerator: {target_accelerator}") + header_lines.append("# ---") + + content = "\n".join(header_lines) + "\n" + if final_lines: + content += "\n".join(final_lines) + if not content.endswith("\n"): + content += "\n" + + path.write_text(content, encoding="utf-8") + + +def main(): + parser = argparse.ArgumentParser(description="Merge multiple requirements files (txt or yaml) into one.") + parser.add_argument("--source-dir", default="notebooks", help="Directory to scan for requirements files") + parser.add_argument("--output", default="requirements.txt", help="Output requirements file to write") + parser.add_argument("--sort", action="store_true", help="Sort final requirements alphabetically") + args = parser.parse_args() + + source = Path(args.source_dir) + output = Path(args.output) + + discovered_files = find_requirements_files(source) + + skip_outputs = {output.resolve()} + skip_outputs.add(get_gpu_output_path(output).resolve()) + for os_name in OS_FIELD_MAP: + skip_outputs.add(get_os_output_path(output, os_name).resolve()) + + files = [f for f in discovered_files if f.resolve() not in skip_outputs] + if not files: + print(f"No requirements files found under {source}, writing empty {output}") + output.write_text("# Generated requirements (none found)\n", encoding="utf-8") + gpu_output = get_gpu_output_path(output) + gpu_output.write_text("# Generated GPU requirements (none found)\n", encoding="utf-8") + return 0 + + base_ctx = RequirementContext() + os_contexts = {name: RequirementContext() for name in OS_FIELD_MAP} + python_version = None + + for f in files: + if f.suffix.lower() in {".yaml", ".yml"}: + candidate_py = read_requirements_yaml(f, base_ctx.pkg_order, base_ctx.pkgs, base_ctx.other_lines, os_contexts) + python_version = merge_python_version(f, python_version, candidate_py) + else: + read_requirements_file(f, base_ctx.pkg_order, base_ctx.pkgs, base_ctx.other_lines) + + ensure_jl_hidecode(base_ctx) + jl_ver = determine_jupyterlab_version(base_ctx.pkgs) + ensure_ipywidgets(base_ctx, python_version, jl_ver) + + base_final_lines = build_final_lines(base_ctx, args.sort) + write_requirements_output(output, files, base_final_lines) + print(f"Wrote {output} with {len(base_final_lines)} entries (from {len(files)} files)") + + gpu_output = get_gpu_output_path(output) + gpu_final_lines = build_gpu_lines(base_final_lines) + write_requirements_output( + gpu_output, + files, + gpu_final_lines, + target_accelerator=GPU_DISPLAY_NAME, + ) + print(f"Wrote {gpu_output} with {len(gpu_final_lines)} entries (from {len(files)} files, target {GPU_DISPLAY_NAME})") + + for os_name, ctx in os_contexts.items(): + if not ctx.pkg_order and not ctx.other_lines: + continue + os_final_lines = build_final_lines(ctx, args.sort) + os_output = get_os_output_path(output, os_name) + write_requirements_output(os_output, files, os_final_lines, target_os=os_name) + print(f"Wrote {os_output} with {len(os_final_lines)} entries (from {len(files)} files, target {os_name})") + + # Create the environment.yaml if not present + env_path = Path("environment.yaml") + if not env_path.exists(): + print(f"{env_path} not found, please be sure to create one.") + raise FileNotFoundError(f"{env_path} not found.") + else: + data = yaml.safe_load(env_path.read_text(encoding="utf-8")) + dependencies = data.get("dependencies", []) + python_updated = False + jl_updated = False + for i, dep in enumerate(dependencies): + if not isinstance(dep, str): + continue + if python_version and dep.startswith("python="): + dependencies[i] = f"python={python_version}" + python_updated = True + if dep.startswith("jupyterlab="): + dependencies[i] = f"jupyterlab={jl_ver}" + jl_updated = True + data["dependencies"] = dependencies + env_path.write_text(yaml.dump(data, sort_keys=False), encoding="utf-8") + if python_updated or jl_updated: + parts = [] + if python_updated: + parts.append(f"python={python_version}") + if jl_updated: + parts.append(f"jupyterlab={jl_ver}") + print(f"Updated {env_path} with {', '.join(parts)}") + else: + print(f"No python/jupyterlab entries updated in {env_path}") + + # Replace on setup.py the Python version + setup_path = Path("setup.py") + if setup_path.exists() and python_version: + setup_text = setup_path.read_text(encoding="utf-8") + # Require only the major.minor series: environment.yaml pins the exact + # patch release for the installer, but a source install should accept + # any patch release of that series (e.g. 3.12.3 for 3.12.13). + python_series = ".".join(str(python_version).split(".")[:2]) + pattern = r'python_requires\s*=\s*["\'][><=\.0-9 ]+(PYTHON\_VERSION)?["\']' + new_setup_text = re.sub(pattern, f'python_requires=">={python_series}"', setup_text) + setup_path.write_text(new_setup_text, encoding="utf-8") + print(f"Updated {setup_path} with python_requires>={python_series}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tools/docs/migration_0_1_30.md b/.tools/docs/migration_0_1_30.md new file mode 100644 index 0000000..e2c7ac0 --- /dev/null +++ b/.tools/docs/migration_0_1_30.md @@ -0,0 +1,54 @@ +# Template 0.1.30: no silent change of a notebook's package versions, no lost work when a notebook is updated + +Design note for the 0.1.30 migration (`.template_sync/migrations/v0_1_29_to_v0_1_30.py`), issues [#48](https://github.com/CellMigrationLab/LabConstrictor/issues/48) and [#49](https://github.com/CellMigrationLab/LabConstrictor/issues/49). How a change reaches an installed copy is described in [migration_0_1_16.md](migration_0_1_16.md): a new installer is needed, nothing edits an existing install. + +## 1. The problems + +**#48, the requirements merge changed versions without a word.** `merge_requirements.py` (run by **Updates on notebook change**) merged the `requirements.yaml` of every notebook and, for a package required by several notebooks, kept the highest version: the policy of issue #4 ("if two versions are in conflict in principle take the newest one"). For scientific workflows a successful install does not mean the same results, so a notebook validated with `numpy==1.26.4` silently ran with `numpy==2.0.0`. Running it showed more than the code review that found it (all exited with 0): + +| Notebook A | Notebook B | Merged until 0.1.29 | +|---|---|---| +| `numpy==1.26.4` | `numpy==2.0.0` | `numpy==2.0.0`, in either file order | +| `numpy<2.0` | | `numpy==2.0`: every requirement was written as `==`, here the one excluded version | +| `numpy<2` | `numpy>=2.1` | `numpy==2.1` | +| `numpy==1.26.4 ; sys_platform == "win32"` | `numpy==2.0.0 ; sys_platform != "win32"` | only the second line: no numpy on Windows | + +NucleiSky's notebooks pin `cellpose`, `ipywidgets`, `scikit-image`, `tifffile`, `torch` and `zarr` to different versions, and LabChronicle's pin `tqdm` to two versions: they relied on this. The old merge also treated `scikit-image` and `scikit_image` as two packages (pip then refuses the file), and wrote `ipywidgets==>=7.0.0` when no notebook gave a Python version. + +**#49, the notebook update could overwrite the user's work.** The **Update** button of the Welcome notebook downloaded the new notebook and wrote it over the user's copy, with their code changes, parameters and saved outputs, with no question and no copy. A failed write could leave a broken notebook, and a private repository without a token crashed the button. + +## 2. What changes + +| Part | Change | +|---|---| +| `.tools/python/merge_requirements.py` | Requirements are kept as written and combined per package (`numpy>=1.24,<2`); names are compared as pip compares them. Different exact pins, a pin outside another notebook's range, or ranges no version meets stop the merge, before any file is written, with every notebook and requirement (sorted, so the file order cannot hide one). `requirements_overrides.yaml` can decide explicitly, with a required reason; each decision is written at the top of the generated files. Requirements with different markers stay separate lines. A notebook's OS-specific requirements are checked against the common ones. `--check` only looks for conflicts. | +| `.tools/requirements.txt` | `packaging==25.0` (versions and specifiers as pip reads them). | +| `requirements_overrides.yaml` (new, only where needed) | See section 3. | +| `.github/workflows/update_on_notebook_change.yml` | A push of `requirements_overrides.yaml` runs the workflow and the merge. | +| `.github/workflows/release_constructor.yml` | The bump-version job runs `merge_requirements.py --check` after `check_placeholders.py`: a release stops while a conflict is unresolved (a failed merge leaves the previous `requirements.txt` in place, which would otherwise be released). | +| `.tools/templates/Welcome_template.ipynb` | **Update** asks first, says where the copy goes and to close the notebook's tab; then downloads, checks that the download is a notebook, copies the current file to `.backup-.ipynb` (never overwriting an earlier backup), writes the new notebook to a temporary file and moves it into place, and only then records the new version. Cancel or any failure changes nothing. The app-version check that decides whether a notebook can be updated is unchanged. | + +Why not detect local changes: JupyterLab saves outputs into the notebook (and autosaves), so a used notebook always differs from the installed one; a check would ask every time anyway. Asking every time and always keeping a copy is simpler and loses nothing. + +Why a range check of our own: a lock failure is only a warning (the installer then installs the plain `requirements.txt`), so an impossible range would reach the users. `satisfiable()` tries every version a requirement names and versions just above it; a range only a closer version could meet is reported as a conflict (stop and decide), never guessed. + +## 3. What the migration does + +Every step is independent, idempotent, never deletes, never raises for a customised file (the sync pull request is still created) and keeps the line endings of each file. + +* **`merge_requirements.py`** is replaced by the 0.1.30 file when it is still the 0.1.29 template file; a customised one is left as it is (`SKIPPED`). +* **`.tools/requirements.txt`** gets `packaging==25.0` unless it installs packaging already. +* **`requirements_overrides.yaml`** is created only when the notebooks' `dependencies` already pin a package to different versions. Each entry keeps the version the merge chose until now (the newest pin), so the app installs what it installed before, and its reason says so and lists the pins: check each one and replace the reason with yours. An existing file is left as it is. +* **`update_on_notebook_change.yml`:** three known lines (push paths, changed files, the merge condition); a customised workflow is `SKIPPED` (run the workflow by hand after changing the overrides). +* **`release_constructor.yml`:** the check goes after the first `check_placeholders.py` command (the bump-version job), only when the repository's `merge_requirements.py` knows `--check`. +* **`Welcome_template.ipynb`:** the update handler of the notebook table is replaced (compared line by line, trailing spaces ignored) and the backup function added; a customised table cell is `SKIPPED`. The Welcome notebook of the next release is made from it. + +## 4. Adopting it + +Take this template version (the synchronization pull request). If it created `requirements_overrides.yaml`, check each decision with the notebooks that pin another version; change the notebooks or the decision as needed. Then release a new installer: users get the safe update with it. + +## 5. What is checked here, and what is not + +* Tests (`tests/test_template_sync_0_1_30.py`): the merge on the cases of #48 (both file orders, ranges, markers, names, OS-specific lists, overrides with and without a reason, `--check` writes nothing); the backup function (first install, backup, two updates in the same second, a download that is not a notebook); the migration on 0.1.29 files and on customised ones, the seeded overrides of a NucleiSky-like app and the merge accepting them, CRLF, a second run; the sync chain from 0.1.29. +* The migration and the merge were run on copies of the real apps. +* The confirmation dialog and the buttons of the Welcome notebook are not run by these tests: they need a person in an installed app. diff --git a/.tools/docs/notebook_requirements.md b/.tools/docs/notebook_requirements.md index a79d331..ac4a480 100644 --- a/.tools/docs/notebook_requirements.md +++ b/.tools/docs/notebook_requirements.md @@ -72,6 +72,28 @@ When you push a notebook with its `requirements.yaml`, the workflow **Updates on The installer uses `requirements_gpu.txt` when it finds an NVIDIA GPU on Windows or Linux. In that file `tensorflow` becomes `tensorflow[and-cuda]` on Linux only: TensorFlow has had no Windows GPU build since version 2.10, and its CUDA packages exist only for Linux, so Windows keeps plain `tensorflow` (CPU). +### When notebooks ask for different versions of a package + +The merge never changes a notebook's requirement silently. Requirements are kept as written (`numpy>=1.24` stays `numpy>=1.24`), and the requirements of several notebooks are combined (`numpy>=1.24,<2`). The workflow stops, and no requirements file is written, when: + +* two notebooks pin a package to different versions (`numpy==1.26.4` and `numpy==2.0.0`); +* a pin does not meet another notebook's requirement (`numpy==2.0.0` and `numpy<2`); +* no version meets all the requirements (`numpy<2` and `numpy>=2.1`). + +The log of the **Run merge script** step names the package, every notebook and its requirement. Then either: + +* **make the notebooks agree:** change their `requirements.yaml` to a version that works for every notebook; or +* **decide explicitly** in `requirements_overrides.yaml`, at the top of the repository, which requirement every notebook gets, and why: + +```yaml +overrides: + numpy: + use: "==2.0.0" + reason: "Notebook A was re-run with numpy 2.0.0 and gives the same results; notebook B needs 2.0." +``` + +The decision and its reason are written at the top of the generated requirements files. Pushing `requirements_overrides.yaml` runs the workflow again. A release also stops while a conflict is not resolved. Requirements with different platform markers (`; sys_platform == "win32"`) are kept as separate lines, and `NumPy`, `numpy` and `num_py` count as the same package, as for pip. + It then writes a **lock file** next to each of them (`requirements.lock.txt`, ...). A lock file lists every package the installer installs, including the packages your packages depend on, each with its exact version, for Windows, macOS and Linux. The installer installs the lock file, so every installation of a release gets the same versions. * A lock file changes only when its requirements file, the Python version or the JupyterLab version changes. Pushing a notebook without new requirements does not move any version. diff --git a/.tools/docs/notebook_upload.md b/.tools/docs/notebook_upload.md index 41a83d0..1341f14 100644 --- a/.tools/docs/notebook_upload.md +++ b/.tools/docs/notebook_upload.md @@ -152,7 +152,8 @@ Follow the steps in the [Obtain requirements of the notebook](notebook_requireme If you need help merging the pull request, see the [Accept a Pull Request](accept_pull_request.md) guide for instructions. An automatic workflow will check your notebook and its dependencies. * **Check Status:** See [How to check workflow status](workflow_status.md). -* Failure Protocol: If the workflow fails, for example, because of dependency conflicts, the system will automatically undo the merge. Check the logs, fix the problem in your local notebook or requirements, and upload again. +* Failure Protocol: If the merged requirements cannot be installed, the system automatically undoes the merge. Check the logs, fix the problem in your local notebook or requirements, and upload again. +* If notebooks ask for different versions of the same package, the workflow stops before anything is installed and the requirements files are not changed. See [When notebooks ask for different versions of a package](notebook_requirements.md#when-notebooks-ask-for-different-versions-of-a-package). --- diff --git a/.tools/docs/notebook_usage.md b/.tools/docs/notebook_usage.md index c4e2c01..2f86a28 100644 --- a/.tools/docs/notebook_usage.md +++ b/.tools/docs/notebook_usage.md @@ -16,6 +16,10 @@ After some seconds, Jupyter Lab will open in your default web browser. The welco ![Welcome Notebook](https://github.com/CellMigrationLab/LabConstrictor/blob/doc_source/Welcome_Notebook.png) +### Updating a notebook + +When a newer version of a notebook is published, the Welcome notebook shows **Update** next to it. Updating replaces your copy of the notebook, so before anything changes it asks you to confirm, and it keeps your current copy, with your changes and outputs, as `.backup-.ipynb` in the same folder. Close the notebook's tab before updating: otherwise JupyterLab can save the old copy over the new one. If the download fails, your notebook is not changed. + ## 3. How to Run Notebooks? You might be used to work with notebooks on JupyterLab (in case you need some guidance on JupyterLab check [this guide](https://github.com/CellMigrationLab/docs-hub/blob/main/docs/JupyterLab/JupyterLab_Guide.md)). Still, LabConstrictor uses some features that can slightly change how you interact with notebooks. Please check [this guide](code_hiding.md) to understand how to run notebooks packaged with LabConstrictor. diff --git a/.tools/docs/troubleshooting.md b/.tools/docs/troubleshooting.md index acdff6c..d8f142c 100644 --- a/.tools/docs/troubleshooting.md +++ b/.tools/docs/troubleshooting.md @@ -38,6 +38,12 @@ The log of the failed step lists each file and line. Replace the placeholder the The `platforms` job of the release workflow reads `distribution.platforms` in `labconstrictor-app.yaml` to decide which installers to build. It stops when the file is not valid YAML, when `platforms` is not a list, or when it lists none of the systems the release builds installers for: `windows-x86_64`, `linux-x86_64`, `macos-arm64` and `macos-x86_64`. The log of the step says which. Fix `labconstrictor-app.yaml` (see [App Centre manifest](app_manifest.md)), commit, then publish the release again. See [migration_0_1_24.md](migration_0_1_24.md). +## "The notebooks ask for incompatible package versions" + +The **Run merge script** step of **Updates on notebook change**, or the `merge_requirements.py --check` command of a release, stops when notebooks pin a package to different versions, or ask for versions that no single version meets. Its log lists the package, each notebook and its requirement. Make the notebooks agree, or decide in `requirements_overrides.yaml` which requirement every notebook gets and why, then push it: the workflow runs again. See [When notebooks ask for different versions of a package](notebook_requirements.md#when-notebooks-ask-for-different-versions-of-a-package) and [migration_0_1_30.md](migration_0_1_30.md). + +If the template synchronization to 0.1.30 created `requirements_overrides.yaml`, its entries keep the versions your app installed until now (the newest of the pins). Check each one with every notebook that uses the package, and replace its reason with yours. + ## Publishing to PyPI fails The workflow **Publish the Python package to PyPI** (`publish_pypi.yml`) runs at a release only when the repository variable `PYPI_PUBLISH` is `true` or `testpypi` (see [Upload external code](external_code_upload.md#4-publish-the-code-on-pypi-optional)). Its first job checks and builds the package; its second job uploads it. The installers do not depend on it: a failure here leaves the release and its installers as they are. diff --git a/.tools/docs/troubleshooting_installed_app.md b/.tools/docs/troubleshooting_installed_app.md index a6b8d11..b5d5193 100644 --- a/.tools/docs/troubleshooting_installed_app.md +++ b/.tools/docs/troubleshooting_installed_app.md @@ -33,6 +33,10 @@ If someone once ran `python -m ipykernel install --user` on the computer (common Install a release made with template 0.1.24 or later: its launcher always uses the app's own kernel. Until then, removing that personal kernel folder also works, but it affects the other Jupyter installations that use it. See [migration_0_1_24.md](migration_0_1_24.md). +## My changes to a notebook are gone after an update + +Since template 0.1.30, the **Update** button of the Welcome notebook keeps your copy before it replaces a notebook: look for `.backup-.ipynb` in the same folder, and open it in JupyterLab. Apps made with template 0.1.29 or earlier replace the notebook without a copy. + ## JupyterLab does not start because Windows reports an ASN.1 certificate error LabConstrictor-based installers include a TLS-resilient launcher. It attempts diff --git a/.tools/python/merge_requirements.py b/.tools/python/merge_requirements.py index 1fccecb..37fc5fe 100644 --- a/.tools/python/merge_requirements.py +++ b/.tools/python/merge_requirements.py @@ -2,9 +2,9 @@ """Merge requirements files under a directory (e.g. notebooks/*) into a single root requirements file. -The script now supports both classic requirements.txt files and the new +The script supports both classic requirements.txt files and the requirements.yaml format used by notebooks. For YAML files, it merges the -`python_version` (picking the newest) and the `dependencies` list while +`python_version` (every notebook must agree) and the `dependencies` list while discarding per-notebook descriptions. It also recognizes optional per-OS fields (`windows_dependencies`, @@ -17,23 +17,40 @@ wheels exist only for Linux; other systems keep plain `tensorflow`) and adds the PyTorch CUDA wheel index when `torch`, `torchvision`, or `torchaudio` are present. -It resolves simple version conflicts by choosing the newest version seen for a -given package (and emits a pinned `pkg==`). It still preserves -non-package lines (pip options, VCS/URLs) and supports simple `-r` includes. +Version requirements are never changed silently (template 0.1.30, CellMigrationLab/LabConstrictor#48): + * the requirements of a package are kept as written: `numpy>=1.24` stays `numpy>=1.24`, and the + requirements of several notebooks are combined (`numpy<2,>=1.24`); the lock step then finds the versions; + * when notebooks pin a package to different versions (`numpy==1.26.4` and `numpy==2.0.0`), or a pin does not + meet another notebook's requirement, nothing is written and the script fails, naming every notebook and its + requirement. The developer makes the notebooks agree, or decides explicitly in requirements_overrides.yaml + (next to the output), with the reason; the decision is written at the top of every generated file; + * requirements with different environment markers (`; sys_platform == "win32"`) are kept as separate lines; + * package names are compared as pip compares them (`NumPy`, `numpy` and `num_py` are one package). +`python merge_requirements.py --check` only looks for conflicts: it writes nothing (the release runs it). + +It preserves non-package lines (pip options, VCS/URLs) and supports simple `-r` includes. It also automatically adds ipywidgets if missing, with a version determined by Python, JupyterLab, and matplotlib versions from environment.yaml. """ -from pathlib import Path import argparse +import re import sys from collections import OrderedDict -from datetime import datetime -import re +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + import yaml +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version +DOCS_URL = "https://github.com/CellMigrationLab/LabConstrictor/blob/main/.tools/docs/migration_0_1_30.md" +OVERRIDES_FILE = "requirements_overrides.yaml" -# Regex to capture name, optional extras, operator and version, and optional markers +# Regex to capture name, optional extras, operator and version, and optional markers (used for the GPU rewrite of final lines) pkg_re = re.compile(r"^(?P[A-Za-z0-9_.+-]+)(?P\[[^\]]+\])?\s*(?P==|>=|<=|~=|!=|>|<)\s*(?P[^;\s]+)(?P\s*;.*)?$") OS_FIELD_MAP = { @@ -52,13 +69,70 @@ PLAIN_PKG_RE = re.compile( r"^(?P[A-Za-z0-9_.+-]+)(?P\[[^\]]+\])?(?P\s*;.*)?$" ) +ADDED_BY_SCRIPT = "added by merge_requirements.py" + + +@dataclass +class Entry: + """One requirement line of one file, as written.""" + + source: str + line: str + name: str + extras: tuple[str, ...] + spec: str + marker: str + +@dataclass +class Override: + """A decision of requirements_overrides.yaml: every notebook gets `use` for this package.""" + name: str + use: str + reason: str + used: bool = False + + +@dataclass +class Resolved: + """The line written for one package (and marker), and how it was decided.""" + + entries: list[Entry] + spec: str + extras: list[str] + override: Override | None = None + + @property + def name(self) -> str: + return self.entries[0].name + + @property + def marker(self) -> str: + return self.entries[0].marker + + def line(self) -> str: + extras = f"[{','.join(self.extras)}]" if self.extras else "" + marker = f" ; {self.marker}" if self.marker else "" + return f"{self.name}{extras}{self.spec}{marker}" + + +@dataclass class RequirementContext: - def __init__(self): - self.pkg_order: list[str] = [] - self.pkgs: dict[str, dict] = {} - self.other_lines: OrderedDict[str, None] = OrderedDict() + groups: OrderedDict[tuple[str, str], list[Entry]] = field(default_factory=OrderedDict) + other_lines: OrderedDict[str, None] = field(default_factory=OrderedDict) + errors: list[str] = field(default_factory=list) + + +class Conflict(Exception): + """Requirements of one package that no single version meets, or that the script will not choose between.""" + + def __init__(self, package: str, reason: str, entries: list[Entry], where: str = ""): + super().__init__(reason) + self.package = package + self.reason = reason + self.entries = entries + self.where = where def normalize_line(line: str) -> str: @@ -68,45 +142,15 @@ def normalize_line(line: str) -> str: return line.strip() -try: - # Prefer packaging.version for robust version comparison - from packaging.version import Version, InvalidVersion - - def parse_version(v: str): - try: - return Version(v) - except InvalidVersion: - return v - - def is_version_greater(a, b): - try: - return Version(a) > Version(b) - except Exception: - return str(a) > str(b) - -except Exception: - # Fallback: simple numeric-aware comparator - def _split_parts(v: str): - parts = re.split(r"[.\-+_]|(?=[a-zA-Z])", v) - key = [] - for p in parts: - if not p: - continue - if p.isdigit(): - key.append((0, int(p))) - else: - key.append((1, p)) - return tuple(key) - - def parse_version(v: str): - return _split_parts(v) - - def is_version_greater(a, b): - return parse_version(a) > parse_version(b) +def display_path(path: Path) -> str: + try: + return path.resolve().relative_to(Path.cwd().resolve()).as_posix() + except ValueError: + return path.as_posix() -def merge_python_version(filepath: str | None, current: str | None, candidate: str | None): - """Pick the newest python_version string, if provided.""" +def merge_python_version(filepath: str | Path | None, current: str | None, candidate: str | None): + """Every notebook that names a python_version must name the same one.""" if not candidate: return current if current is None: @@ -117,7 +161,7 @@ def merge_python_version(filepath: str | None, current: str | None, candidate: s raise ValueError(f"Conflicting python_version found {current} vs {candidate} when processing {filepath}") -def process_requirement_line(line: str, pkg_order: list, pkgs: dict, other_lines: OrderedDict, base_dir: Path | None = None): +def process_requirement_line(line: str, ctx: RequirementContext, source: str, base_dir: Path | None = None): """Handle a single requirement line (shared between txt and yaml inputs).""" if not line: return @@ -127,51 +171,34 @@ def process_requirement_line(line: str, pkg_order: list, pkgs: dict, other_lines include_path = Path(parts[1]) if base_dir is None else (base_dir / parts[1]) include = include_path.resolve() if include.exists(): - read_requirements_file(include, pkg_order, pkgs, other_lines) + read_requirements_file(include, ctx) return # Keep pip option lines (indexes, find-links, etc.) as-is if line.startswith("--") or line.startswith("-f ") or line.startswith("-i ") or line.startswith("-e "): - other_lines.setdefault(line, None) + ctx.other_lines.setdefault(line, None) return # VCS or URL lines (git+, http(s)://, file:) keep as-is if any(line.startswith(prefix) for prefix in ("git+", "http://", "https://", "file:", "ssh://")): - other_lines.setdefault(line, None) + ctx.other_lines.setdefault(line, None) return - m = pkg_re.match(line) - if not m: - # plain package (unpinned) or marker-only spec - key = line.split(";", 1)[0].strip().lower() - if key not in pkgs: - pkgs[key] = {"name": line, "pinned": False, "ver": None} - pkg_order.append(key) + try: + requirement = Requirement(line) + except InvalidRequirement as exc: + ctx.errors.append(f"{source}: '{line}' is not a valid requirement ({exc})") return - - name = m.group("name") - extras = m.group("extras") or "" - ver = m.group("ver") - marker = m.group("marker") or "" - - base = name.lower() - if base not in pkgs: - pkgs[base] = {"name": name + (extras or ""), "pinned": True, "ver": ver, "marker": marker} - pkg_order.append(base) - else: - existing = pkgs[base] - if not existing.get("pinned"): - pkgs[base] = {"name": name + (extras or ""), "pinned": True, "ver": ver, "marker": marker} - else: - try: - if is_version_greater(ver, existing.get("ver")): - pkgs[base] = {"name": name + (extras or ""), "pinned": True, "ver": ver, "marker": marker} - except Exception: - if str(ver) > str(existing.get("ver")): - pkgs[base] = {"name": name + (extras or ""), "pinned": True, "ver": ver, "marker": marker} + if requirement.url: + # name @ url: installed from that address, as written + ctx.other_lines.setdefault(line, None) + return + marker = str(requirement.marker) if requirement.marker else "" + entry = Entry(source, line, requirement.name, tuple(sorted(requirement.extras)), str(requirement.specifier), marker) + ctx.groups.setdefault((canonicalize_name(requirement.name), marker), []).append(entry) -def read_requirements_file(path: Path, pkg_order: list, pkgs: dict, other_lines: OrderedDict): +def read_requirements_file(path: Path, ctx: RequirementContext): try: text = path.read_text(encoding="utf-8") except Exception: @@ -179,10 +206,10 @@ def read_requirements_file(path: Path, pkg_order: list, pkgs: dict, other_lines: for raw in text.splitlines(): line = normalize_line(raw) - process_requirement_line(line, pkg_order, pkgs, other_lines, base_dir=path.parent) + process_requirement_line(line, ctx, display_path(path), base_dir=path.parent) -def read_requirements_yaml(path: Path, pkg_order: list, pkgs: dict, other_lines: OrderedDict, os_contexts: dict[str, RequirementContext]) -> str | None: +def read_requirements_yaml(path: Path, ctx: RequirementContext, os_contexts: dict[str, RequirementContext]) -> str | None: """Read requirements.yaml and return its python_version if present.""" try: data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} @@ -191,21 +218,22 @@ def read_requirements_yaml(path: Path, pkg_order: list, pkgs: dict, other_lines: python_version = data.get("python_version") dependencies = data.get("dependencies", []) or [] + source = display_path(path) for dep in dependencies: line = normalize_line(str(dep)) - process_requirement_line(line, pkg_order, pkgs, other_lines, base_dir=path.parent) + process_requirement_line(line, ctx, source, base_dir=path.parent) - for os_name, field in OS_FIELD_MAP.items(): - os_deps = data.get(field) or [] + for os_name, field_name in OS_FIELD_MAP.items(): + os_deps = data.get(field_name) or [] if not os_deps: continue - ctx = os_contexts.get(os_name) - if ctx is None: + os_ctx = os_contexts.get(os_name) + if os_ctx is None: continue for dep in os_deps: line = normalize_line(str(dep)) - process_requirement_line(line, ctx.pkg_order, ctx.pkgs, ctx.other_lines, base_dir=path.parent) + process_requirement_line(line, os_ctx, f"{source} ({field_name})", base_dir=path.parent) return python_version @@ -222,17 +250,175 @@ def find_requirements_files(source_dir): return sorted({f.resolve() for f in files}) +# -------------------------------------------------------------------------------------------------------------------------------- +# Deciding the requirement of each package +# -------------------------------------------------------------------------------------------------------------------------------- +def load_overrides(path: Path) -> dict[str, Override]: + """The decisions of requirements_overrides.yaml, by package name as pip compares them (none when the file is absent).""" + if not path.is_file(): + return {} + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + raise ValueError(f"{display_path(path)} is not valid YAML: {exc}") from exc + entries = data.get("overrides") if isinstance(data, dict) else None + if entries is None: + return {} + if not isinstance(entries, dict): + raise ValueError(f"{display_path(path)}: `overrides` must map package names to {{use: ..., reason: ...}}") + overrides: dict[str, Override] = {} + for name, value in entries.items(): + use = value.get("use") if isinstance(value, dict) else None + reason = value.get("reason") if isinstance(value, dict) else None + if not isinstance(use, str) or not use.strip(): + raise ValueError(f"{display_path(path)}: {name} needs `use`, the requirement every notebook gets, for example \"==2.0.0\"") + if not isinstance(reason, str) or not reason.strip(): + raise ValueError(f"{display_path(path)}: {name} needs `reason`, why this requirement is right for every notebook that uses {name}") + try: + SpecifierSet(use.strip()) + except InvalidSpecifier as exc: + raise ValueError(f"{display_path(path)}: {name}: `use: {use}` is not a version requirement such as \"==2.0.0\" or \">=1.24,<2\" ({exc})") from exc + overrides[canonicalize_name(str(name))] = Override(str(name), use.strip(), " ".join(reason.split())) + return overrides + + +def _exact_versions(entry: Entry) -> list[str]: + return [s.version for s in SpecifierSet(entry.spec) if s.operator in ("==", "===") and "*" not in s.version] + + +def _same_version(a: str, b: str) -> bool: + try: + return Version(a) == Version(b) + except InvalidVersion: + return a == b + + +def _union_extras(entries: list[Entry]) -> list[str]: + extras: list[str] = [] + for entry in entries: + extras.extend(extra for extra in entry.extras if extra not in extras) + return extras + + +def resolve_group(package: str, entries: list[Entry], checked_with: list[Entry], override: Override | None, where: str = "") -> Resolved: + """The requirement written for `entries` (one package, one marker); `checked_with` are other requirements that apply too. + + An override decides. Otherwise the requirements are combined: several different exact pins, or a pin that does not meet another + requirement, raise Conflict (whatever the order of the files). Without a pin, the combined ranges are written as they are. + """ + extras = _union_extras(entries) + if override is not None: + override.used = True + return Resolved(entries, override.use, extras, override) + everything = entries + checked_with + pins: list[tuple[str, Entry]] = [(version, entry) for entry in everything for version in _exact_versions(entry)] + distinct: list[str] = [] + for version, _ in pins: + if not any(_same_version(version, seen) for seen in distinct): + distinct.append(version) + if len(distinct) > 1: + raise Conflict(package, f"pinned to different versions ({', '.join(sorted(distinct))})", everything, where) + if distinct: + pin = distinct[0] + for entry in everything: + if entry.spec and not SpecifierSet(entry.spec).contains(pin, prereleases=True): + raise Conflict(package, f"the pin =={pin} does not meet {entry.name}{entry.spec} of {entry.source}", everything, where) + elif not satisfiable(SpecifierSet(",".join(entry.spec for entry in everything if entry.spec))): + raise Conflict(package, "no version meets all of these requirements", everything, where) + # The line keeps the requirements of its own entries; `checked_with` (the common requirements of an OS file) only had to agree. + own_pins = [version for entry in entries for version in _exact_versions(entry)] + if own_pins: + return Resolved(entries, f"=={own_pins[0]}", extras) + return Resolved(entries, str(SpecifierSet(",".join(entry.spec for entry in entries if entry.spec))), extras) + + +def satisfiable(specifiers: SpecifierSet) -> bool: + """Whether some version meets every specifier (`<2,>=2.1` does not). + + Every version a specifier names is tried, and versions just above it (2.1, 2.1.1, 2.1.0.1, 2.1.0.0.1): a range that only versions + closer than that could meet is reported as unsatisfiable, which stops the merge (an override then decides) instead of guessing. + """ + if not len(specifiers): + return True + candidates: list[str] = [] + for specifier in specifiers: + version = specifier.version.removesuffix(".*") + candidates += [version, f"{version}.1", f"{version}.0.1", f"{version}.0.0.1"] + for candidate in candidates: + try: + if specifiers.contains(candidate, prereleases=True): + return True + except InvalidVersion: + continue + return False + + +def resolve_context(ctx: RequirementContext, overrides: dict[str, Override], conflicts: list[Conflict], + base: RequirementContext | None = None, where: str = "") -> list[Resolved]: + """Resolve every package of a context; an OS context is also checked against the common requirements that apply on that OS.""" + resolved: list[Resolved] = [] + for (package, marker), entries in ctx.groups.items(): + checked_with: list[Entry] = [] + if base is not None: + for (base_package, base_marker), base_entries in base.groups.items(): + if base_package == package and base_marker in ("", marker): + checked_with.extend(base_entries) + try: + resolved.append(resolve_group(package, entries, checked_with, overrides.get(package), where)) + except Conflict as conflict: + conflicts.append(conflict) + return resolved + + +def conflict_report(conflicts: list[Conflict], errors: list[str], overrides_path: Path) -> str: + lines = ["The notebooks ask for incompatible package versions, so no requirements file was written.", ""] + for error in errors: + lines.append(f" {error}") + for conflict in sorted(conflicts, key=lambda c: (c.package, c.where)): + where = f" ({conflict.where})" if conflict.where else "" + lines.append(f" {conflict.package}{where}: {conflict.reason}") + for entry in sorted(conflict.entries, key=lambda e: (e.source, e.line)): + lines.append(f" {entry.source}: {entry.line}") + if conflicts: + example = sorted(conflicts, key=lambda c: c.package)[0].package + lines += [ + "", + f"Make the notebooks agree (edit their requirements.yaml), or decide in {display_path(overrides_path)} which requirement every", + "notebook gets, and why. It is written at the top of the generated requirements files. For example:", + "", + "overrides:", + f" {example}:", + ' use: "=="', + f' reason: ""', + ] + lines += ["", f"See {DOCS_URL}"] + return "\n".join(lines) + + +def override_header(resolved: list[Resolved]) -> list[str]: + lines = [] + for item in resolved: + if item.override is None: + continue + asked = "; ".join(f"{entry.source}: {entry.line}" for entry in item.entries if entry.source != ADDED_BY_SCRIPT) + lines.append(f"# Decided in {OVERRIDES_FILE}: {item.line()} (the notebooks ask for: {asked}). Reason: {item.override.reason}") + return lines + + +# -------------------------------------------------------------------------------------------------------------------------------- +# Packages the app always needs +# -------------------------------------------------------------------------------------------------------------------------------- def resolve_ipywidgets_version(py_version, jl_version): """Resolve ipywidgets version based on Python, JupyterLab versions. - + Returns a version spec string like ">=7.0.0,<8.0.0" or a pinned version. """ - + # Simple heuristic: ipywidgets 8.x for modern stacks, 7.x for older try: py_major, py_minor = map(int, py_version.split(".")[:2]) jl_major = int(jl_version.split(".")[0]) - + if jl_major >= 4 and py_major >= 3 and py_minor >= 9: return "8.1.7" # Latest stable for modern Python + JupyterLab 4 elif py_major >= 3 and py_minor >= 7: @@ -243,58 +429,48 @@ def resolve_ipywidgets_version(py_version, jl_version): return ">=7.0.0" # Safe fallback -def determine_jupyterlab_version(pkgs: dict, default: str = "4.4.0") -> str: - jl_info = pkgs.get("jupyterlab") - if jl_info and jl_info.get("pinned") and jl_info.get("ver"): - return jl_info["ver"] +def _has_package(ctx: RequirementContext, package: str) -> bool: + return any(name == package for name, _ in ctx.groups) + + +def _add_package(ctx: RequirementContext, name: str, spec: str) -> None: + ctx.groups[(canonicalize_name(name), "")] = [Entry(ADDED_BY_SCRIPT, f"{name}{spec}", name, (), spec, "")] + + +def determine_jupyterlab_version(resolved: list[Resolved], default: str = "4.4.0") -> str: + for item in resolved: + if canonicalize_name(item.name) == "jupyterlab" and not item.marker and item.spec.startswith("==") and "," not in item.spec: + return item.spec[2:] return default def ensure_jl_hidecode(ctx: RequirementContext): - if "jl-hidecode" in ctx.pkgs: + if _has_package(ctx, "jl-hidecode"): return - ctx.pkgs["jl-hidecode"] = { - "name": "jl-hidecode", - "pinned": True, - "ver": "0.0.1", - "marker": "", - } - ctx.pkg_order.append("jl-hidecode") + _add_package(ctx, "jl-hidecode", "==0.0.1") print("Added jl-hidecode==0.0.1 (enforced)") -def ensure_ipywidgets(ctx: RequirementContext, python_version: str | None, jl_ver: str): - if "ipywidgets" in ctx.pkgs: - return +def ensure_ipywidgets(ctx: RequirementContext, python_version: str | None, jl_ver: str) -> bool: + if _has_package(ctx, "ipywidgets"): + return False iw_version = resolve_ipywidgets_version(python_version, jl_ver) - ctx.pkgs["ipywidgets"] = { - "name": "ipywidgets", - "pinned": True, - "ver": iw_version, - "marker": "", - } - ctx.pkg_order.append("ipywidgets") - print(f"Added ipywidgets=={iw_version} (resolved)") - - -def build_final_lines(ctx: RequirementContext, sort: bool) -> list[str]: - final_lines: list[str] = [] - for ol in ctx.other_lines.keys(): - final_lines.append(ol) + spec = iw_version if iw_version[:1] in "<>=!~" else f"=={iw_version}" + _add_package(ctx, "ipywidgets", spec) + print(f"Added ipywidgets{spec} (resolved)") + return True - pkg_keys = sorted(ctx.pkg_order, key=str.casefold) if sort else ctx.pkg_order - for key in pkg_keys: - info = ctx.pkgs.get(key) - if not info: - continue - if info.get("pinned") and info.get("ver"): - final_lines.append(f"{info['name']}=={info['ver']}{info.get('marker','')}") - else: - final_lines.append(info["name"]) +def build_final_lines(ctx: RequirementContext, resolved: list[Resolved], sort: bool) -> list[str]: + final_lines: list[str] = list(ctx.other_lines.keys()) + items = sorted(resolved, key=lambda item: item.name.casefold()) if sort else resolved + final_lines.extend(item.line() for item in items) return final_lines +# -------------------------------------------------------------------------------------------------------------------------------- +# Output files +# -------------------------------------------------------------------------------------------------------------------------------- def get_os_output_path(output: Path, os_name: str) -> Path: suffix = output.suffix stem = output.stem @@ -410,15 +586,18 @@ def write_requirements_output( final_lines: list[str], target_os: str | None = None, target_accelerator: str | None = None, + decisions: list[str] | None = None, ): + generated = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() header_lines = [ - f"# Generated by .tools/python/merge_requirements.py on {datetime.utcnow().isoformat()}Z", + f"# Generated by .tools/python/merge_requirements.py on {generated}Z", f"# Source files: {', '.join(str(p) for p in files)}", ] if target_os: header_lines.append(f"# Target OS: {OS_DISPLAY_NAMES.get(target_os, target_os.capitalize())}") if target_accelerator: header_lines.append(f"# Target accelerator: {target_accelerator}") + header_lines.extend(decisions or []) header_lines.append("# ---") content = "\n".join(header_lines) + "\n" @@ -430,15 +609,18 @@ def write_requirements_output( path.write_text(content, encoding="utf-8") -def main(): +def main(argv: list[str] | None = None): parser = argparse.ArgumentParser(description="Merge multiple requirements files (txt or yaml) into one.") parser.add_argument("--source-dir", default="notebooks", help="Directory to scan for requirements files") parser.add_argument("--output", default="requirements.txt", help="Output requirements file to write") parser.add_argument("--sort", action="store_true", help="Sort final requirements alphabetically") - args = parser.parse_args() + parser.add_argument("--overrides", help=f"Decisions for conflicting requirements (default: {OVERRIDES_FILE} next to the output)") + parser.add_argument("--check", action="store_true", help="Only check that the notebooks' requirements do not conflict; write nothing") + args = parser.parse_args(argv) source = Path(args.source_dir) output = Path(args.output) + overrides_path = Path(args.overrides) if args.overrides else output.with_name(OVERRIDES_FILE) discovered_files = find_requirements_files(source) @@ -449,6 +631,9 @@ def main(): files = [f for f in discovered_files if f.resolve() not in skip_outputs] if not files: + if args.check: + print(f"No requirements files found under {source}: nothing to check.") + return 0 print(f"No requirements files found under {source}, writing empty {output}") output.write_text("# Generated requirements (none found)\n", encoding="utf-8") gpu_output = get_gpu_output_path(output) @@ -461,17 +646,43 @@ def main(): for f in files: if f.suffix.lower() in {".yaml", ".yml"}: - candidate_py = read_requirements_yaml(f, base_ctx.pkg_order, base_ctx.pkgs, base_ctx.other_lines, os_contexts) + candidate_py = read_requirements_yaml(f, base_ctx, os_contexts) python_version = merge_python_version(f, python_version, candidate_py) else: - read_requirements_file(f, base_ctx.pkg_order, base_ctx.pkgs, base_ctx.other_lines) + read_requirements_file(f, base_ctx) + + try: + overrides = load_overrides(overrides_path) + except ValueError as exc: + print(f"{exc}\nSee {DOCS_URL}", file=sys.stderr) + return 1 ensure_jl_hidecode(base_ctx) - jl_ver = determine_jupyterlab_version(base_ctx.pkgs) - ensure_ipywidgets(base_ctx, python_version, jl_ver) + conflicts: list[Conflict] = [] + base_resolved = resolve_context(base_ctx, overrides, conflicts) + jl_ver = determine_jupyterlab_version(base_resolved) + if ensure_ipywidgets(base_ctx, python_version, jl_ver): + key = next(reversed(base_ctx.groups)) + base_resolved.append(resolve_group(key[0], base_ctx.groups[key], [], overrides.get(key[0]))) + os_resolved = { + os_name: resolve_context(ctx, overrides, conflicts, base=base_ctx, where=f"{OS_FIELD_MAP[os_name]}, with the common dependencies") + for os_name, ctx in os_contexts.items() + } + errors = base_ctx.errors + [error for ctx in os_contexts.values() for error in ctx.errors] + if conflicts or errors: + print(conflict_report(conflicts, errors, overrides_path), file=sys.stderr) + return 1 + for override in overrides.values(): + if not override.used: + print(f"Note: {display_path(overrides_path)} decides {override.name}, which no notebook requires any more; the entry can be removed.") + if args.check: + print(f"The requirements of {len(files)} file(s) do not conflict" + + (f" ({sum(o.used for o in overrides.values())} decision(s) from {display_path(overrides_path)})." if overrides else ".")) + return 0 - base_final_lines = build_final_lines(base_ctx, args.sort) - write_requirements_output(output, files, base_final_lines) + decisions = override_header(base_resolved) + base_final_lines = build_final_lines(base_ctx, base_resolved, args.sort) + write_requirements_output(output, files, base_final_lines, decisions=decisions) print(f"Wrote {output} with {len(base_final_lines)} entries (from {len(files)} files)") gpu_output = get_gpu_output_path(output) @@ -481,15 +692,16 @@ def main(): files, gpu_final_lines, target_accelerator=GPU_DISPLAY_NAME, + decisions=decisions, ) print(f"Wrote {gpu_output} with {len(gpu_final_lines)} entries (from {len(files)} files, target {GPU_DISPLAY_NAME})") for os_name, ctx in os_contexts.items(): - if not ctx.pkg_order and not ctx.other_lines: + if not ctx.groups and not ctx.other_lines: continue - os_final_lines = build_final_lines(ctx, args.sort) + os_final_lines = build_final_lines(ctx, os_resolved[os_name], args.sort) os_output = get_os_output_path(output, os_name) - write_requirements_output(os_output, files, os_final_lines, target_os=os_name) + write_requirements_output(os_output, files, os_final_lines, target_os=os_name, decisions=override_header(os_resolved[os_name])) print(f"Wrote {os_output} with {len(os_final_lines)} entries (from {len(files)} files, target {os_name})") # Create the environment.yaml if not present diff --git a/.tools/requirements.txt b/.tools/requirements.txt index 8492066..eb02274 100644 --- a/.tools/requirements.txt +++ b/.tools/requirements.txt @@ -3,3 +3,5 @@ pyyaml==6.0.2 nbformat==5.10.4 # lock_requirements.py (requirements*.txt -> requirements*.lock.txt) uv==0.13.3 +# merge_requirements.py (versions compared as pip compares them) +packaging==25.0 diff --git a/.tools/templates/Welcome_template.ipynb b/.tools/templates/Welcome_template.ipynb index 8cde2b9..33699e5 100644 --- a/.tools/templates/Welcome_template.ipynb +++ b/.tools/templates/Welcome_template.ipynb @@ -217,6 +217,37 @@ ")\n", "\n", "\n", + "def replace_notebook_with_backup(notebook_path: Path, new_content: str):\n", + " \"\"\"Replace a notebook by its update, keeping the current file (with the user's changes and outputs) as a backup.\n", + "\n", + " The new content must be a notebook. It is written to a temporary file next to the notebook and then moved into place, so a failed\n", + " download or write never leaves a broken notebook. Returns the path of the backup, or None when there was no notebook yet.\n", + " \"\"\"\n", + " import os\n", + " from datetime import datetime\n", + "\n", + " if not isinstance(json.loads(new_content), dict):\n", + " raise ValueError(\"the downloaded file is not a notebook\")\n", + " notebook_path.parent.mkdir(parents=True, exist_ok=True)\n", + " backup_path = None\n", + " if notebook_path.exists():\n", + " stamp = datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n", + " backup_path = notebook_path.with_name(f\"{notebook_path.stem}.backup-{stamp}{notebook_path.suffix}\")\n", + " number = 1\n", + " while backup_path.exists():\n", + " number += 1\n", + " backup_path = notebook_path.with_name(f\"{notebook_path.stem}.backup-{stamp}-{number}{notebook_path.suffix}\")\n", + " shutil.copy2(notebook_path, backup_path)\n", + " temporary_path = notebook_path.with_name(f\".{notebook_path.name}.update\")\n", + " try:\n", + " temporary_path.write_text(new_content, encoding=\"utf-8\")\n", + " os.replace(temporary_path, notebook_path)\n", + " finally:\n", + " if temporary_path.exists():\n", + " temporary_path.unlink()\n", + " return backup_path\n", + "\n", + "\n", "def apply_grid_layout(grid):\n", " \"\"\"Apply responsive sizing while allowing long text to wrap safely.\"\"\"\n", " grid.add_class(\"labconstrictor-responsive-grid\")\n", @@ -476,54 +507,82 @@ " grid[idx, 4] = table_cell(f\"❌ Needs Update to {online_version}\")\n", " \n", " def button_update(main_folder, subfolder, row_idx):\n", - " def show(button):\n", - " button.disabled = True\n", + " # Updating replaces the user's copy of the notebook: ask first, keep that copy as a backup, and change the\n", + " # recorded version only once the new notebook is in place (CellMigrationLab/LabConstrictor#49).\n", + " local_notebook_path = Path(\".\") / main_folder / f\"{subfolder}.ipynb\"\n", + " new_version = online_latest_versions[main_folder][subfolder]\n", + "\n", + " def reset(button):\n", + " button.disabled = False\n", + " button.description = \"Update\"\n", + "\n", + " def do_update(button):\n", " button.description = \"Updating...\"\n", - " \n", - " # Download the notebook\n", " notebook_url = f\"https://api.github.com/repos/{github_owner}/{github_repo_name}/contents/notebooks/{main_folder}/{subfolder}.ipynb?ref={github_branch}\"\n", + " headers = {}\n", " if private_repository:\n", - " if github_token.value:\n", - " notebook_response = requests.get(notebook_url, headers={\"Authorization\": f\"token {github_token.value}\"})\n", - " else:\n", - " grip_output.value = \"❌ Please check your Presonal Access Token\"\n", - " else:\n", - " notebook_response = requests.get(notebook_url)\n", - " \n", - " if notebook_response.status_code == 200:\n", - " try:\n", - " # Decode the notebook content\n", - " notebook_content = base64.b64decode(\n", - " notebook_response.json()['content']\n", - " ).decode('utf-8')\n", - " \n", - " # Save to local path\n", - " local_notebook_path = Path(\".\") / main_folder / f\"{subfolder}.ipynb\"\n", - " local_notebook_path.parent.mkdir(parents=True, exist_ok=True)\n", - " local_notebook_path.write_text(notebook_content, encoding='utf-8')\n", - " \n", - " # Update the local version YAML\n", - " local_latest_versions[main_folder][subfolder] = online_latest_versions[main_folder][subfolder]\n", - " local_latest_version_path.write_text(yaml.dump(local_latest_versions, default_flow_style=False))\n", - " \n", - " grip_output.value = f\"✅ Updated {subfolder} to version {online_latest_versions[main_folder][subfolder]}\"\n", - " \n", - " # Update the grid to show up-to-date\n", - " grid[row_idx, 3] = table_cell(online_latest_versions[main_folder][subfolder])\n", - " grid[row_idx, 4] = table_cell(\"✅ Up-to-date\")\n", - " grid[row_idx, 5] = table_cell(\"-\")\n", - " apply_grid_layout(grid)\n", - " \n", - " except Exception as e:\n", - " grip_output.value = f\"❌ Error saving notebook: {e}\"\n", - " button.disabled = False\n", - " button.description = \"Update\"\n", - " else:\n", - " grip_output.value = f\"❌ Failed to download notebook (HTTP {notebook_response.status_code})\"\n", + " if not github_token.value:\n", + " grip_output.value = \"❌ Please check your Personal Access Token. Your notebook was not changed.\"\n", + " reset(button)\n", + " return\n", + " headers = {\"Authorization\": f\"token {github_token.value}\"}\n", + " try:\n", + " notebook_response = requests.get(notebook_url, headers=headers, timeout=60)\n", + " except requests.RequestException as e:\n", + " grip_output.value = f\"❌ Failed to download notebook: {e}. Your notebook was not changed.\"\n", + " reset(button)\n", + " return\n", + " if notebook_response.status_code != 200:\n", + " grip_output.value = f\"❌ Failed to download notebook (HTTP {notebook_response.status_code}). Your notebook was not changed.\"\n", " display(notebook_response.text[:300])\n", - " button.disabled = False\n", - " button.description = \"Update\"\n", - " \n", + " reset(button)\n", + " return\n", + " try:\n", + " notebook_content = base64.b64decode(notebook_response.json()['content']).decode('utf-8')\n", + " backup_path = replace_notebook_with_backup(local_notebook_path, notebook_content)\n", + " except Exception as e:\n", + " grip_output.value = f\"❌ Error saving notebook: {e}. Your notebook was not changed.\"\n", + " reset(button)\n", + " return\n", + "\n", + " # The new notebook is in place: only now record its version\n", + " local_latest_versions[main_folder][subfolder] = new_version\n", + " local_latest_version_path.write_text(yaml.dump(local_latest_versions, default_flow_style=False))\n", + " saved = f\" Your previous copy is saved as {backup_path.name}, next to it.\" if backup_path else \"\"\n", + " grip_output.value = f\"✅ Updated {subfolder} to version {new_version}.{saved}\"\n", + "\n", + " # Update the grid to show up-to-date\n", + " grid[row_idx, 3] = table_cell(new_version)\n", + " grid[row_idx, 4] = table_cell(\"✅ Up-to-date\")\n", + " grid[row_idx, 5] = table_cell(\"-\")\n", + " apply_grid_layout(grid)\n", + "\n", + " def show(button):\n", + " button.disabled = True\n", + " button.description = \"Confirm below\"\n", + " question = widgets.HTML(\n", + " f\"Update {subfolder} to version {new_version}?
\"\n", + " f\"The new version replaces {local_notebook_path.as_posix()}. Your current copy, with your changes \"\n", + " f\"and outputs, is first saved as {subfolder}.backup-<date>.ipynb in the same folder.
\"\n", + " \"If the notebook is open in a tab, close that tab first: otherwise JupyterLab can save the old copy over the update.\"\n", + " )\n", + " confirm = widgets.Button(description=\"Back up and update\", button_style=\"warning\", layout=widgets.Layout(width=\"auto\"))\n", + " cancel = widgets.Button(description=\"Cancel\")\n", + " dialog = widgets.VBox([question, widgets.HBox([confirm, cancel])])\n", + "\n", + " def on_confirm(_):\n", + " dialog.close()\n", + " do_update(button)\n", + "\n", + " def on_cancel(_):\n", + " dialog.close()\n", + " grip_output.value = f\"Update of {subfolder} cancelled: nothing was changed.\"\n", + " reset(button)\n", + "\n", + " confirm.on_click(_widget_callback_output_cell_1.capture(clear_output=True, wait=True)(on_confirm))\n", + " cancel.on_click(_widget_callback_output_cell_1.capture(clear_output=True, wait=True)(on_cancel))\n", + " display(dialog)\n", + "\n", " return show\n", "\n", " update_button = widgets.Button(description=\"Update\", disable=False) \n", diff --git a/tests/_v0_1_25.py b/tests/_v0_1_25.py index 44d6035..e5de837 100644 --- a/tests/_v0_1_25.py +++ b/tests/_v0_1_25.py @@ -11,6 +11,8 @@ from pathlib import Path from types import ModuleType +from _v0_1_30 import before_0_1_30 + ROOT = Path(__file__).resolve().parents[1] MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_24_to_v0_1_25.py" @@ -26,6 +28,7 @@ def _migration() -> ModuleType: def before_0_1_25(relative: Path | str, data: bytes) -> bytes: """`data` (a live template file at `relative`) without the change 0.1.25 made to it; any other file is returned as it is.""" + data = before_0_1_30(relative, data) # the later change of 0.1.30 first (tests/_v0_1_30.py) migration = _migration() if Path(relative) != migration.BUMP_VERSION: return data diff --git a/tests/_v0_1_29.py b/tests/_v0_1_29.py index f1b025f..ecff498 100644 --- a/tests/_v0_1_29.py +++ b/tests/_v0_1_29.py @@ -11,6 +11,8 @@ from pathlib import Path from types import ModuleType +from _v0_1_30 import before_0_1_30 + ROOT = Path(__file__).resolve().parents[1] MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_28_to_v0_1_29.py" @@ -26,6 +28,7 @@ def _migration() -> ModuleType: def before_0_1_29(relative: Path | str, data: bytes) -> bytes: """`data` (a live template file at `relative`) without the corrections 0.1.29 made to it; any other file is returned as it is.""" + data = before_0_1_30(relative, data) # the later change of 0.1.30 first (tests/_v0_1_30.py) migration = _migration() edits = migration.EDITS.get(Path(relative)) if not edits: diff --git a/tests/_v0_1_30.py b/tests/_v0_1_30.py new file mode 100644 index 0000000..1fadba6 --- /dev/null +++ b/tests/_v0_1_30.py @@ -0,0 +1,53 @@ +"""Helper of the tests: a live template file as it was before template 0.1.30. + +Template 0.1.30 replaced merge_requirements.py, added packaging to .tools/requirements.txt, three lines to update_on_notebook_change.yml, +the requirements check to release_constructor.yml and the safe notebook update to Welcome_template.ipynb. The resources of the older +migrations are frozen; `before_0_1_30` gives the bytes of a live file without the 0.1.30 change, using the migration's own constants (the +frozen 0.1.29 copies for merge_requirements.py and the Welcome template), so their tests keep checking what they checked. The 0.1.30 +changes are tested in test_template_sync_0_1_30.py. +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import ModuleType + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_29_to_v0_1_30.py" +WELCOME_0_1_29 = ROOT / "tests/fixtures/template_0_1_29/Welcome_template.ipynb" + + +def _migration() -> ModuleType: + spec = importlib.util.spec_from_file_location("migration_0_1_30_for_older_tests", MIGRATION_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {MIGRATION_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def before_0_1_30(relative: Path | str, data: bytes) -> bytes: + """`data` (a live template file at `relative`) without the change 0.1.30 made to it; any other file is returned as it is.""" + migration = _migration() + relative = Path(relative) + if relative == migration.MERGE: + return (migration.PREVIOUS_ROOT / migration.MERGE).read_bytes() + if relative == migration.WELCOME_TEMPLATE: + return WELCOME_0_1_29.read_bytes() + text = data.decode("utf-8") + newline = "\r\n" if "\r\n" in text else "\n" + text = text.replace("\r\n", "\n") + if relative == migration.TOOLS_REQUIREMENTS: + text = text.replace(migration.PACKAGING_LINES, "", 1) + elif relative == migration.NOTEBOOK_WORKFLOW: + for old, new in reversed(migration.NOTEBOOK_WORKFLOW_BLOCKS): + text = text.replace(new, old, 1) + elif relative == migration.RELEASE_WORKFLOW: + match = migration.PLACEHOLDER_STEP.search(text) + if match is not None: + step = migration.REQUIREMENTS_CHECK_STEP.format(indent=match.group("indent"), log=match.group("log")) + if text[match.end() :].startswith(step): + text = text[: match.end()] + text[match.end() + len(step) :] + else: + return data + return text.replace("\n", newline).encode("utf-8") diff --git a/tests/fixtures/template_0_1_29/Welcome_template.ipynb b/tests/fixtures/template_0_1_29/Welcome_template.ipynb new file mode 100644 index 0000000..8cde2b9 --- /dev/null +++ b/tests/fixtures/template_0_1_29/Welcome_template.ipynb @@ -0,0 +1,658 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "335f15f8", + "metadata": {}, + "source": [ + "# Welcome to PROJECT_NAME! \n", + "---\n", + "\n", + "## How does it work?\n", + "\n", + "This welcome notebook provides two main features:\n", + "\n", + "### 1. Browse and Open Notebooks\n", + "\n", + "You will find below an interactive table with all available notebooks, each designed to facilitate a specific aspect of experiment management. Simply click on any \"Open the notebook\" button to launch it in JupyterLab.\n", + "\n", + "### 2. Check for Updates\n", + "\n", + "Stay up-to-date with the latest notebook versions! Run the cell below to compare your local notebooks with the latest versions in the app's GitHub repository. If the repository is private, enter your GitHub Personal Access Token and click \"Load Private GitHub Repo\". You can update any outdated notebook with a single click.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd6a16d4", + "metadata": { + "cellView": "form", + "collapsed": false, + "jupyter": { + "source_hidden": true + }, + "jupyterlabHideCode": { + "locked": true + } + }, + "outputs": [], + "source": [ + "# Run this cell to see the interactive notebook list ⬇️⬇️⬇️\n", + "\n", + "from IPython.display import display, Markdown, HTML, Javascript, clear_output\n", + "from ipywidgets import widgets, GridspecLayout\n", + "from pathlib import Path\n", + "import subprocess\n", + "import requests\n", + "import tempfile\n", + "import zipfile\n", + "import base64\n", + "import shutil\n", + "import json\n", + "import yaml\n", + "import sys\n", + "\n", + "# Capture stdout, stderr, rich displays, figures, warnings, and tracebacks from widget callbacks.\n", + "_widget_callback_output_cell_1 = widgets.Output()\n", + "\n", + "\n", + "# Define notebooks with their metadata\n", + "notebooks = DICT_OF_NOTEBOOKS\n", + "\n", + "# Load the local information from all the notebooks\n", + "local_latest_version_path = Path(\"..\") / \"notebook_latest_versions.yaml\"\n", + "local_latest_versions = yaml.safe_load(local_latest_version_path.read_text())\n", + "\n", + "# Load the local construct information\n", + "construct_file_path = Path(\"..\") / \"construct.yaml\"\n", + "construct_info = yaml.safe_load(construct_file_path.read_text())\n", + "local_project_version = construct_info.get('version', '0.0.0')\n", + "\n", + "# Load the online information (GitHub) from all the notebooks\n", + "github_owner = \"GITHUB_OWNER\"\n", + "github_repo_name = \"GITHUB_REPO_NAME\"\n", + "github_branch = \"main\" # Change to \"dev\" if you want to check against dev branch\n", + "\n", + "def build_open_button(notebook_path: str, label: str = \"Open the notebook\") -> widgets.Widget:\n", + " normalized = Path(notebook_path).as_posix().lstrip(\"/\")\n", + "\n", + " button = widgets.Button(\n", + " description=label,\n", + " tooltip=f\"Open {normalized}\",\n", + " layout=widgets.Layout(width=\"auto\", min_width=\"140px\")\n", + " )\n", + " button.add_class(\"open-notebook-button\")\n", + "\n", + " js_output = widgets.Output(\n", + " layout=widgets.Layout(width=\"0px\", height=\"0px\", overflow=\"hidden\")\n", + " )\n", + "\n", + " def on_open_button_clicked(_):\n", + " js = f\"\"\"\n", + " (() => {{\n", + " const notebookPath = {json.dumps(normalized)};\n", + " const pathname = window.location.pathname;\n", + " const marker = \"/tree/\";\n", + " const treeIndex = pathname.indexOf(marker);\n", + "\n", + " if (treeIndex === -1) {{\n", + " window.open(notebookPath, \"_blank\");\n", + " return;\n", + " }}\n", + "\n", + " const prefix = pathname.slice(0, treeIndex + marker.length);\n", + " const currentNotebookPath = pathname.slice(treeIndex + marker.length);\n", + "\n", + " const currentParts = currentNotebookPath.split(\"/\").filter(Boolean);\n", + " if (currentParts.length > 0) {{\n", + " currentParts.pop();\n", + " }}\n", + "\n", + " const notebooksIndex = currentParts.indexOf(\"notebooks\");\n", + " const projectParts = notebooksIndex >= 0 ? currentParts.slice(0, notebooksIndex) : currentParts;\n", + "\n", + " const targetPath = [\n", + " ...projectParts,\n", + " ...notebookPath.split(\"/\").filter(Boolean)\n", + " ].join(\"/\");\n", + "\n", + " window.open(prefix + encodeURI(targetPath), \"_blank\");\n", + " }})();\n", + " \"\"\"\n", + " with js_output:\n", + " clear_output(wait=True)\n", + " display(Javascript(js))\n", + "\n", + " button.on_click(on_open_button_clicked)\n", + "\n", + " return widgets.VBox(\n", + " [button, js_output],\n", + " layout=widgets.Layout(width=\"100%\", min_width=\"0\", align_items=\"center\", justify_content=\"center\")\n", + " )\n", + "\n", + "def download_and_install_src(github_pat=None) -> None:\n", + " # Download the repository into a temporal foler\n", + " with tempfile.TemporaryDirectory() as tmpdir:\n", + " tmpdir_path = Path(tmpdir)\n", + "\n", + " staging_dir = tmpdir_path / \"managed_src_new\"\n", + " backup_dir = tmpdir_path / \"managed_src_old\"\n", + " \n", + " zip_path = tmpdir_path / \"repo.zip\"\n", + " extract_root = tmpdir_path / \"extract\"\n", + "\n", + " download_url = f\"https://api.github.com/repos/{github_owner}/{github_repo_name}/zipball/{github_branch}\"\n", + " headers = {\"Authorization\": f\"token {github_pat}\"} if github_pat else {}\n", + " response = requests.get(download_url, headers=headers, stream=True, timeout=60)\n", + " response.raise_for_status()\n", + "\n", + " with zip_path.open(\"wb\") as fh:\n", + " for chunk in response.iter_content(chunk_size=1024 * 1024):\n", + " if chunk:\n", + " fh.write(chunk)\n", + "\n", + " with zipfile.ZipFile(zip_path, \"r\") as zf:\n", + " zf.extractall(extract_root)\n", + "\n", + " extracted_roots = [p for p in extract_root.iterdir() if p.is_dir()]\n", + " if not extracted_roots:\n", + " raise RuntimeError(\"Could not find extracted repository root\")\n", + "\n", + " repo_root = extracted_roots[0]\n", + " downloaded_src_dir = repo_root / \"src\"\n", + "\n", + " if not downloaded_src_dir.exists():\n", + " raise RuntimeError(\"Downloaded repository does not contain a 'src' folder\")\n", + "\n", + " shutil.copytree(downloaded_src_dir, staging_dir)\n", + "\n", + " # Find the external code in a local folder\n", + " src_folder = Path(\"..\") / \"src\"\n", + "\n", + " try:\n", + " # Create a backup of the current src folder in case something goes wrong during the update process.\n", + " if src_folder.exists():\n", + " src_folder.rename(backup_dir)\n", + "\n", + " # Remove content in the src folder\n", + " if src_folder.exists():\n", + " shutil.rmtree(src_folder, ignore_errors=True)\n", + "\n", + " # Move the new src folder from the staging area to the src folder\n", + " staging_dir.rename(src_folder)\n", + "\n", + " # Install the external code with the new src folder's content\n", + " cmd = [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"..\",\n", + " \"--upgrade\",\n", + " \"--force-reinstall\"\n", + " ]\n", + " subprocess.check_call(cmd)\n", + "\n", + " # Once the new code is installed successfully, we can remove the backup folder since we don't need it anymore.\n", + " if backup_dir.exists():\n", + " shutil.rmtree(backup_dir)\n", + "\n", + " except Exception:\n", + " if src_folder.exists():\n", + " shutil.rmtree(src_folder, ignore_errors=True)\n", + "\n", + " if backup_dir.exists():\n", + " backup_dir.rename(src_folder)\n", + "\n", + " raise\n", + "\n", + "GRID_COLUMN_WIDTHS = (\n", + " \"minmax(0, 1.4fr) \" # Notebook Name\n", + " \"minmax(0, 1.4fr) \" # Notebook Topic\n", + " \"minmax(0, 2.8fr) \" # Description\n", + " \"minmax(85px, 0.8fr) \" # Local Version\n", + " \"minmax(100px, 1.2fr) \" # Status\n", + " \"minmax(70px, 0.7fr) \" # Update?\n", + " \"minmax(150px, 1.6fr)\" # Open Notebook\n", + ")\n", + "\n", + "\n", + "def apply_grid_layout(grid):\n", + " \"\"\"Apply responsive sizing while allowing long text to wrap safely.\"\"\"\n", + " grid.add_class(\"labconstrictor-responsive-grid\")\n", + " grid.layout.width = \"100%\"\n", + " grid.layout.min_width = \"0\"\n", + " grid.layout.grid_template_columns = GRID_COLUMN_WIDTHS\n", + " grid.layout.grid_auto_rows = \"auto\"\n", + " grid.layout.overflow = \"auto\"\n", + "\n", + "\n", + "def table_cell(value, align=\"center\", extra_class=\"\"):\n", + " \"\"\"Create a table cell that can shrink and wrap long scientific identifiers.\"\"\"\n", + " classes = [\"labconstrictor-table-cell\"]\n", + " if align == \"left\":\n", + " classes.append(\"labconstrictor-table-cell-left\")\n", + " else:\n", + " classes.append(\"labconstrictor-table-cell-center\")\n", + " if extra_class:\n", + " classes.append(extra_class)\n", + "\n", + " class_names = \" \".join(classes)\n", + "\n", + " return widgets.HTML(\n", + " value=f\"
{value}
\",\n", + " layout=widgets.Layout(width=\"100%\", min_width=\"0\"),\n", + " )\n", + "\n", + "\n", + "def table_header(value):\n", + " \"\"\"Create a responsive table header.\"\"\"\n", + " return widgets.HTML(\n", + " value=f\"
{value}
\",\n", + " layout=widgets.Layout(width=\"100%\", min_width=\"0\"),\n", + " )\n", + "\n", + "\n", + "def load_table(version_response, project_version_response, notebooks):\n", + " if version_response is None:\n", + " display(Markdown(\"⚠️ Not checking versions. Could not access the version file from GitHub.\"))\n", + " online_latest_versions = local_latest_versions\n", + " else:\n", + " try:\n", + " online_latest_versions = yaml.safe_load(\n", + " base64.b64decode(version_response.json()['content']).decode('utf-8')\n", + " )\n", + " if not isinstance(online_latest_versions, dict):\n", + " display(Markdown(\"⚠️ Not checking versions. The online version file format is invalid.\"))\n", + " online_latest_versions = local_latest_versions\n", + " except Exception as e:\n", + " display(Markdown(f\"⚠️ Not checking versions. Could not parse version file: {e}\"))\n", + " online_latest_versions = local_latest_versions\n", + "\n", + " if project_version_response is None:\n", + " online_project_version = local_project_version\n", + " else:\n", + " try:\n", + " project_construct_info = yaml.safe_load(\n", + " base64.b64decode(project_version_response.json()['content']).decode('utf-8')\n", + " )\n", + " online_project_version = project_construct_info.get('version', '0.0.0')\n", + " except Exception as e:\n", + " display(Markdown(f\"⚠️ Could not parse construct.yaml file: {e}\"))\n", + " online_project_version = local_project_version\n", + "\n", + " if online_project_version != local_project_version:\n", + " display(Markdown(f\"❌ The project version is outdated. Local version: {local_project_version}, Online version: {online_project_version}\"))\n", + " display(Markdown(\"Please update the project to the latest version by following the update instructions. Until you update the project, you will not be able to update individual notebooks.\"))\n", + " project_needs_update = True\n", + " else:\n", + " display(Markdown(f\"✅ The project version is up-to-date. Version: {local_project_version}\"))\n", + " project_needs_update = False\n", + "\n", + " # Check if there is external code and in case it is, check the version of the external code\n", + " src_change_path = Path(\"..\") / \"src_change.yaml\"\n", + " if src_change_path.exists():\n", + " # Load local src change info\n", + " local_src_change_info = yaml.safe_load(src_change_path.read_text())\n", + " local_src_date = local_src_change_info.get('src_last_modified', '1970-01-01_00:00:00')\n", + "\n", + " # Load online src change info\n", + " src_change_url = f\"https://api.github.com/repos/{github_owner}/{github_repo_name}/contents/.tools/meta/src_change.yaml?ref={github_branch}\"\n", + " try:\n", + " src_headers = {}\n", + " if globals().get('private_repository', False):\n", + " token_widget = globals().get('github_token')\n", + " token_value = token_widget.value if token_widget else ''\n", + " if token_value:\n", + " src_headers = {\"Authorization\": f\"token {token_value}\"}\n", + " src_change_response = requests.get(src_change_url, headers=src_headers)\n", + " if src_change_response.status_code == 200:\n", + " online_src_change_info = yaml.safe_load(\n", + " base64.b64decode(src_change_response.json()['content']).decode('utf-8')\n", + " )\n", + " online_src_date = online_src_change_info.get('src_last_modified', '1970-01-01_00:00:00')\n", + "\n", + " if online_src_date != local_src_date:\n", + " display(Markdown(f\"⚠️ The source code has been updated since you last updated the notebooks.\"))\n", + " display(Markdown(\"Please click on the button bellow to update the source code to the latest version.\"))\n", + " \n", + " update_src_button = widgets.Button(description=\"Update Source Code\", button_style='warning')\n", + " src_update_output = widgets.HTML(\"\")\n", + "\n", + " def on_update_src_button_clicked(b):\n", + " src_update_output.value = \"\"\n", + " try:\n", + " src_pat = None\n", + " if globals().get('private_repository', False):\n", + " token_widget = globals().get('github_token')\n", + " src_pat = token_widget.value if token_widget else None\n", + " src_update_output.value += \"⏳ Updating source code...
\"\n", + " download_and_install_src(github_pat=src_pat)\n", + " # Update the local src change info with the online info since we just updated to that version\n", + " src_change_path.write_text(yaml.dump(online_src_change_info, default_flow_style=False))\n", + " # Update the local version info to trigger the notebook update check and show the correct status in the grid without needing to refresh the page\n", + " local_latest_version_path.write_text(yaml.dump(online_latest_versions, default_flow_style=False))\n", + " src_update_output.value += \"✅ Source code updated successfully!
\"\n", + " src_update_output.value += \"Please restart the notebook kernel to apply the changes, and then re-run this cell to check for notebook updates.
\"\n", + " src_update_output.value += \"To restart the kernel, go to the menu bar and click on Kernel -> Restart Kernel.\"\n", + " except Exception as e:\n", + " src_update_output.value += f\"❌ Failed to update source code: {e}\"\n", + "\n", + " update_src_button.on_click(_widget_callback_output_cell_1.capture(clear_output=True, wait=True)(on_update_src_button_clicked))\n", + " display(\n", + " Markdown(\"---\"),\n", + " update_src_button, \n", + " src_update_output,\n", + " Markdown(\"---\")\n", + " )\n", + " else:\n", + " display(Markdown(f\"✅ The source code is up to date!\"))\n", + " else:\n", + " display(Markdown(f\"⚠️ Could not access external code control file from GitHub (HTTP {src_change_response.status_code}). Not checking for source code updates.\"))\n", + " except Exception as e:\n", + " display(Markdown(f\"⚠️ Could not access external code control file from GitHub: {e}. Not checking for source code updates.\"))\n", + "\n", + " # Go through all the notebooks and compare versions\n", + " num_rows = len(notebooks)\n", + " grid = GridspecLayout(1 + num_rows, 7)\n", + " apply_grid_layout(grid)\n", + " grip_output = widgets.HTML()\n", + "\n", + " # Add custom CSS for responsive table styling\n", + " display(widgets.HTML(\"\"\"\n", + " \n", + " \"\"\"))\n", + "\n", + "\n", + " grid[0, 0] = table_header(\"Notebook Name\")\n", + " grid[0, 1] = table_header(\"Notebook Topic\")\n", + " grid[0, 2] = table_header(\"Description\")\n", + " grid[0, 3] = table_header(\"Local Version\")\n", + " grid[0, 4] = table_header(\"Status\")\n", + " grid[0, 5] = table_header(\"Update?\")\n", + " grid[0, 6] = table_header(\"Open Notebook\")\n", + "\n", + " needs_update = False\n", + "\n", + " idx = 1\n", + " nb = {}\n", + " \n", + " for main_folder in online_latest_versions:\n", + " if main_folder in local_latest_versions:\n", + " for subfolder in online_latest_versions[main_folder]:\n", + " # Find the notebook path from the notebooks list\n", + " notebook_found = False\n", + " for n in notebooks:\n", + " if subfolder in n['path']:\n", + " nb = n\n", + " notebook_found = True\n", + " break\n", + "\n", + " if not notebook_found:\n", + " continue\n", + " \n", + " grid[idx, 0] = table_cell(nb[\"name\"])\n", + " grid[idx, 1] = table_cell(main_folder)\n", + " \n", + " if nb:\n", + " grid[idx, 2] = table_cell(nb[\"description\"], align=\"left\")\n", + " else:\n", + " grid[idx, 2] = table_cell(\"-\")\n", + "\n", + " if subfolder in local_latest_versions.get(main_folder, {}):\n", + " online_version = online_latest_versions[main_folder][subfolder]\n", + " local_version = local_latest_versions[main_folder][subfolder]\n", + " \n", + " grid[idx, 3] = table_cell(local_version)\n", + "\n", + " if project_needs_update:\n", + " grid[idx, 4] = table_cell(\"❌ Project Outdated\")\n", + " grid[idx, 5] = table_cell(\"-\")\n", + " elif online_version == local_version:\n", + " grid[idx, 4] = table_cell(\"✅ Up-to-date\")\n", + " grid[idx, 5] = table_cell(\"-\")\n", + " else:\n", + " grid[idx, 4] = table_cell(f\"❌ Needs Update to {online_version}\")\n", + " \n", + " def button_update(main_folder, subfolder, row_idx):\n", + " def show(button):\n", + " button.disabled = True\n", + " button.description = \"Updating...\"\n", + " \n", + " # Download the notebook\n", + " notebook_url = f\"https://api.github.com/repos/{github_owner}/{github_repo_name}/contents/notebooks/{main_folder}/{subfolder}.ipynb?ref={github_branch}\"\n", + " if private_repository:\n", + " if github_token.value:\n", + " notebook_response = requests.get(notebook_url, headers={\"Authorization\": f\"token {github_token.value}\"})\n", + " else:\n", + " grip_output.value = \"❌ Please check your Presonal Access Token\"\n", + " else:\n", + " notebook_response = requests.get(notebook_url)\n", + " \n", + " if notebook_response.status_code == 200:\n", + " try:\n", + " # Decode the notebook content\n", + " notebook_content = base64.b64decode(\n", + " notebook_response.json()['content']\n", + " ).decode('utf-8')\n", + " \n", + " # Save to local path\n", + " local_notebook_path = Path(\".\") / main_folder / f\"{subfolder}.ipynb\"\n", + " local_notebook_path.parent.mkdir(parents=True, exist_ok=True)\n", + " local_notebook_path.write_text(notebook_content, encoding='utf-8')\n", + " \n", + " # Update the local version YAML\n", + " local_latest_versions[main_folder][subfolder] = online_latest_versions[main_folder][subfolder]\n", + " local_latest_version_path.write_text(yaml.dump(local_latest_versions, default_flow_style=False))\n", + " \n", + " grip_output.value = f\"✅ Updated {subfolder} to version {online_latest_versions[main_folder][subfolder]}\"\n", + " \n", + " # Update the grid to show up-to-date\n", + " grid[row_idx, 3] = table_cell(online_latest_versions[main_folder][subfolder])\n", + " grid[row_idx, 4] = table_cell(\"✅ Up-to-date\")\n", + " grid[row_idx, 5] = table_cell(\"-\")\n", + " apply_grid_layout(grid)\n", + " \n", + " except Exception as e:\n", + " grip_output.value = f\"❌ Error saving notebook: {e}\"\n", + " button.disabled = False\n", + " button.description = \"Update\"\n", + " else:\n", + " grip_output.value = f\"❌ Failed to download notebook (HTTP {notebook_response.status_code})\"\n", + " display(notebook_response.text[:300])\n", + " button.disabled = False\n", + " button.description = \"Update\"\n", + " \n", + " return show\n", + "\n", + " update_button = widgets.Button(description=\"Update\", disable=False) \n", + "\n", + " update_button.on_click(_widget_callback_output_cell_1.capture(clear_output=True, wait=True)(button_update(main_folder, subfolder, row_idx=idx)))\n", + "\n", + " grid[idx, 5] = update_button\n", + "\n", + " needs_update = True\n", + " else:\n", + " grid[idx, 3] = table_cell(\"Not Found\")\n", + " grid[idx, 4] = table_cell(\"❌ Missing Notebook\")\n", + " needs_update = True\n", + "\n", + " # Open button (centered)\n", + " grid[idx, 6] = build_open_button(nb[\"path\"]) \n", + "\n", + " idx += 1\n", + " else:\n", + " for subfolder in online_latest_versions[main_folder]:\n", + " grid[idx, 0] = table_cell(subfolder)\n", + " grid[idx, 1] = table_cell(main_folder)\n", + " grid[idx, 2] = table_cell(\"-\")\n", + " grid[idx, 3] = table_cell(\"Not Found\")\n", + " grid[idx, 4] = table_cell(\"❌ Missing Notebook\")\n", + " grid[idx, 5] = table_cell(\"-\")\n", + " grid[idx, 6] = table_cell(\"-\")\n", + " needs_update = True\n", + "\n", + " idx += 1\n", + "\n", + " \n", + " if needs_update:\n", + " display(Markdown(\"⚠️ Some notebooks need to be updated.\"))\n", + " else:\n", + " display(Markdown(\"✅ All notebooks are up-to-date.\"))\n", + "\n", + " apply_grid_layout(grid)\n", + " display(grid, grip_output)\n", + "\n", + "version_file_path = \"notebooks/notebook_latest_versions.yaml\"\n", + "version_url = f\"https://api.github.com/repos/{github_owner}/{github_repo_name}/contents/{version_file_path}?ref={github_branch}\"\n", + "\n", + "project_version_file_path = \"construct.yaml\"\n", + "project_version_url = f\"https://api.github.com/repos/{github_owner}/{github_repo_name}/contents/{project_version_file_path}?ref={github_branch}\"\n", + "try:\n", + " version_response = requests.get(version_url)\n", + " project_version_response = requests.get(project_version_url)\n", + " error = False\n", + "except Exception as e:\n", + " load_table(None, None, notebooks)\n", + " error = True\n", + "\n", + "# Debug: show status and raw response if not 200\n", + "if not error and version_response.status_code != 200:\n", + " private_repository = True\n", + " explanation = widgets.HTML(\n", + " value=f\"\"\"\n", + " It looks like your repository might be private, please provide a Personal Access Token (PAT) to access it.\n", + " If you don't have one, you can create it by following this guide.\n", + " \"\"\"\n", + " )\n", + "\n", + " github_token = widgets.Password(\n", + " value=\"\",\n", + " description=\"*Personal Access Token:\",\n", + " placeholder=\"e.g. ghp_XXXXXXXXXXXXXXXXXXXX\",\n", + " style={'description_width': 'initial'},\n", + " layout=widgets.Layout(width='99%')\n", + " )\n", + " \n", + " test_button = widgets.Button(\n", + " description=\"Load Private GitHub Repo\",\n", + " button_style='success',\n", + " style={'description_width': 'initial'},\n", + " layout=widgets.Layout(width='99%')\n", + " )\n", + "\n", + " output_area = widgets.HTML()\n", + " table_area = widgets.Output()\n", + "\n", + " def on_test_button_clicked(b):\n", + " headers = {\n", + " \"Authorization\": f\"token {github_token.value}\"\n", + " }\n", + " auth_version_response = requests.get(version_url, headers=headers) \n", + " auth_project_version_response = requests.get(project_version_url, headers=headers)\n", + " if auth_version_response.status_code == 200 and auth_project_version_response.status_code == 200: \n", + " output_area.value = \"✅ Successfully accessed private repository!\"\n", + " with table_area:\n", + " clear_output()\n", + " load_table(auth_version_response, auth_project_version_response, notebooks)\n", + " else:\n", + " output_area.value = f\"❌ Failed to access private repository (HTTP {auth_version_response.status_code}).\"\n", + " output_area.value += f\"
{auth_version_response.text[:500]}
\"\n", + " with table_area:\n", + " clear_output()\n", + " load_table(None, None, notebooks)\n", + " test_button.on_click(on_test_button_clicked)\n", + "\n", + " display(explanation, github_token, test_button, output_area, table_area)\n", + "elif not error:\n", + " private_repository = False\n", + " load_table(version_response, project_version_response, notebooks)\n", + "\n", + "# Display callback-generated output directly below this cell.\n", + "display(_widget_callback_output_cell_1)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "labchronicle", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test_template_sync_0_1_20.py b/tests/test_template_sync_0_1_20.py index 356326f..995749f 100644 --- a/tests/test_template_sync_0_1_20.py +++ b/tests/test_template_sync_0_1_20.py @@ -535,7 +535,7 @@ def old(self, relative: Path) -> str: """The 0.1.19 file of an app (rendered with the app name).""" if relative in MIGRATION.REPLACED_FILES: return self.rendered(lf(MIGRATION.PREVIOUS_ROOT / relative)) - text = lf(ROOT / relative) + text = before_0_1_24(relative, (ROOT / relative).read_bytes()).decode("utf-8").replace("\r\n", "\n") # without later changes if relative == MIGRATION.POST_INSTALL_SH: for old, new in MIGRATION.SH_BLOCKS: text = text.replace(new, old) diff --git a/tests/test_template_sync_0_1_24.py b/tests/test_template_sync_0_1_24.py index 453bd42..02009ea 100644 --- a/tests/test_template_sync_0_1_24.py +++ b/tests/test_template_sync_0_1_24.py @@ -21,6 +21,7 @@ from _v0_1_24 import before_0_1_24 from _v0_1_25 import before_0_1_25 +from _v0_1_30 import before_0_1_30 ROOT = Path(__file__).resolve().parents[1] @@ -56,6 +57,11 @@ def new(relative: Path) -> str: return before_0_1_25(relative, (ROOT / relative).read_bytes()).decode("utf-8").replace("\r\n", "\n") +def live(relative: Path) -> str: + """The live file without the changes of later templates (0.1.30 added the requirements check to the release workflow).""" + return before_0_1_30(relative, (ROOT / relative).read_bytes()).decode("utf-8").replace("\r\n", "\n") + + def old(relative: Path) -> str: """The 0.1.23 file: the live file without the 0.1.24 change.""" return before_0_1_24(relative, (ROOT / relative).read_bytes()).decode("utf-8").replace("\r\n", "\n") @@ -100,7 +106,7 @@ def test_the_live_files_are_the_0_1_23_files_migrated(self) -> None: self.assertEqual(MIGRATION.apply_blocks(old(LAUNCHER), (MIGRATION.LAUNCHER_BLOCK,)), lf(ROOT / LAUNCHER)) text, count = MIGRATION.add_tag_lines(old(WORKFLOW)) self.assertEqual(count, 2) - self.assertEqual(MIGRATION.build_from_platforms(text), lf(ROOT / WORKFLOW)) + self.assertEqual(MIGRATION.build_from_platforms(text), live(WORKFLOW)) def test_without_platforms_the_matrix_is_the_old_four_builds_in_the_same_order(self) -> None: old_matrix = yaml.safe_load(textwrap.dedent(MIGRATION.MATRIX_OLD.split("strategy:\n", 1)[1]))["matrix"] @@ -119,7 +125,7 @@ def test_every_button_of_the_download_template_belongs_to_a_built_system(self) - self.assertTrue(link.endswith(endings), link) def test_the_workflow_builds_from_the_platforms_job(self) -> None: - jobs = yaml.safe_load(lf(ROOT / WORKFLOW))["jobs"] + jobs = yaml.safe_load(live(WORKFLOW))["jobs"] self.assertEqual(jobs["build-installers"]["needs"], ["bump-version", "platforms"]) self.assertEqual(jobs["build-installers"]["strategy"]["matrix"], "${{ fromJSON(needs.platforms.outputs.matrix) }}") self.assertEqual(jobs["platforms"]["outputs"]["matrix"], "${{ steps.matrix.outputs.matrix }}") @@ -413,7 +419,7 @@ def test_the_sync_script_runs_the_migration_from_0_1_23(self) -> None: result = subprocess.run(command, capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertIn("[migration 0.1.24]", result.stdout) - self.assertEqual(lf(self.repo / WORKFLOW), lf(ROOT / WORKFLOW)) + self.assertEqual(lf(self.repo / WORKFLOW), live(WORKFLOW)) again = subprocess.run(command, capture_output=True, text=True) self.assertIn("no repository changes were required", again.stdout) diff --git a/tests/test_template_sync_0_1_29.py b/tests/test_template_sync_0_1_29.py index aebe1b6..3ed13f7 100644 --- a/tests/test_template_sync_0_1_29.py +++ b/tests/test_template_sync_0_1_29.py @@ -47,7 +47,7 @@ def index_of_0_1_28() -> bytes: class TemplateTests(unittest.TestCase): def test_the_manifest_lists_the_migration_and_the_version(self) -> None: text = (ROOT / ".template_sync/manifest.yaml").read_text(encoding="utf-8") - self.assertIn("template_version: 0.1.29", text) + self.assertRegex(text, r"template_version: 0\.1\.(29|[3-9][0-9])\b") self.assertIn("script: migrations/v0_1_28_to_v0_1_29.py", text) def test_the_live_files_hold_each_correction_once_and_no_old_text(self) -> None: diff --git a/tests/test_template_sync_0_1_30.py b/tests/test_template_sync_0_1_30.py new file mode 100644 index 0000000..a981efd --- /dev/null +++ b/tests/test_template_sync_0_1_30.py @@ -0,0 +1,468 @@ +"""Tests of the 0.1.30 migration: requirements that disagree stop the merge (#48); the notebook update asks and keeps a backup (#49).""" +from __future__ import annotations + +import ast +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from types import ModuleType +from typing import Any, Callable +from unittest import mock + +from _v0_1_30 import WELCOME_0_1_29, before_0_1_30 + + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION_PATH = ROOT / ".template_sync/migrations/v0_1_29_to_v0_1_30.py" +SYNC_PATH = ROOT / ".template_sync/sync.py" +MERGE_PATH = ROOT / ".tools/python/merge_requirements.py" +ENVIRONMENT = "name: app\nchannels:\n- conda-forge\ndependencies:\n- python=3.11.13\n- pip\n- jupyterlab=4.4.0\n" + + +def load(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +MIGRATION = load(MIGRATION_PATH, "migration_0_1_30") + + +def lf(path: Path) -> str: + return path.read_bytes().decode("utf-8").replace("\r\n", "\n") + + +def old(relative: Path) -> str: + """The 0.1.29 file: the live file without the 0.1.30 change.""" + return before_0_1_30(relative, (ROOT / relative).read_bytes()).decode("utf-8").replace("\r\n", "\n") + + +def helpers() -> ModuleType: + return MIGRATION._load("v0_1_9_to_v0_1_10.py", "helpers_for_0_1_30_tests") + + +def table_cell(notebook: Path) -> str: + cells = json.loads(lf(notebook))["cells"] + return next("".join(c["source"]) for c in cells if c["cell_type"] == "code" and "def apply_grid_layout(" in "".join(c["source"])) + + +def backup_function() -> Callable[[Path, str], Path | None]: + """replace_notebook_with_backup of the live Welcome template, run on its own.""" + tree = ast.parse(table_cell(ROOT / MIGRATION.WELCOME_TEMPLATE).replace("DICT_OF_NOTEBOOKS", "{}")) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "replace_notebook_with_backup") + scope: dict[str, Any] = {"json": json, "shutil": shutil, "Path": Path} + exec(compile(ast.Module([function], []), "replace_notebook_with_backup", "exec"), scope) # noqa: S102 - the template's own function + return scope["replace_notebook_with_backup"] + + +class TemplateConsistencyTests(unittest.TestCase): + def test_the_manifest_lists_the_migration_and_the_new_version(self) -> None: + text = lf(ROOT / ".template_sync/manifest.yaml") + self.assertRegex(text, r"template_version: 0\.1\.([3-9][0-9])\b") + self.assertIn("script: migrations/v0_1_29_to_v0_1_30.py", text) + + def test_chains_from_every_older_version_reach_0_1_30(self) -> None: + sync = load(SYNC_PATH, "sync_for_0_1_30") + manifest = sync.parse_manifest(ROOT / ".template_sync/manifest.yaml") + for start in ("0.0.0", "0.1.16", "0.1.24", "0.1.29"): + self.assertEqual(sync.resolve_migration_chain(manifest["migrations"], start, "0.1.30")[-1]["to"], "0.1.30", start) + + def test_the_design_note_exists_and_the_migration_links_it(self) -> None: + self.assertTrue((ROOT / ".tools/docs/migration_0_1_30.md").is_file()) + self.assertIn(".tools/docs/migration_0_1_30.md", MIGRATION.DOCS_URL) + + def test_the_resources_are_the_live_merge_script_and_the_0_1_29_one(self) -> None: + self.assertEqual((MIGRATION.RESOURCE_ROOT / MIGRATION.MERGE).read_bytes(), MERGE_PATH.read_bytes()) + self.assertNotEqual(lf(MIGRATION.PREVIOUS_ROOT / MIGRATION.MERGE), lf(MERGE_PATH)) + self.assertIn('"--check"', lf(MERGE_PATH)) + + def test_the_live_files_are_the_0_1_29_files_migrated(self) -> None: + self.assertEqual(MIGRATION.apply_blocks(old(MIGRATION.NOTEBOOK_WORKFLOW), MIGRATION.NOTEBOOK_WORKFLOW_BLOCKS), lf(ROOT / MIGRATION.NOTEBOOK_WORKFLOW)) + self.assertEqual(MIGRATION.add_requirements_check(old(MIGRATION.RELEASE_WORKFLOW)), lf(ROOT / MIGRATION.RELEASE_WORKFLOW)) + self.assertEqual(old(MIGRATION.TOOLS_REQUIREMENTS) + MIGRATION.PACKAGING_LINES, lf(ROOT / MIGRATION.TOOLS_REQUIREMENTS)) + with tempfile.TemporaryDirectory() as tmp: + welcome = Path(tmp) / "Welcome_template.ipynb" + shutil.copy(WELCOME_0_1_29, welcome) + self.assertTrue(MIGRATION._update_welcome(helpers(), welcome).startswith("updated")) + self.assertEqual(welcome.read_bytes(), (ROOT / MIGRATION.WELCOME_TEMPLATE).read_bytes()) + + def test_the_release_checks_the_requirements_in_the_bump_version_job_before_the_commit(self) -> None: + text = lf(ROOT / MIGRATION.RELEASE_WORKFLOW) + check = text.index("merge_requirements.py --source-dir notebooks --output requirements.txt --check") + self.assertLess(text.index("python .tools/python/check_placeholders.py"), check) + self.assertLess(check, text.index("Commit updated versioned files")) + self.assertEqual(text.count("--output requirements.txt --check"), 1) + + def test_the_tools_install_packaging(self) -> None: + self.assertIn(MIGRATION.PACKAGING_PIN, lf(ROOT / MIGRATION.TOOLS_REQUIREMENTS).splitlines()) + + +class MergeCase(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + (self.root / "environment.yaml").write_text(ENVIRONMENT, encoding="utf-8") + + def tearDown(self) -> None: + self._tmp.cleanup() + + def notebook(self, name: str, *dependencies: str, **os_lists: list[str]) -> None: + folder = self.root / "notebooks" / name + folder.mkdir(parents=True, exist_ok=True) + data: dict[str, Any] = {"dependencies": list(dependencies), "python_version": "3.11.13", **os_lists} + (folder / "requirements.yaml").write_text(json.dumps(data), encoding="utf-8") # JSON is YAML + + def overrides(self, text: str) -> None: + (self.root / "requirements_overrides.yaml").write_text(text, encoding="utf-8") + + def merge(self, *extra: str) -> subprocess.CompletedProcess[str]: + command = [sys.executable, "-W", "ignore", str(MERGE_PATH), "--source-dir", "notebooks", "--output", "requirements.txt", *extra] + return subprocess.run(command, cwd=self.root, capture_output=True, text=True) + + def lines(self, name: str = "requirements.txt") -> list[str]: + return [line for line in lf(self.root / name).splitlines() if line and not line.startswith("#")] + + def assert_conflict(self, result: subprocess.CompletedProcess[str], *expected: str) -> None: + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + for text in expected: + self.assertIn(text, result.stderr) + self.assertFalse((self.root / "requirements.txt").exists(), "nothing is written when requirements conflict") + + +class MergeRequirementsTests(MergeCase): + def test_different_exact_pins_stop_the_merge_in_either_file_order(self) -> None: + for first, second in (("A", "B"), ("B", "A")): + with self.subTest(order=first + second): + shutil.rmtree(self.root / "notebooks", ignore_errors=True) + self.notebook(first, "numpy==1.26.4") + self.notebook(second, "numpy==2.0.0") + self.assert_conflict(self.merge(), "numpy: pinned to different versions (1.26.4, 2.0.0)", + f"notebooks/{first}/requirements.yaml: numpy==1.26.4", f"notebooks/{second}/requirements.yaml: numpy==2.0.0", + "requirements_overrides.yaml") + + def test_identical_pins_and_spellings_of_one_package_merge_into_one_line(self) -> None: + self.notebook("A", "NumPy==1.26.4", "scikit_image==0.25.2") + self.notebook("B", "numpy==1.26.4", "scikit-image==0.25.2") + result = self.merge() + self.assertEqual(result.returncode, 0, result.stderr) + lines = self.lines() + self.assertIn("NumPy==1.26.4", lines) + self.assertIn("scikit_image==0.25.2", lines) + self.assertEqual(sum(line.lower().startswith(("numpy", "scikit")) for line in lines), 2) + + def test_requirements_that_are_not_pins_are_kept_as_written_and_combined(self) -> None: + self.notebook("A", "numpy<2.0", "pandas>=2.0,<3") + self.notebook("B", "numpy>=1.24") + self.assertEqual(self.merge().returncode, 0) + self.assertIn("numpy<2.0,>=1.24", self.lines()) + self.assertIn("pandas<3,>=2.0", self.lines()) + + def test_a_pin_inside_the_other_ranges_is_used(self) -> None: + self.notebook("A", "numpy>=1.24,<2") + self.notebook("B", "numpy==1.26.4") + self.assertEqual(self.merge().returncode, 0) + self.assertIn("numpy==1.26.4", self.lines()) + + def test_a_pin_outside_another_range_and_ranges_no_version_meets_stop_the_merge(self) -> None: + self.notebook("A", "numpy<2") + self.notebook("B", "numpy==2.0.0") + self.assert_conflict(self.merge(), "the pin ==2.0.0 does not meet numpy<2 of notebooks/A/requirements.yaml") + self.notebook("B", "numpy>=2.1") + self.assert_conflict(self.merge(), "numpy: no version meets all of these requirements") + + def test_requirements_for_different_platforms_stay_separate_lines(self) -> None: + self.notebook("A", 'numpy==1.26.4 ; sys_platform == "win32"') + self.notebook("B", 'numpy==2.0.0 ; sys_platform != "win32"') + self.assertEqual(self.merge().returncode, 0) + self.assertIn('numpy==1.26.4 ; sys_platform == "win32"', self.lines()) + self.assertIn('numpy==2.0.0 ; sys_platform != "win32"', self.lines()) + + def test_extras_are_combined(self) -> None: + self.notebook("A", "dask[array]==2024.8.0") + self.notebook("B", "dask[dataframe]==2024.8.0") + self.assertEqual(self.merge().returncode, 0) + self.assertIn("dask[array,dataframe]==2024.8.0", self.lines()) + + def test_an_os_specific_pin_is_checked_against_the_common_one(self) -> None: + self.notebook("A", "numpy==1.26.4", linux_dependencies=["numpy==2.0.0"]) + self.assert_conflict(self.merge(), "numpy (linux_dependencies, with the common dependencies): pinned to different versions") + self.notebook("A", "numpy==1.26.4", linux_dependencies=["numpy>=1.20"]) + self.assertEqual(self.merge().returncode, 0) + self.assertIn("numpy>=1.20", self.lines("requirements-linux.txt")) + + def test_an_override_decides_and_is_written_with_its_reason_in_every_file(self) -> None: + self.notebook("A", "numpy==1.26.4", "tensorflow==2.20.0") + self.notebook("B", "numpy==2.0.0") + self.overrides('overrides:\n NumPy:\n use: "==2.0.0"\n reason: "A was re-run with 2.0.0: same results."\n') + result = self.merge() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("numpy==2.0.0", self.lines()) + for name in ("requirements.txt", "requirements_gpu.txt"): + header = [line for line in lf(self.root / name).splitlines() if line.startswith("# Decided")] + self.assertEqual(len(header), 1, name) + self.assertIn("notebooks/A/requirements.yaml: numpy==1.26.4", header[0]) + self.assertIn("Reason: A was re-run with 2.0.0: same results.", header[0]) + + def test_an_override_needs_a_reason_and_a_version_requirement(self) -> None: + self.notebook("A", "numpy==1.26.4") + self.overrides('overrides:\n numpy:\n use: "==2.0.0"\n') + self.assert_conflict(self.merge(), "numpy needs `reason`") + self.overrides('overrides:\n numpy:\n use: "2.0.0"\n reason: "x"\n') + self.assert_conflict(self.merge(), "is not a version requirement") + + def test_an_override_no_notebook_uses_is_reported(self) -> None: + self.notebook("A", "numpy==1.26.4") + self.overrides('overrides:\n torch:\n use: "==2.9.0"\n reason: "old"\n') + result = self.merge() + self.assertEqual(result.returncode, 0) + self.assertIn("decides torch, which no notebook requires any more", result.stdout) + + def test_check_writes_nothing_and_fails_on_a_conflict(self) -> None: + self.notebook("A", "numpy==1.26.4") + self.notebook("B", "numpy==1.26.4") + result = self.merge("--check") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sorted(p.name for p in self.root.iterdir()), ["environment.yaml", "notebooks"]) + self.assertEqual(lf(self.root / "environment.yaml"), ENVIRONMENT) + self.notebook("B", "numpy==2.0.0") + self.assert_conflict(self.merge("--check"), "pinned to different versions") + + def test_the_app_packages_are_still_added(self) -> None: + self.notebook("A", "numpy==1.26.4") + self.assertEqual(self.merge().returncode, 0) + self.assertIn("jl-hidecode==0.0.1", self.lines()) + self.assertIn("ipywidgets==8.1.7", self.lines()) + + +class SatisfiableTests(unittest.TestCase): + def test_ranges(self) -> None: + from packaging.specifiers import SpecifierSet + + merge = load(MERGE_PATH, "merge_requirements_0_1_30") + for spec, expected in (("<2,>=2.1", False), (">=1.24,<2", True), (">1.0,<1.1", True), ("~=1.11.0,<1.11", False), + ("==1.*,>=1.5", True), ("==1.*,>=2", False), ("!=1.0", True), ("", True)): + with self.subTest(spec=spec): + self.assertEqual(merge.satisfiable(SpecifierSet(spec)), expected) + + +class NotebookBackupTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.notebook = Path(self._tmp.name) / "Folder" / "Analysis.ipynb" + self.replace = backup_function() + + def tearDown(self) -> None: + self._tmp.cleanup() + + def files(self) -> list[str]: + return sorted(p.name for p in self.notebook.parent.iterdir()) + + def test_a_first_install_needs_no_backup(self) -> None: + self.assertIsNone(self.replace(self.notebook, '{"cells": []}')) + self.assertEqual(self.files(), ["Analysis.ipynb"]) + + def test_the_user_copy_with_changes_and_outputs_is_kept(self) -> None: + mine = '{"cells": [{"source": "my parameter", "outputs": ["my result"]}]}' + self.notebook.parent.mkdir(parents=True) + self.notebook.write_text(mine, encoding="utf-8") + backup = self.replace(self.notebook, '{"cells": ["new"]}') + assert backup is not None + self.assertRegex(backup.name, r"^Analysis\.backup-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.ipynb$") + self.assertEqual(backup.read_text(encoding="utf-8"), mine) + self.assertEqual(self.notebook.read_text(encoding="utf-8"), '{"cells": ["new"]}') + second = self.replace(self.notebook, '{"cells": ["newer"]}') + assert second is not None + self.assertNotEqual(second, backup, "an earlier backup is never overwritten") + self.assertEqual(second.read_text(encoding="utf-8"), '{"cells": ["new"]}') + self.assertEqual(len(self.files()), 3) + + def test_a_download_that_is_not_a_notebook_changes_nothing(self) -> None: + self.notebook.parent.mkdir(parents=True) + self.notebook.write_text('{"cells": ["mine"]}', encoding="utf-8") + for content in ("rate limit", "[]"): + with self.subTest(content=content), self.assertRaises(ValueError): + self.replace(self.notebook, content) + self.assertEqual(self.notebook.read_text(encoding="utf-8"), '{"cells": ["mine"]}') + self.assertEqual(self.files(), ["Analysis.ipynb"]) + + def test_a_failed_write_keeps_the_notebook_and_leaves_no_temporary_file(self) -> None: + self.notebook.parent.mkdir(parents=True) + self.notebook.write_text('{"cells": ["mine"]}', encoding="utf-8") + with mock.patch("os.replace", side_effect=OSError("disk full")), self.assertRaises(OSError): + self.replace(self.notebook, '{"cells": ["new"]}') + self.assertEqual(self.notebook.read_text(encoding="utf-8"), '{"cells": ["mine"]}') + self.assertFalse(any(name.endswith(".update") for name in self.files())) + + def test_the_handler_asks_first_and_records_the_version_after_the_replacement(self) -> None: + cell = table_cell(ROOT / MIGRATION.WELCOME_TEMPLATE) + handler = cell[cell.index("def button_update(") :] + self.assertNotIn("local_notebook_path.write_text", cell) + replace = handler.index("backup_path = replace_notebook_with_backup(local_notebook_path, notebook_content)") + self.assertLess(replace, handler.index("local_latest_versions[main_folder][subfolder] = new_version")) + self.assertLess(handler.index('widgets.Button(description="Back up and update"'), handler.index("return show")) + self.assertIn("confirm.on_click(", handler) + self.assertIn("nothing was changed", handler) + compile(cell.replace("DICT_OF_NOTEBOOKS", "{}"), "Welcome table cell", "exec") + + +class MigrationCase(unittest.TestCase): + NUCLEISKY_LIKE = { + "App": ["torch==2.10.0", "tifffile==2026.4.11", "numpy==2.0.2", "scikit-image==0.26.0"], + "Benchmark": ["torch==2.9.0", "tifffile==2026.1.28", "numpy==2.0.2", "scikit-image==0.25.2"], + } + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.repo = Path(self._tmp.name) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def put(self, relative: Path | str, text: str, newline: str = "\n") -> Path: + path = self.repo / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.replace("\n", newline).encode("utf-8")) + return path + + def put_0_1_29_app(self, notebooks: dict[str, list[str]] | None = None, newline: str = "\n") -> None: + self.put(MIGRATION.MERGE, lf(MIGRATION.PREVIOUS_ROOT / MIGRATION.MERGE), newline) + for relative in (MIGRATION.TOOLS_REQUIREMENTS, MIGRATION.NOTEBOOK_WORKFLOW, MIGRATION.RELEASE_WORKFLOW): + self.put(relative, old(relative), newline) + self.put(MIGRATION.WELCOME_TEMPLATE, lf(WELCOME_0_1_29), newline) + self.put("environment.yaml", ENVIRONMENT) + for name, dependencies in (notebooks or {}).items(): + self.put(f"notebooks/{name}/requirements.yaml", "python_version: 3.11.13\ndependencies:\n" + "".join(f"- {d}\n" for d in dependencies)) + + def run_migration(self) -> str: + buffer = StringIO() + with redirect_stdout(buffer): + MIGRATION.migrate(self.repo, {}) + return buffer.getvalue() + + def merge(self, *extra: str) -> subprocess.CompletedProcess[str]: + command = [sys.executable, "-W", "ignore", str(self.repo / MIGRATION.MERGE), "--source-dir", "notebooks", "--output", "requirements.txt", *extra] + return subprocess.run(command, cwd=self.repo, capture_output=True, text=True) + + +class MigrationBehaviourTests(MigrationCase): + def test_a_0_1_29_app_reaches_the_0_1_30_files_and_a_second_run_changes_nothing(self) -> None: + self.put_0_1_29_app() + output = self.run_migration() + for relative in (MIGRATION.MERGE, MIGRATION.TOOLS_REQUIREMENTS, MIGRATION.NOTEBOOK_WORKFLOW, MIGRATION.RELEASE_WORKFLOW, MIGRATION.WELCOME_TEMPLATE): + with self.subTest(file=str(relative)): + self.assertRegex(output, rf"{re.escape(str(relative))}: updated") + self.assertEqual(lf(self.repo / relative), lf(ROOT / relative)) + self.assertIn(f"{MIGRATION.OVERRIDES}: not needed", output) + self.assertNotIn("SKIPPED", output) + before = {p: p.read_bytes() for p in self.repo.rglob("*") if p.is_file()} + again = self.run_migration() + self.assertIn("no repository changes were required", again) + self.assertEqual({p: p.read_bytes() for p in self.repo.rglob("*") if p.is_file()}, before) + + def test_pins_that_disagree_are_recorded_and_the_app_keeps_its_versions(self) -> None: + self.put_0_1_29_app(self.NUCLEISKY_LIKE) + before = subprocess.run([sys.executable, "-W", "ignore", str(MIGRATION.PREVIOUS_ROOT / MIGRATION.MERGE), "--source-dir", "notebooks", + "--output", "before.txt"], cwd=self.repo, capture_output=True, text=True) + self.assertEqual(before.returncode, 0, before.stderr) + new_merge = subprocess.run([sys.executable, "-W", "ignore", str(MERGE_PATH), "--source-dir", "notebooks", "--output", "requirements.txt", "--check"], + cwd=self.repo, capture_output=True, text=True) + self.assertEqual(new_merge.returncode, 1, "the 0.1.29 notebooks conflict for the new merge") + output = self.run_migration() + self.assertIn("requirements_overrides.yaml: updated: created; it records the version used until now for scikit-image, tifffile, torch", output) + decisions = lf(self.repo / MIGRATION.OVERRIDES) + self.assertIn(' torch:\n use: "==2.10.0"\n', decisions) + self.assertIn(' scikit-image:\n use: "==0.26.0"\n', decisions) + self.assertIn("notebooks/App/requirements.yaml ==2.10.0, notebooks/Benchmark/requirements.yaml ==2.9.0", decisions) + self.assertNotIn("numpy", decisions) + result = self.merge() + self.assertEqual(result.returncode, 0, result.stderr) + pins = {line.split("==")[0].lower().replace("_", "-"): line.split("==")[1] for line in lf(self.repo / "requirements.txt").splitlines() if "==" in line and not line.startswith("#")} + old_pins = {line.split("==")[0].lower().replace("_", "-"): line.split("==")[1] for line in lf(self.repo / "before.txt").splitlines() if "==" in line and not line.startswith("#")} + self.assertEqual(pins, old_pins, "the recorded decisions give the versions the 0.1.29 merge gave") + self.assertEqual(lf(self.repo / "requirements.txt").count("# Decided in requirements_overrides.yaml"), 3) + + def test_spellings_of_one_package_are_one_decision(self) -> None: + self.put_0_1_29_app({"A": ["scikit-image==0.26.0"], "B": ["scikit_image==0.25.2"], "C": ["Scikit.Image==0.25.2"]}) + self.run_migration() + decisions = lf(self.repo / MIGRATION.OVERRIDES) + self.assertEqual(decisions.count(" scikit-image:\n"), 1) + self.assertIn('use: "==0.26.0"', decisions) + self.assertEqual(self.merge().returncode, 0) + + def test_without_yaml_the_simple_list_forms_are_read(self) -> None: + text = "python_version: 3.11.1\ndependencies: \n - ipywidgets==8.1.2 # widgets\n - 'pyyaml==6.0.2'\ndescription: \"x\"\nlinux_dependencies: []\n" + with mock.patch.dict(sys.modules, {"yaml": None}): + self.assertEqual(MIGRATION.notebook_dependencies(text), ["ipywidgets==8.1.2", "pyyaml==6.0.2"]) + self.assertEqual(MIGRATION.notebook_dependencies("dependencies: [numpy==1.0, 'scipy==1.1']\n"), ["numpy==1.0", "scipy==1.1"]) + + def test_crlf_files_keep_their_line_endings(self) -> None: + self.put_0_1_29_app(newline="\r\n") + self.run_migration() + for relative in (MIGRATION.MERGE, MIGRATION.TOOLS_REQUIREMENTS, MIGRATION.NOTEBOOK_WORKFLOW, MIGRATION.RELEASE_WORKFLOW): + with self.subTest(file=str(relative)): + data = (self.repo / relative).read_bytes() + self.assertNotIn(b"\n", data.replace(b"\r\n", b"")) + self.assertEqual(data.decode("utf-8").replace("\r\n", "\n"), lf(ROOT / relative)) + + def test_a_customised_merge_script_is_kept_and_the_release_does_not_call_check(self) -> None: + self.put_0_1_29_app(self.NUCLEISKY_LIKE) + custom = self.put(MIGRATION.MERGE, lf(MIGRATION.PREVIOUS_ROOT / MIGRATION.MERGE) + "# my change\n") + output = self.run_migration() + self.assertIn(f"{MIGRATION.MERGE}: SKIPPED: customised", output) + self.assertIn(f"{MIGRATION.RELEASE_WORKFLOW}: SKIPPED: the repository's merge_requirements.py has no --check", output) + self.assertTrue(lf(custom).endswith("# my change\n")) + self.assertNotIn("--check", lf(self.repo / MIGRATION.RELEASE_WORKFLOW)) + self.assertTrue((self.repo / MIGRATION.OVERRIDES).is_file()) + + def test_an_existing_overrides_file_and_customised_files_are_left_as_they_are(self) -> None: + self.put_0_1_29_app(self.NUCLEISKY_LIKE) + mine = self.put(MIGRATION.OVERRIDES, "overrides: {}\n") + welcome = self.repo / MIGRATION.WELCOME_TEMPLATE + welcome.write_text(lf(welcome).replace("def button_update(main_folder, subfolder, row_idx):", "def my_update(main_folder, subfolder, row_idx):"), encoding="utf-8") + workflow = self.put(MIGRATION.NOTEBOOK_WORKFLOW, "name: mine\n") + output = self.run_migration() + self.assertIn(f"{MIGRATION.OVERRIDES}: already exists: unchanged", output) + self.assertIn(f"{MIGRATION.WELCOME_TEMPLATE}: SKIPPED", output) + self.assertIn(f"{MIGRATION.NOTEBOOK_WORKFLOW}: SKIPPED", output) + self.assertEqual(lf(mine), "overrides: {}\n") + self.assertEqual(lf(workflow), "name: mine\n") + self.assertIn("def my_update(", lf(welcome)) + + def test_missing_files_are_not_created(self) -> None: + output = self.run_migration() + self.assertIn(f"{MIGRATION.MERGE}: not present", output) + self.assertIn(f"{MIGRATION.OVERRIDES}: not needed", output) + self.assertEqual(list(self.repo.rglob("*")), []) + + +class ChainTests(MigrationCase): + def test_the_sync_script_runs_the_migration_from_0_1_29(self) -> None: + self.put_0_1_29_app(self.NUCLEISKY_LIKE) + shutil.copytree(ROOT / ".template_sync", self.repo / ".template_sync") + # sync.py refuses a target other than the manifest version: pin the copy to 0.1.30 (later templates add migrations). + manifest = self.repo / ".template_sync/manifest.yaml" + manifest.write_text(re.sub(r"template_version: [0-9.]+", "template_version: 0.1.30", manifest.read_text(encoding="utf-8")), encoding="utf-8") + command = [sys.executable, str(self.repo / ".template_sync/sync.py"), "--from-version", "0.1.29", "--to-version", "0.1.30", "--repository-root", str(self.repo)] + env = {key: value for key, value in os.environ.items() if key != "PYTHONPATH"} + result = subprocess.run(command, capture_output=True, text=True, env=env) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("[migration 0.1.30]", result.stdout) + self.assertEqual(lf(self.repo / MIGRATION.MERGE), lf(MERGE_PATH)) + self.assertEqual(self.merge("--check").returncode, 0) + again = subprocess.run(command, capture_output=True, text=True, env=env) + self.assertIn("no repository changes were required", again.stdout) + + +if __name__ == "__main__": + unittest.main()