diff --git a/.github/journal-utilities-pin.json b/.github/journal-utilities-pin.json index de6156768..69444c4b6 100644 --- a/.github/journal-utilities-pin.json +++ b/.github/journal-utilities-pin.json @@ -5,10 +5,10 @@ "recorded": "2026-11-12", "roles": { "gate": "integrity gate, Pages builder and local ./run.sh (docs, fast, full)", - "ingest": "weekly-ingest only: journal_utilities.ingest.ytdlp does not exist at the gate revision (verified: 4 ingest modules at the gate pin, 5 at this one)" + "ingest": "weekly ingest and public scaffold preview with playlist-aware routing" }, "ingest": { - "revision": "ebdd09147422f330b12ef22f3d6b0c069d950359", - "recorded": "2026-11-12" + "revision": "edce353c1cf0ec8b108ef3606669507085acca27", + "recorded": "2026-11-20" } } diff --git a/AGENTS.md b/AGENTS.md index f679d90d3..e840506d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,7 @@ curated materials from the Active Inference Institute video library. Read - **Human:** [`INDEX.md`](INDEX.md), [`README.md`](README.md). - **Content:** [`data/video/activeinferenceinstitute/`](data/video/activeinferenceinstitute/). - **Schema/docs:** [`docs/`](docs/README.md) (`SCHEMA.md`, `ORGANIZATION.md`, `PIPELINE.md`, `OPERATIONS.md`). +- **YouTube:** [channel handoff and public ingest review](docs/youtube/HANDOFF.md). - **Open work:** [`TO-DO.md`](TO-DO.md) — scoped improvement list (docs pass 2026-08-02, refreshed pass 4 2026-11-12). ## Rules for agents diff --git a/TO-DO.md b/TO-DO.md index cd67fa660..02d0e06c8 100644 --- a/TO-DO.md +++ b/TO-DO.md @@ -205,11 +205,11 @@ overhaul, cross-cutting refactors. Each remaining file needs a human to confirm its `video_id` and language before the two-step `git mv` — no script can settle a guess, which is why the M2 pass stopped here. Owner: volunteers, coordinated from that page. -- **Weekly ingest remains a dispatch-only draft; scheduling/policy is pending.** - `weekly-ingest.yml` has no `schedule:` and rejects non-dispatch runs. - `arm_manifest.py --strict` already exists and fails age/duplicate-ID warnings; - `full` does not select it. Owner: maintainers, after reviewing the producer's - `docs/m3-ingest.md` and deliberately deciding scheduling and strictness. +- **Weekly public ingest.** The [YouTube handoff](docs/youtube/HANDOFF.md#review-the-public-ingest-backlog) + documents Data API capture, public backlog review, and dry-run scaffolding on + an approved self-hosted machine. Maintainers own credential setup, runner + validation, weekly scheduling, and review of the first measured backlog. + The GitHub ingest workflow remains dispatch-only. - **Coordinate member adoption of the Journal's GNN pin.** The lead pin advances to `v5.1.0` (`75c834bd`). Each of the eight member copies moves with its own conformance evidence after the lead revision is accepted. Follow the diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 3e23acfe7..6c9ab0b61 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -120,6 +120,9 @@ and uploads an ingest worklist, a channel manifest and a reconciliation report a artifacts. Add a `schedule:` only after reviewing Journal-Utilities `docs/m3-ingest.md` and removing the guard. +For the self-hosted Data API inventory and public backlog review, use the +[YouTube handoff](youtube/HANDOFF.md#review-the-public-ingest-backlog). + ## Dependency updates [`.github/dependabot.yml`](../.github/dependabot.yml) opens grouped, weekly pull diff --git a/docs/README.md b/docs/README.md index d6b943017..f5ef039ca 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ materials from the Active Inference Institute video library. | [`ORGANIZATION.md`](ORGANIZATION.md) | The top-level namespace (`data/video//`, `data///`), how series and items are organized, and how to add other channels / non-video sources. | | [`PIPELINE.md`](PIPELINE.md) | How content is produced and kept complete + idempotent — the collaboration with the [Journal_Utilities](https://github.com/ActiveInferenceInstitute/Journal_Utilities) engine. | | [`OPERATIONS.md`](OPERATIONS.md) | The gate (`./run.sh` arms), the CI workflows (integrity, GNN conformance, Pages deploy, weekly ingest), pins, dependency updates, local validation, and the branch model. | +| [`youtube/HANDOFF.md`](youtube/HANDOFF.md) | Channel handoff, public backlog review, captions, and editorial follow-through. | | [`m4-site-spec.md`](m4-site-spec.md) | Spec for the M4 static per-item pages + `sitemap.xml`/`robots.txt`, implemented by the Journal-Utilities builder. | | [`translation-migration-map.md`](translation-migration-map.md) | Item-by-item map of the legacy `Translations/` → `translations/..srt` migration; status: executed (part 2 + pass 2b), 423 files remain in `Translations/`. | | [`volunteer-transcript-tasks.md`](volunteer-transcript-tasks.md) | Volunteer triage list (I16): the 423 remaining translation files and the 600-file conflicts backlog, by item. | diff --git a/docs/documentation-index.md b/docs/documentation-index.md index cfe44bcc9..3792169f7 100644 --- a/docs/documentation-index.md +++ b/docs/documentation-index.md @@ -12,6 +12,7 @@ only by memory. Generated facts live in [`_generated/`](_generated/README.md). | [`ORGANIZATION.md`](ORGANIZATION.md) | Top-level namespace (`data/video//`, `data///`), series and item layout, adding other channels or non-video sources. | hand-written | | [`PIPELINE.md`](PIPELINE.md) | How content is produced and kept complete and idempotent, and the Journal-Utilities boundary. | hand-written | | [`OPERATIONS.md`](OPERATIONS.md) | CI workflows, the gate arms (`./run.sh`), pins, dependency updates, local validation, branch model. | hand-written | +| [`youtube/HANDOFF.md`](youtube/HANDOFF.md) | Channel handoff, public backlog review, and channel editorial workflows. | hand-written | | [`m4-site-spec.md`](m4-site-spec.md) | Spec for the static per-item pages plus `sitemap.xml`/`robots.txt`, implemented by the Journal-Utilities builder. | hand-written | | [`translation-migration-map.md`](translation-migration-map.md) | Item-by-item map of the legacy `Translations/` migration and the quarantined conflicts. | hand-written | | [`volunteer-transcript-tasks.md`](volunteer-transcript-tasks.md) | Volunteer triage list of remaining translation files and conflict backlog. | hand-written | diff --git a/docs/youtube/HANDOFF.md b/docs/youtube/HANDOFF.md new file mode 100644 index 000000000..532b37815 --- /dev/null +++ b/docs/youtube/HANDOFF.md @@ -0,0 +1,99 @@ +# Active Inference Institute YouTube handoff + +The November 20, 2026 handoff was reviewed against Journal commit `cd32e2f4`. +The public channel is [@ActiveInference](https://www.youtube.com/@ActiveInference), +with channel ID `UCbPq2w41ZaJSWtpCq4BE6Dg`. Use the channel ID for inventory +and match entries to the Journal by the YouTube video ID in +[`INDEX.json`](../../INDEX.json). + +## Work already recorded + +The handoff records updated upload defaults and an added Journal link in the +default description. The recent channel pass corrected broken description +links, put the affected textbook playlists in session order, and standardized +the Parr cohort playlist titles. GuestStreams and ModelStreams were enabled +as podcasts. The English-caption pilot covered Insights 001, 003, and 021. +Use a fresh audit to find remaining gaps before proposing additional edits. + +## Review the public ingest backlog + +Run the read-only Data API audit from a Journal-Utilities checkout at the +reviewed [ingest pin](../../.github/journal-utilities-pin.json). Supply the +restricted `YOUTUBE_API_KEY` from the local credential store. +Keep report paths in a run directory on the approved private storage volume. +From the Journal root, use absolute paths for the utilities checkout and the +existing report directory. + +```sh +journal="$(pwd -P)" +utilities="/path/to/Journal_Utilities" +reports="/path/to/private/report-directory" + +uv run --project "$utilities" --frozen --extra youtube python "$utilities/scripts/youtube_channel_audit.py" audit \ + --journal "$journal" --worklist "$reports/public-backlog.json" && +python3 scripts/review_youtube_ingest.py \ + --worklist "$reports/public-backlog.json" --output "$reports/reviewed-worklist.json" && +uv run --project "$utilities" --frozen python -m journal_utilities.ingest.scaffold \ + --journal "$journal" --worklist "$reports/reviewed-worklist.json" +``` + +The audit combines the channel's uploads with videos in its public playlists, +deduplicates video IDs, and verifies channel ownership and completion before +saving the public backlog. The Journal review requires explicit public, +processed, completed status for every row, refuses repeated IDs, and removes +IDs already covered by the current Journal index. It preserves public playlist +titles as labels for human review. The dry-run scaffolder gives established +video title patterns precedence and uses approved public playlist mappings +to route selected series. The classifier supplies the directory names and +derives item names from video metadata. + +Prioritize the 2026 Applied Active Inference Symposium, Fundamentals Cohort 1, +and Parr Cohorts 8 and 9. Then review GuestStream, ModelStream, and GEO-INFER. +Use `Applied Active Inference Symposium/2026` for the 2026 Symposium and +`TextbookGroup/ParrPezzuloFriston2022/Cohort_9` for Parr Cohort 9 under the +Journal's [video namespace](../../data/video/activeinferenceinstitute/). +Create item paths from reviewed public source metadata and retain each video's +correct series and session. Unmatched sessions need source verification before +they become Journal records. + +Use API enumeration for the weekly inventory and run media acquisition on an +approved self-hosted machine. The [GitHub workflow](../OPERATIONS.md#weekly-ingest-draft) +is dispatch-only. The local job can run on the approved host's weekly schedule +once its credential and storage bindings have been validated. For the weekly +QA run, add `--fail-on-findings` to the audit and retain its report for review. + +After ingest, regenerate the Journal indexes using Journal-Utilities and run +`./run.sh full`. Add a `RESOURCES & TRANSCRIPT` backlink using the published +item URL once the Journal item is live. Add cohort and Symposium playlist +backlinks after their Journal collections are published. + +## Captions, descriptions, and chapters + +- Render an English SRT from each item's timed WhisperX `transcript.json`. + Review cue timing, speaker changes, and named entities against the item + metadata. Use reviewed name corrections for Karl Friston, Iain McGilchrist, + and Darius Parvizi-Wayne where applicable. +- Keep existing human-edited English tracks. Resolve conflicting video and + metadata language settings before submitting an English track. Use the + reviewed English track as the translation source and record each completed + upload by video ID and SRT SHA-256 in the private operator ledger. +- Review link-audit findings and exact replacement URLs before updating a + description. Keep each series' structure and use the Institute's maintained + footer. Preserve series-first titles and the Fundamentals session wording, + applying only verified date or typo corrections to those sessions. +- Spot-check chapter proposals against the video. Start at `0:00` and require + at least three entries separated by ten seconds or more. Keep automatic + chapters available where no reviewed manual list exists. + +The [Journal-Utilities YouTube guide](https://github.com/ActiveInferenceInstitute/Journal_Utilities/blob/main/docs/youtube.md) +describes the tooling boundary. Record approval and verification for channel +mutations in the private operator record. Channel visibility decisions, caption +conflicts, and auto-dubbing remain with the channel owner. + +## Editorial follow-through + +Keep CC BY and the standing trailer decision. Review series thumbnail designs +before comparing their performance in Studio. Short-form edits need a +self-contained excerpt, captions, human review, and a source-video link. +YouTube Studio is the review path for end screens, cards, featured video, and +channel home sections. Review any newcomer playlist with the channel owner. diff --git a/scripts/review_youtube_ingest.py b/scripts/review_youtube_ingest.py new file mode 100644 index 000000000..e1f093026 --- /dev/null +++ b/scripts/review_youtube_ingest.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Prepare a public channel worklist for the Journal-Utilities scaffold preview.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import tempfile +from datetime import datetime +from pathlib import Path + + +CHANNEL_ID = "UCbPq2w41ZaJSWtpCq4BE6Dg" +VIDEO_ID = re.compile(r"[A-Za-z0-9_-]{11}") +PUBLIC_STATE = { + "privacy_status": "public", + "upload_status": "processed", + "broadcast_status": "none", + "publication_state": "published", +} + + +def index_video_ids(index: object) -> set[str]: + items = index.get("items") if isinstance(index, dict) else None + if not isinstance(items, list) or not items: + raise ValueError("INDEX.json must contain a nonempty items list") + video_ids: set[str] = set() + for position, item in enumerate(items, 1): + parts = item.get("parts") if isinstance(item, dict) else None + if not isinstance(parts, list) or any( + not isinstance(video_id, str) or VIDEO_ID.fullmatch(video_id) is None + for video_id in parts + ): + raise ValueError(f"INDEX.json item {position} has invalid video IDs") + video_ids.update(parts) + if not video_ids: + raise ValueError("INDEX.json contains no video IDs") + return video_ids + + +def timestamp(value: object, field: str) -> str: + if isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + pass + else: + if parsed.utcoffset() is not None: + return value + raise ValueError(f"{field} must be an ISO 8601 timestamp with an offset") + + +def public_entry(entry: object, position: int) -> dict: + label = f"worklist row {position}" + if not isinstance(entry, dict): + raise ValueError(f"{label} must be an object") + if entry.get("is_scheduled") is not False or any( + entry.get(field) != value for field, value in PUBLIC_STATE.items() + ): + raise ValueError(f"{label} must be a completed public video") + video_id = entry.get("video_id") + if not isinstance(video_id, str) or VIDEO_ID.fullmatch(video_id) is None: + raise ValueError(f"{label} has an invalid video ID") + title = entry.get("title") + if not isinstance(title, str) or not title.strip(): + raise ValueError(f"{label} has no title") + duration = entry.get("duration_seconds") + if duration is not None and (type(duration) is not int or duration < 0): + raise ValueError(f"{label} has an invalid duration") + published_at = entry.get("published_at", "") + if published_at != "": + published_at = timestamp(published_at, f"{label} published_at") + reviewed = { + "video_id": video_id, + "title": title, + "published_at": published_at, + "duration_seconds": duration, + **PUBLIC_STATE, + "description": "", + "published_date": published_at[:10], + "is_scheduled": False, + "url": f"https://www.youtube.com/watch?v={video_id}", + } + if "playlist_titles" in entry: + playlist_titles = entry["playlist_titles"] + if not isinstance(playlist_titles, list) or any( + not isinstance(title, str) or not title.strip() for title in playlist_titles + ): + raise ValueError(f"{label} playlist_titles must be a list of nonempty strings") + reviewed["playlist_titles"] = playlist_titles.copy() + return reviewed + + +def review_worklist(worklist: object, indexed: set[str]) -> dict: + if not isinstance(worklist, dict) or worklist.get("channel_id") != CHANNEL_ID: + raise ValueError("worklist must identify the Institute YouTube channel") + generated_at = timestamp(worklist.get("generated_at"), "worklist generated_at") + rows = worklist.get("new_videos") + if not isinstance(rows, list): + raise ValueError("worklist new_videos must be a list") + count = worklist.get("new_video_count") + channel_count = worklist.get("channel_video_count") + if type(count) is not int or count != len(rows): + raise ValueError("worklist new_video_count differs from the row count") + if type(channel_count) is not int or channel_count < count: + raise ValueError("worklist channel_video_count must cover its rows") + + seen: set[str] = set() + new_videos: list[dict] = [] + for position, row in enumerate(rows, 1): + video = public_entry(row, position) + video_id = video["video_id"] + if video_id in seen: + raise ValueError(f"worklist row {position} duplicates a video ID") + seen.add(video_id) + if video_id not in indexed: + new_videos.append(video) + return { + "channel_id": CHANNEL_ID, + "generated_at": generated_at, + "index_video_count": len(indexed), + "channel_video_count": channel_count, + "new_video_count": len(new_videos), + "new_videos": new_videos, + } + + +def write_atomically(output: Path, report: dict) -> None: + if output.is_symlink(): + raise ValueError("output must be a regular file path") + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=output.parent, delete=False + ) as stream: + temporary = Path(stream.name) + json.dump(report, stream, indent=2, ensure_ascii=False) + stream.write("\n") + os.replace(temporary, output) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) + parser.add_argument("--journal", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--worklist", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + + index_path = args.journal / "INDEX.json" + try: + if args.output.resolve() in {args.worklist.resolve(), index_path.resolve()}: + raise ValueError("output must be separate from the worklist and INDEX.json") + indexed = index_video_ids(json.loads(index_path.read_text(encoding="utf-8"))) + report = review_worklist(json.loads(args.worklist.read_text(encoding="utf-8")), indexed) + write_atomically(args.output, report) + except (OSError, ValueError) as exc: + print(f"Cannot review YouTube ingest: {exc}", file=sys.stderr) + return 1 + print(f"Reviewed {report['new_video_count']} new public videos against {len(indexed)} indexed IDs") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())