diff --git a/.claude/skills b/.claude/skills new file mode 120000 index 0000000..42c5394 --- /dev/null +++ b/.claude/skills @@ -0,0 +1 @@ +../skills \ No newline at end of file diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 0000000..c5d9593 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,2 @@ +[codespell] +skip=./requirements,./jupyter,./node_modules,./venv diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..7ccffc3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,7 @@ +[flake8] +ignore = D205,D413,D400,D401 +max-line-length =120 +max-complexity = 10 +exclude = "venv" +extend-exclude = "*__init__.py,*__version__.py,venv" +select = "C101" diff --git a/.github/actions/tests/pre-commit/action.yml b/.github/actions/tests/pre-commit/action.yml deleted file mode 100644 index a5db706..0000000 --- a/.github/actions/tests/pre-commit/action.yml +++ /dev/null @@ -1,77 +0,0 @@ ---- -#------------------------------------------------------------------------------ -# Run pre-commit -#------------------------------------------------------------------------------ -name: Test pre-commit -branding: - icon: "git-pull-request" - color: "orange" -inputs: - python-version: - description: "The version of Python to use, such as 3.8.0" - required: true - type: string - -runs: - using: "composite" - steps: - - name: Checkout code - id: checkout - uses: actions/checkout@v4 - - - name: Check for pre-commit in requirements - shell: bash - run: | - if ! grep -q "pre-commit" ./requirements/dev.txt; then - echo "pre-commit not found in requirements/dev.txt" >&2 - exit 1 - fi - - - name: Check for black in requirements - shell: bash - run: | - if ! grep -q "black" ./requirements/dev.txt; then - echo "black not found in requirements/dev.txt" >&2 - exit 1 - fi - - - name: Check for flake8 in requirements - shell: bash - run: | - if ! grep -q "flake8" ./requirements/dev.txt; then - echo "flake8 not found in requirements/dev.txt" >&2 - exit 1 - fi - - - name: Check for flake8-coding in requirements - shell: bash - run: | - if ! grep -q "flake8-coding" ./requirements/dev.txt; then - echo "flake8-coding not found in requirements/dev.txt" >&2 - exit 1 - fi - - - name: Cache Python dependencies - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements/dev.txt') }} - restore-keys: | - ${{ runner.os }}-pip - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - pip install -r ./requirements/dev.txt - - # see: https://pre-commit.ci/lite.html - - name: pre-commit ci - id: pre-commit-ci - if: always() - uses: pre-commit-ci/lite-action@v1 diff --git a/.github/actions/tests/python/action.yml b/.github/actions/tests/python/action.yml deleted file mode 100644 index d440ac8..0000000 --- a/.github/actions/tests/python/action.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- -#------------------------------------------------------------------------------ -# Run Python unit tests -#------------------------------------------------------------------------------ -name: Test Python -branding: - icon: "git-pull-request" - color: "orange" -inputs: - python-version: - description: "The version of Python to use, such as 3.13" - required: true - type: string - kaggle-json: - description: "Kaggle JSON credentials for accessing datasets" - required: true - type: string - -runs: - using: "composite" - steps: - - name: Checkout code - id: checkout - uses: actions/checkout@v4 - - - name: Add config.json - shell: bash - run: | - echo '{"subscription_id": "ADD-ME-PLASE", "resource_group": "ubc-cdl10", "workspace_name": "ubc-cdl10"}' > config.json - - #-------------------------------------------------------------------------------- - # Add kaggle.json for accessing Kaggle datasets - # example: {"username":"lawrencemcdaniel","key":"2f00 --32 characters -- 26a6"} - #-------------------------------------------------------------------------------- - - name: Add kaggle.json - shell: bash - run: | - mkdir -p ~/.kaggle - echo '${{ inputs.kaggle-json }}' > ~/.kaggle/kaggle.json - chmod 600 ~/.kaggle/kaggle.json - - - name: Check for azureml-core in requirements - shell: bash - run: | - if ! grep -q "azureml-core" requirements/base.txt; then - echo "azureml-core not found in requirements/base.txt" >&2 - exit 1 - fi - - - name: Cache Python dependencies - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements/prod.txt') }} - restore-keys: | - ${{ runner.os }}-pip - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python-version }} - - - name: locate site-packages path - shell: bash - run: | - echo "SITE_PACKAGES_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')" >> $GITHUB_ENV - - - name: Install pip - shell: bash - run: | - python -m pip install --upgrade pip - - - name: Install dependencies - shell: bash - run: | - pip install -r requirements/prod.txt - env: - SITE_PACKAGES_PATH: ${{ env.SITE_PACKAGES_PATH }} - - - name: Run Python unit tests - shell: bash - env: - GITHUB_ACTIONS: "true" - DEBUG_MODE: "true" - run: | - make test-ci diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0fdb524..0b71963 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,7 @@ version: 2 updates: - package-ecosystem: "pip" - directory: "requirements/in/" + directory: "requirements/" schedule: interval: "monthly" labels: diff --git a/.github/workflows/auto-assign.yml b/.github/workflows/auto-assign.yml index 50d9470..f8fa2b3 100644 --- a/.github/workflows/auto-assign.yml +++ b/.github/workflows/auto-assign.yml @@ -1,18 +1,28 @@ +--- +#------------------------------------------------------------------------------ +# Auto Assign: assigns every new issue and pull request to the maintainer. +#------------------------------------------------------------------------------ name: Auto Assign + on: issues: types: [opened] pull_request: types: [opened] + +permissions: + contents: read + jobs: run: runs-on: ubuntu-latest + timeout-minutes: 5 permissions: issues: write pull-requests: write steps: - name: "Auto-assign issue" - uses: pozil/auto-assign-issue@v4 + uses: pozil/auto-assign-issue@d6e511896e5f1d2408e74db74416647d8ff80c8c # v4.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} assignees: lpm0073 diff --git a/.github/workflows/precommitVersionBumps.yml b/.github/workflows/precommitVersionBumps.yml index 25fef89..f377fff 100644 --- a/.github/workflows/precommitVersionBumps.yml +++ b/.github/workflows/precommitVersionBumps.yml @@ -5,10 +5,12 @@ # # This workflow runs on a cron schedule and checks for updates to the # .pre-commit-config.yaml file. If updates are found, the workflow -# commits the changes to the next branch and pushes the changes to GitHub. +# commits the changes to the alpha branch and pushes the changes to GitHub. # # This is a workaround for the fact that the pre-commit autoupdate command # is not supported by Dependabot. +# +# A scheduled run uses this file as it is on main, the default branch. #------------------------------------------------------------------------------ name: pre-commit Version Bumps @@ -17,67 +19,33 @@ on: - cron: "0 0 * * 3" workflow_dispatch: +# The push to alpha uses the PAT secret, not GITHUB_TOKEN. +permissions: + contents: read + jobs: evaluate_precommit_config: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: alpha persist-credentials: false - - name: Checkout alpha branch - run: | - git fetch - git checkout alpha - git pull origin alpha - - - name: Cache NPM dependencies - uses: actions/cache@v6 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node - - - name: Cache Python dependencies - uses: actions/cache@v6 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements/**') }} - restore-keys: | - ${{ runner.os }}-pip - - name: Set up Python - uses: actions/setup-python@v7 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.13" - - name: locate site-packages path - shell: bash - run: | - echo "SITE_PACKAGES_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')" >> $GITHUB_ENV - - - name: Install pip + # pre-commit autoupdate only reads the hooks' repositories, so it needs + # pre-commit itself, at the version requirements/local.txt pins. + - name: Install pre-commit shell: bash run: | python -m pip install --upgrade pip - - - name: Install dependencies - shell: bash - run: | - pip install -r ./requirements/local.txt - pip install -r ./requirements/prod.txt - env: - SITE_PACKAGES_PATH: ${{ env.SITE_PACKAGES_PATH }} - - - name: Setup Node.js environment - uses: actions/setup-node@v7 - with: - node-version: "20.9.0" - - - name: Install npm dev dependencies - run: npm install + pip install "$(grep -E '^pre-commit==' requirements/local.txt)" - name: Update .pre-commit-config.yaml run: | @@ -87,9 +55,9 @@ jobs: id: check_changes run: | if [[ -n "$(git status --porcelain .pre-commit-config.yaml)" ]]; then - echo "::set-output name=changes::true" + echo "changes=true" >> "$GITHUB_OUTPUT" else - echo "::set-output name=changes::false" + echo "changes=false" >> "$GITHUB_OUTPUT" fi - name: Commit and push changes @@ -99,5 +67,7 @@ jobs: git config --local user.email "action@github.com" git config --local user.name "GitHub Action" git add .pre-commit-config.yaml - git commit -m "chore: [gh] version bumps in .pre-commit-config.yaml [skip ci]" + # No CI skip tag: this commit can become the head of the pull + # request from alpha to main, whose checks would then never start. + git commit -m "chore: [gh] version bumps in .pre-commit-config.yaml" git push https://${{ secrets.PAT }}@github.com/${{ github.repository }}.git HEAD:alpha diff --git a/.github/workflows/pullRequestController.yml b/.github/workflows/pullRequestController.yml deleted file mode 100644 index d05be15..0000000 --- a/.github/workflows/pullRequestController.yml +++ /dev/null @@ -1,115 +0,0 @@ ---- -#------------------------------------------------------------------------------ -# Pull Request Workflow Controller. -# -# Triggers: -# - Called automatically on relevant actions performed on pull requests. -# - Can also be run manually by clicking the "Run workflow" button. -# -# Actions: -# - Use semantic release rules to determine if a new release will be published. -# - run Python tests, but only if Python-related files have changed. -# - run Terraform tests, but only if Terraform-related files have changed. -# - run ReactJS tests, but only if ReactJS-related files have changed. -# - run pre-commit hooks to ensure code is formatted correctly. -# -# To-Do: -# If a new release is to be published then we want to consider running QA tests -# to ensure formatting and documentation is correct. -#------------------------------------------------------------------------------ -name: Pull Request Controller - -on: - workflow_dispatch: - - # GitHub Copilot: The `pull_request` and `pull_request_target` are two different - # event types in GitHub Actions that trigger workflows when activity related - # to pull requests occurs. - # - `pull_request`: This event triggers a workflow run whenever a pull - # request is opened, synchronized, or closed. The workflow runs in the context of the - # pull request, meaning it has access to the code and environment variables of the head - # branch of the pull request. This is safe for pull requests within the same repository, - # but for pull requests from a fork, this could potentially expose sensitive information. - # - # - `pull_request_target`: This event is similar to `pull_request`, but it runs in the context - # of the base of the pull request, rather than the head. This means it has access to the code - # and environment variables of the base branch, not the head branch. This is safer for - # pull requests from forks, as it prevents the fork from accessing sensitive information - # in the base repository. However, it means the workflow does not have access to the code - # in the pull request by default. If you need to access the code in the pull request, - # you can use the `actions/checkout` action with the `ref` input - # set to `github.event.pull_request.head.ref`. - # - # In general, use `pull_request` for workflows that need to access the code in the pull request, - # and `pull_request_target` for workflows that need to be safe for pull requests from forks. - pull_request_target: - types: [opened, synchronize] - paths: - - "**.py" - - "**./requirements/**" - -env: - python-version: "3.13" - -jobs: - check_for_pending_release: - name: test-semantic-release - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Semantic Release - uses: cycjimmy/semantic-release-action@v6 - id: semantic - with: - dry_run: true - branches: | - [ - '+([0-9])?(.{+([0-9]),x}).x', - 'main', - 'next', - 'next-major', - { - name: 'beta', - prerelease: true - }, - { - name: 'alpha', - prerelease: true - } - ] - extra_plugins: | - @semantic-release/git - @semantic-release/changelog - env: - GITHUB_TOKEN: ${{ secrets.PAT }} - - - name: Test Outputs - if: steps.semantic.outputs.new_release_published == 'true' - run: | - echo ${{ steps.semantic.outputs.new_release_version }} - echo ${{ steps.semantic.outputs.new_release_major_version }} - echo ${{ steps.semantic.outputs.new_release_minor_version }} - echo ${{ steps.semantic.outputs.new_release_patch_version }} - - python_tests: - needs: check_for_pending_release - runs-on: ubuntu-latest - - steps: - - name: Checkout code - id: checkout - uses: actions/checkout@v7 - - - name: Check for changed files - id: file_changes - run: | - echo "::set-output name=files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '\.py$' || true)" - echo "::set-output name=requirements_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep 'requirements$' || true)" - - - name: Run Python tests - if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' - uses: ./.github/actions/tests/python - with: - python-version: "${{ env.python-version}}" diff --git a/.github/workflows/pushMain.yml b/.github/workflows/pushMain.yml deleted file mode 100644 index 92037cc..0000000 --- a/.github/workflows/pushMain.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -#--------------------------------------------------------- -# - Create a semantical release -# - Merge main into next, alpha, beta, and next-major -#--------------------------------------------------------- -name: Push to main - -on: - workflow_dispatch: - push: - branches: - - main -jobs: - semantic-release: - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.PAT }} - - steps: - - uses: actions/checkout@v7 - id: checkout - with: - persist-credentials: false - - - name: Semantic Release - uses: cycjimmy/semantic-release-action@v6 - id: semantic - with: - branches: | - [ - '+([0-9])?(.{+([0-9]),x}).x', - 'main', - 'next', - 'next-major', - { - name: 'beta', - prerelease: true - }, - { - name: 'alpha', - prerelease: true - } - ] - extra_plugins: | - @semantic-release/git - @semantic-release/changelog - env: - GIT_COMMITTER_NAME: github-actions[bot] - GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com - GIT_AUTHOR_NAME: github-actions[bot] - GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com - - - name: Publish To GitHub Package Registry - id: publish - if: steps.semantic.outputs.new_release_published == 'true' - run: echo "new release was published" - shell: bash - - - name: Push updates to branch for major version - id: push_major - if: steps.semantic.outputs.new_release_published == 'true' - run: "git push https://x-access-token:${{ env.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git HEAD:refs/heads/v${{steps.semantic.outputs.new_release_major_version}}" - shell: bash diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 120f7b5..56bf5ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,41 +1,81 @@ +--- +#------------------------------------------------------------------------------ +# Release: semantic-release, on every push to main. +# +# Reads the conventional commits since the last tag and, when they call for a +# release (see release.config.js: docs, test, style and refactor commits +# don't), publishes it: the version, CHANGELOG.md, the tag and the GitHub +# release. It then pushes the release to its major version's branch (v1, v2, +# ...), so that anyone can follow a major version. +# +# release.config.js is the one place that lists the release branches and +# plugins. +# +# The PAT secret, rather than GITHUB_TOKEN, lets semantic-release push its +# commit and tag past main's branch protection. Its commit, +# chore(release): , starts this workflow again; the job skips it. +#------------------------------------------------------------------------------ name: Release + on: push: branches: - main - pull_request_target: - types: [closed] - branches: [main] + workflow_dispatch: -jobs: - check_merged: - runs-on: ubuntu-latest - outputs: - merged: ${{ steps.check.outputs.merged }} - steps: - - id: check - run: | - echo "::set-output name=merged::false" - if [[ "${{ github.event.pull_request.merged }}" == "true" ]]; then - echo "::set-output name=merged::true" - fi +# One release at a time. A merge while a release runs waits for it, and the +# next run releases everything since the last tag. +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false +# GITHUB_TOKEN is read-only. Every write uses the PAT secret instead. +permissions: + contents: read + +jobs: release: - needs: check_merged - if: ${{ needs.check_merged.outputs.merged == 'true' }} + if: ${{ !startsWith(github.event.head_commit.message, 'chore(release):') }} runs-on: ubuntu-latest + timeout-minutes: 15 + env: + GITHUB_TOKEN: ${{ secrets.PAT }} steps: - name: Checkout code - id: checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - token: ${{ secrets.PAT }} - - uses: actions/setup-node@v7 + # An LTS release, never "latest": that can be a pre-release Node that + # semantic-release refuses to run on. + - name: Setup Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: - node-version: "latest" - - run: npm install - - run: npx semantic-release + node-version: "lts/*" + + - name: Install npm dependencies + run: npm install + + - name: Semantic Release + uses: cycjimmy/semantic-release-action@ad51787758d9654d5a75150a89a6fa9b3ccef95e # v8.1.4 + id: semantic env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GIT_COMMITTER_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + + - name: Summary + shell: bash + run: | + if [[ "${{ steps.semantic.outputs.new_release_published }}" == "true" ]]; then + echo "Released ${{ steps.semantic.outputs.new_release_git_tag }}." >> "$GITHUB_STEP_SUMMARY" + else + echo "No release: the commits since the last tag don't call for one." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Push the release to its major version's branch + if: steps.semantic.outputs.new_release_published == 'true' + shell: bash + run: | + git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "HEAD:refs/heads/v${{ steps.semantic.outputs.new_release_major_version }}" diff --git a/.github/workflows/semanticVersionBump.yml b/.github/workflows/semanticVersionBump.yml deleted file mode 100644 index 0dd09dd..0000000 --- a/.github/workflows/semanticVersionBump.yml +++ /dev/null @@ -1,123 +0,0 @@ ---- -#------------------------------------------------------------------------------ -# Lawrence McDaniel - https://lawrencemcdaniel.com -# Version Bump Workflow for Python package openai_utils -# -# Calculate the version of the 'next' branch based on semantic-release rules. -# Compares the existing value of __version__.py to the calculated value. -# If they are different, it will update __version__.py and push the changes -# to the main branch. -#------------------------------------------------------------------------------ -name: Semantic Version Bump (next) - -on: - workflow_dispatch: - push: - branches: - - alpha - - beta - -jobs: - bump-version-next: - runs-on: ubuntu-latest - env: - VERSION_FILE: __version__.py - PACKAGE_PATH: ${{ github.workspace }}/azure_ai/ - - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Cache NPM dependencies - uses: actions/cache@v6 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node - - - name: Set up Python 3.13 - uses: actions/setup-python@v7 - with: - python-version: "3.13" - - - name: Setup Node.js environment - uses: actions/setup-node@v7 - with: - node-version: "20.9.0" - - - name: Install npm dev dependencies - run: npm install - - - name: Get current version - # step 1 - # the current version persisted to __version__.py - id: current_version - run: | - cd ${{ env.PACKAGE_PATH }} - echo "CURRENT_VERSION=$(python -c 'from azure_ai.__version__ import __version__; print(__version__)')" >> $GITHUB_ENV - env: - GITHUB_TOKEN: ${{ secrets.PAT }} - - - name: null step - id: null_step1 - run: echo "i ensure that CURRENT_VERSION is set." - - - name: Get next version - # step 2 - # calculate the next version based on semantic-release rules - # this will return a null string is there in fact is no version bump. - # so set NEXT_VERSION to CURRENT_VERSION if there is no version bump. - id: next_version - run: | - NEXT_VERSION=$(npx semantic-release --dry-run --no-ci | awk '/The next release version is/{print $NF}') - echo "NEXT_VERSION=${NEXT_VERSION:-${{ env.CURRENT_VERSION }}}" >> $GITHUB_ENV - env: - GITHUB_TOKEN: ${{ secrets.PAT }} - CURRENT_VERSION: ${{ env.CURRENT_VERSION }} - - - name: null step - id: null_step2 - run: echo "i ensure that NEXT_VERSION is set." - - - name: Check versions - # step 3 - # compare the current version to the next version. - # if they are different, set VERSION_CHANGED to true - id: check_versions - run: | - if [ "$CURRENT_VERSION" != "$NEXT_VERSION" ]; then - echo "VERSION_CHANGED=true" >> $GITHUB_ENV - else - echo "VERSION_CHANGED=false" >> $GITHUB_ENV - fi - env: - CURRENT_VERSION: ${{ env.CURRENT_VERSION }} - NEXT_VERSION: ${{ env.NEXT_VERSION }} - - - name: another null step - id: null_step3 - run: echo "i ensure that CURRENT_VERSION, NEXT_VERSION and VERSION_CHANGED are set." - - - name: Update __version__.py - # step 4 - # if VERSION_CHANGED is true, update __version__.py and push the changes to the - # branch that triggered this workflow. - if: env.VERSION_CHANGED == 'true' - id: update_version - run: | - echo "# -*- coding: utf-8 -*-" > ${{ env.VERSION_FILE }} - echo "# DO NOT EDIT." >> ${{ env.VERSION_FILE }} - echo "# Managed via automated CI/CD in .github/workflows/semanticVersionBump.yml." >> ${{ env.VERSION_FILE }} - echo "__version__ = \"${{ env.NEXT_VERSION }}\"" >> ${{ env.VERSION_FILE }} - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add ${{ env.VERSION_FILE }} - git commit -m "chore: [gh] Update __version__.py to ${{ env.NEXT_VERSION }} [skip ci]" - git push https://${{ secrets.PAT }}@github.com/${{ github.repository }}.git HEAD:${{ github.ref }} - env: - VERSION_FILE: ${{ env.PACKAGE_PATH }}${{ env.VERSION_FILE }} - GITHUB_TOKEN: ${{ secrets.PAT }} - NEXT_VERSION: ${{ env.NEXT_VERSION }} - VERSION_CHANGED: ${{ env.VERSION_CHANGED }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be27aab..cee3c09 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,36 +1,62 @@ --- #------------------------------------------------------------------------------ # Lawrence McDaniel - https://lawrencemcdaniel.com -# Run unit tests for Python code in the repository. +# Test: every pre-commit hook on every file. # +# This repository has no Python package and no unit tests; its notebooks, +# docs, requirements and configuration are checked by the hooks in +# .pre-commit-config.yaml (JSON syntax, notebooks included; YAML and TOML +# syntax; Prettier; codespell; private keys and AWS credentials; whitespace). +# It runs them on Python 3.13 with the pre-commit that requirements/local.txt +# pins. The hooks bring their own environments. +# +# Pull requests from forks run here too: pull_request (unlike +# pull_request_target) runs the pull request's own code without the +# repository's secrets, which this job doesn't need. #------------------------------------------------------------------------------ name: Test on: - workflow_dispatch: pull_request: - paths: - - "**.py" push: branches: - main - next - paths: - - "**.py" + workflow_dispatch: + +# A new push to a pull request cancels the run in progress. +concurrency: + group: test-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read env: python-version: "3.13" jobs: - python-unit-tests: - runs-on: ubuntu-latest + pre-commit: + runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - name: Checkout code - id: checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - name: Run Python tests - uses: ./.github/actions/tests/python + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: "${{ env.python-version}}" - kaggle-json: "${{ secrets.KAGGLE_JSON }}" + python-version: ${{ env.python-version }} + + - name: Install pre-commit + shell: bash + run: | + python -m pip install --upgrade pip + pip install "$(grep -E '^pre-commit==' requirements/local.txt)" + + - name: Run pre-commit hooks + shell: bash + run: | + pre-commit run --all-files --show-diff-on-failure diff --git a/.gitignore b/.gitignore index 1262272..166e5ac 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,6 @@ venv3.12 # ReactJS node_modules package-lock.json + +# Claude Code: personal settings stay local; .claude/skills is committed +.claude/settings.local.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7e976eb..f7b17e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,63 +1,60 @@ default_language_version: # default language version for each language python: python3.13 +# Install the commit-msg hook too, so that commitlint checks each commit message. +# semantic-release versions ai-algorithms from the conventional commit messages. +default_install_hook_types: [pre-commit, commit-msg] repos: + - repo: https://github.com/DanielNoord/pydocstringformatter + rev: v1.0.0 + hooks: + - id: pydocstringformatter - repo: https://github.com/asottile/pyupgrade - rev: v3.20.0 + rev: v3.22.0 hooks: - id: pyupgrade - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.3 hooks: - id: codespell args: ["--ignore-words=codespell.txt"] exclude: ^(requirements|jupyter)/ - - repo: local - hooks: - - id: eslint - name: eslint - entry: bash -c 'cd client && npx eslint' - language: node - types: [javascript] - files: ^client/ - repo: https://github.com/pre-commit/mirrors-prettier rev: v4.0.0-alpha.8 hooks: - id: prettier + args: [--no-error-on-unmatched-pattern] - repo: https://github.com/psf/black - rev: 25.1.0 + rev: 26.10.1 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 7.3.0 + rev: 7.5.0 hooks: - id: flake8 - repo: https://github.com/PyCQA/isort - rev: 6.0.1 + rev: 10.0.0b2 hooks: - id: isort args: ["--settings-path=pyproject.toml"] - # - repo: local - # hooks: - # - id: pylint - # name: pylint - # entry: ./scripts/run_pylint.sh - # language: script - # types: [python] + - repo: https://github.com/PyCQA/autoflake + rev: v2.4.0 + hooks: + - id: autoflake + args: [--remove-all-unused-imports] - repo: https://github.com/PyCQA/bandit - rev: 1.8.5 + rev: 1.9.4 hooks: - id: bandit args: ["-ll"] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: # See https://pre-commit.com/hooks.html for more hooks #- id: check-added-large-files - id: fix-byte-order-marker - id: check-case-conflict - id: check-json - exclude: ".*/reactapp/.*" - id: check-merge-conflict - id: check-symlinks - id: check-toml @@ -70,15 +67,13 @@ repos: - id: end-of-file-fixer - id: forbid-new-submodules - id: trailing-whitespace - - id: check-case-conflict - - id: check-merge-conflict - id: debug-statements - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook - rev: v9.22.0 + rev: v9.27.0 hooks: - id: commitlint stages: [commit-msg] - additional_dependencies: ["@commitlint/config-angular"] + additional_dependencies: ["@commitlint/config-conventional", "@commitlint/config-angular"] ci: # for more information, see https://pre-commit.ci autofix_commit_msg: | @@ -87,5 +82,5 @@ ci: autoupdate_branch: "" autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate" autoupdate_schedule: weekly - skip: [shellcheck, markdown-link-check, commitlint] + skip: [commitlint] submodules: false diff --git a/.prettierrc.json b/.prettierrc.json index 75fa134..9681ca1 100644 --- a/.prettierrc.json +++ b/.prettierrc.json @@ -1,3 +1,10 @@ { - "tabWidth": 2 + "printWidth": 120, + "tabWidth": 2, + "semi": true, + "trailingComma": "all", + "bracketSpacing": true, + "jsxSingleQuote": false, + "arrowParens": "always", + "endOfLine": "lf" } diff --git a/CHANGELOG.md b/CHANGELOG.md index c66404a..05d6f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,4 +3,3 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). - diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..18f83de --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,77 @@ +# CLAUDE.md + +Guidance for Claude Code (and other coding agents) working in this repository. + +ai-algorithms is a teaching repository from Full Stack With Lawrence: Jupyter +notebooks that demonstrate basic AI algorithms in Python (anomaly detection, +regression, k-means clustering), used in university courses and YouTube +tutorials. There is no Python package and no unit test suite; the notebooks +are the product. Clarity matters as much as correctness. + +Detailed know-how lives in the Agent Skills under `skills/` (symlinked to +`.claude/skills`). **Load `ai-algorithms-development` first for any change.** +Its table says which narrower `ai-algorithms-*` skill covers each kind of work. +This file holds only what applies to nearly every task. + +## Hard rules + +1. **Never run git write commands.** "Write a git commit" means print the + command as text. Never run `git commit`, `git add`, `git reset`, `git push` + or anything else that changes the index or history. Format and authorship + are in `ai-algorithms-git-commits`. +2. **Don't run notebook cells that reach Kaggle or Azure** without the user's + OK, and never `%pip install` cells. See `ai-algorithms-notebooks`. +3. **Don't trigger workflows or releases** (`gh workflow run`, `gh run rerun`, + `make release`). Reading runs with `gh run view --log-failed` is fine. See + `ai-algorithms-ci`. +4. **Python 3.13 everywhere**, kept in step across the Makefile, pre-commit + config, `pyproject.toml`, workflows and notebook kernels. Regenerate + `requirements/*.txt` from `requirements/in/*.in` with + `make python-requirements` rather than editing pins by hand. See + `ai-algorithms-python-environment`. +5. **The pre-commit hooks are the lint standard.** See + `ai-algorithms-code-quality`. +6. **This repository is public.** No Kaggle keys, Azure ids, personal paths or + personal data in notebooks (outputs included), docs or skills. `.env`, + `config.json`, `kaggle.json` and `jupyter/data/` are git-ignored and stay + that way. +7. **Tooling follows azureml-example.** This repository and + `FullStackWithLawrence/azureml-example` share a template; when their + tooling differs for no reason, azureml-example's is the reference. + +## Repository map + +| Path | Contents | +| -------------------- | --------------------------------------------------------------------------- | +| `jupyter/` | The course notebooks; `jupyter/data/` holds Kaggle downloads (ignored) | +| `docs/` | Markdown guides (several are template leftovers; see the development skill) | +| `requirements/` | `in/*.in` sources compiled into `base.txt` and `local.txt` | +| `.github/workflows/` | Lint on pull requests, semantic-release on `main`, pre-commit version bumps | +| `skills/` | Agent Skills for developing this repository | + +## Common commands + +```bash +make init # build ./venv with Python 3.13 and requirements/local.txt (deletes venv/ first) +source venv/bin/activate +make python-requirements # recompile requirements/*.txt +make lint # every pre-commit hook on every file + +# Lint the changed files (zsh doesn't word-split $FILES, so use xargs) +git diff --name-only | xargs pre-commit run --files +``` + +- `make init` deletes and rebuilds `venv/`, which is also the notebooks' + kernel. Ask before running it. +- `make pre-commit` runs `pre-commit autoupdate`, which edits + `.pre-commit-config.yaml`. +- `make release` makes an empty commit and pushes it. Never run it. + +## Definition of done + +- The change follows an existing pattern. +- Notebooks you changed still parse, and you said which cells you ran. +- Pre-commit hooks pass on the changed files. +- Nothing was pushed, released, or run against Kaggle or Azure without the + user's say-so. +- Your summary says what changed, what you verified, and what you skipped. diff --git a/Makefile b/Makefile index 0812df1..696672a 100644 --- a/Makefile +++ b/Makefile @@ -2,94 +2,149 @@ SHELL := /bin/bash REPO_NAME := ai-algorithms ifeq ($(OS),Windows_NT) - PYTHON = python.exe - ACTIVATE_VENV = venv\Scripts\activate + PYTHON := python.exe + ACTIVATE_VENV := venv/Scripts/activate else - PYTHON = python3.13 - ACTIVATE_VENV = source ./venv/bin/activate + PYTHON := python3.13 + ACTIVATE_VENV := source venv/bin/activate endif -PIP = $(PYTHON) -m pip +PIP := $(PYTHON) -m pip -ifneq ("$(wildcard .env)","") - include .env -else - $(shell echo -e "ENVIRONMENT=dev" >> .env) +# scaffold .env from .env.example on first use +ifeq ("$(wildcard .env)","") + $(shell cp .env.example .env) endif +include .env -.PHONY: analyze pre-commit init lint clean test build release all python-init help +.PHONY: all init clean lint analyze pre-commit pre-commit-init pre-commit-run release \ + check-python python-init python-clean python-requirements \ + help # Default target executed when no arguments are given to make. all: help -# ------------------------------------------------------------------------- -# Install and run pre-commit hooks -# ------------------------------------------------------------------------- -pre-commit: - pre-commit install - pre-commit autoupdate - pre-commit run --all-files - -# --------------------------------------------------------- -# create a local python virtual environment. Includes linters -# and other tools for local development. -# --------------------------------------------------------- -python-init: - make clean - $(PYTHON) -m venv venv && \ - $(ACTIVATE_VENV) && \ - $(PIP) install --upgrade pip && \ - $(PIP) install -r requirements/local.txt && \ - deactivate - -###################### -# CORE COMMANDS -###################### +# initialize local development environment. init: - make clean && \ - make python-init && \ - npm install && \ - $(ACTIVATE_VENV) && \ - $(PIP) install -U 'pip<25.3' && \ - $(PIP) install -r requirements/local.txt && \ - pre-commit install && \ - deactivate + @echo "===============================================================================" + @echo "Initializing local development environment. This will verify and set up your" + @echo "Python virtual environment, install all 3rd-party package requirements," + @echo "and install the pre-commit hooks. This may take a few minutes..." + @echo "===============================================================================" + make check-python # verify Python 3.13 is installed + make python-init # create/replace Python virtual environment and install dependencies + make pre-commit-init # install and configure pre-commit + @echo "" + @echo "===============================================================================" + @echo "Initialization complete!" + @echo "" + @echo "Run 'source venv/bin/activate' to activate the Python virtual environment." + @echo " 'make help' to see all available commands." + @echo "===============================================================================" -test: - $(ACTIVATE_VENV) && python -m unittest discover -s azure_ai/; \ +clean: + make python-clean -test-ci: - python -m unittest discover -s azure_ai/; +# --------------------------------------------------------- +# Code management +# --------------------------------------------------------- lint: - isort . && \ - pre-commit run --all-files && \ - black . && \ - flake8 ./azure_ai/ && \ - pylint ./azure_ai/**/*.py - -clean: - rm -rf venv node_modules azure_ai/__pycache__ package-lock.json + make pre-commit-run analyze: + @echo "===============================================================================" + @echo "Generating code analysis report using cloc ..." + @echo "===============================================================================" cloc . --exclude-ext=svg,json,zip,csv,yml,yaml --vcs=git +pre-commit: + make pre-commit-init + make pre-commit-run + +pre-commit-init: + @echo "===============================================================================" + @echo "Installing and configuring pre-commit ..." + @echo "===============================================================================" + pre-commit install + pre-commit autoupdate + +pre-commit-run: + @echo "===============================================================================" + @echo "Running pre-commit hooks on all files ..." + @echo "===============================================================================" + pre-commit run --all-files + release: + @echo "===============================================================================" + @echo "Forcing a new semantic release on GitHub by creating an empty commit and pushing to the repository ..." + @echo "===============================================================================" git commit -m "fix: force a new release" --allow-empty && git push +# --------------------------------------------------------- +# Python +# --------------------------------------------------------- +check-python: + @echo "" + @echo "===============================================================================" + @echo "Verifying that Python $(PYTHON) is installed ..." + @echo "===============================================================================" + @echo "" + @command -v $(PYTHON) >/dev/null 2>&1 || { \ + echo >&2 "This project requires $(PYTHON) but it's not installed. Aborting."; \ + echo >&2 "python --version output:"; \ + python --version 2>&1; \ + exit 1; \ +} + +# venv/ is also the notebooks' Jupyter kernel, "venv (3.13.x)". +python-init: + @echo "===============================================================================" + @echo "Initializing Python virtual environment and installing dependencies. This may take a few minutes..." + @echo "===============================================================================" + make check-python + make python-clean && \ + npm install && \ + $(PYTHON) -m venv venv && \ + $(ACTIVATE_VENV) && \ + $(PIP) install --upgrade pip && \ + $(PIP) install setuptools wheel pip-tools && \ + $(PIP) install -r requirements/local.txt + +python-clean: + @echo "" + @echo "===============================================================================" + @echo "Cleaning Python virtual environment and __pycache__ directories ..." + @echo "===============================================================================" + @echo "" + rm -rf venv + find . -path ./node_modules -prune -o -name __pycache__ -type d -exec rm -rf {} + + +python-requirements: + @echo "===============================================================================" + @echo "Compiling and updating Python dependency files using pip-compile ..." + @echo "===============================================================================" + pip install --upgrade setuptools wheel "pip-tools>=7.6.1" + pip-compile --strip-extras requirements/in/base.in -o requirements/base.txt + pip-compile --strip-extras requirements/in/local.in -o requirements/local.txt - -###################### -# HELP -###################### +# --------------------------------------------------------- +# Help +# --------------------------------------------------------- help: @echo '====================================================================' - @echo 'analyze - generate code analysis report' - @echo 'release - force a new GitHub release' - @echo 'init - create a Python virtual environment and install prod dependencies' - @echo 'python-init - create a Python virtual environment with local dependencies' - @echo 'pre-commit - install and run pre-commit hooks' - @echo 'test - run Python unit tests' - @echo 'lint - run Python linting' - @echo 'clean - destroy the Python virtual environment' + @echo 'init - Initialize the local development environment' + @echo 'clean - Destroy the Python virtual environment and __pycache__ directories' + @echo '<************************** Code Management **************************>' + @echo 'lint - Run all pre-commit hooks on all files' + @echo 'analyze - Generate code analysis report using cloc' + @echo 'pre-commit - Install, configure and run pre-commit hooks' + @echo 'pre-commit-init - Install and configure pre-commit' + @echo 'pre-commit-run - Run all pre-commit hooks on all files' + @echo 'release - Force a new GitHub release' + @echo '<************************** Python **************************>' + @echo 'check-python - Verify Python 3.13 is installed' + @echo 'python-init - Create a Python virtual environment and install dependencies' + @echo 'python-clean - Destroy the Python virtual environment and remove __pycache__ directories' + @echo 'python-requirements - Compile requirements/in/*.in into requirements/*.txt' @echo '====================================================================' diff --git a/commitlint.config.js b/commitlint.config.js index b58fa47..00d0adc 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -5,10 +5,10 @@ const Configuration = { */ extends: ["@commitlint/config-conventional", "@commitlint/config-angular"], /* - * Resolve and load conventional-changelog-atom from node_modules. - * Referenced packages must be installed + * The parser of the conventional commits that semantic-release reads comes with + * @commitlint/config-conventional. (conventional-changelog-atom, which this file + * previously named, parses Atom's emoji commit headers, and is not installed.) */ - parserPreset: "conventional-changelog-atom", /* * Resolve and load @commitlint/format from node_modules. * Referenced package must be installed @@ -17,7 +17,18 @@ const Configuration = { /* * Any rules defined here will override rules from @commitlint/config-conventional */ - rules: {}, + rules: { + // Commit messages have one paragraph, or bullet list, per line (one git commit -m + // per paragraph), so their body and footer lines are longer than config-conventional's 100. + "body-max-line-length": [0, "always"], + "footer-max-line-length": [0, "always"], + // config-angular's types, plus chore, which this repository uses for housekeeping and releases. + "type-enum": [ + 2, + "always", + ["build", "chore", "ci", "docs", "feat", "fix", "perf", "refactor", "revert", "style", "test"], + ], + }, /* * Array of functions that return true if commitlint should ignore the given message. * Given array is merged with predefined functions, which consist of matchers like: @@ -38,8 +49,7 @@ const Configuration = { /* * Custom URL to show upon failure */ - helpUrl: - "https://github.com/conventional-changelog/commitlint/#what-is-commitlint", + helpUrl: "https://github.com/conventional-changelog/commitlint/#what-is-commitlint", /* * Custom prompt configs */ diff --git a/mypy.ini b/mypy.ini index b44c7e5..8c181ee 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,3 @@ [mypy] -plugins = mypy_django_plugin.main +# third-party packages without type information are treated as untyped. +ignore_missing_imports = True diff --git a/package.json b/package.json index 82c0559..40b4a4f 100644 --- a/package.json +++ b/package.json @@ -6,14 +6,9 @@ "@semantic-release/changelog": "^7.0.0", "@semantic-release/commit-analyzer": "^13.0.0", "@semantic-release/git": "^11.0.1", - "@semantic-release/github": "^12.0.1", + "@semantic-release/github": "^12.0.6", "@semantic-release/release-notes-generator": "^14.0.1", - "prettier": "^3.3.3", - "typescript": "^7.0.2" - }, - "dependencies": { - "gulp": "^5.0.0", - "gulp-cli": "^3.0.0", - "yarn": "^1.22.22" + "conventional-changelog-conventionalcommits": "^9.3.1", + "prettier": "^3.3.3" } } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ba6350f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[project] +name = "ai-algorithms" +version = "0.0.0" # semantic-release tracks the real version in git tags and CHANGELOG.md +requires-python = ">=3.13" +description = "Jupyter notebook examples of basic AI algorithms in Python" +authors = [{ name = "Lawrence McDaniel", email = "lpm0073@gmail.com" }] +license = { file = "LICENSE" } +keywords = ["AI", "Machine Learning", "Jupyter", "scikit-learn", "Python"] +readme = "README.md" +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Intended Audience :: Education", + "Intended Audience :: Developers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Natural Language :: English", +] + +[project.urls] +homepage = "https://github.com/FullStackWithLawrence/ai-algorithms" +documentation = "https://github.com/FullStackWithLawrence/ai-algorithms" +repository = "https://github.com/FullStackWithLawrence/ai-algorithms" +bugtracker = "https://github.com/FullStackWithLawrence/ai-algorithms/issues" +changelog = "https://github.com/FullStackWithLawrence/ai-algorithms/blob/main/CHANGELOG.md" + +# The isort hook in .pre-commit-config.yaml reads this file (--settings-path=pyproject.toml). +[tool.isort] +profile = "black" + +[tool.black] +line-length = 120 +target-version = ['py313'] +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.mypy_cache + | \.tox + | \.venv + | \.env + | venv + | node_modules + | build + | dist + | __pypackages__ + | \.pypi_cache + | \.pytest_cache + | \.eggs + | \.idea +)/ +''' diff --git a/release.config.js b/release.config.js index 74742f5..ed79161 100644 --- a/release.config.js +++ b/release.config.js @@ -1,27 +1,52 @@ module.exports = { - branches: ["main", "beta", "alpha"], + branches: ["main", "next", { name: "beta", prerelease: true }, { name: "alpha", prerelease: true }], dryRun: false, plugins: [ - "@semantic-release/commit-analyzer", - "@semantic-release/release-notes-generator", + [ + "@semantic-release/commit-analyzer", + { + preset: "conventionalcommits", + releaseRules: [ + { type: "docs", release: false }, + { type: "test", release: false }, + { type: "style", release: false }, + { type: "refactor", release: false }, + ], + parserOpts: { + noteKeywords: ["BREAKING CHANGE", "BREAKING CHANGES"], + }, + }, + ], + [ + "@semantic-release/release-notes-generator", + { + preset: "conventionalcommits", + presetConfig: { + types: [ + { type: "feat", section: "Features", hidden: false }, + { type: "fix", section: "Bug Fixes", hidden: false }, + { type: "refactor", section: "Refactoring", hidden: false }, + { type: "perf", section: "Performance", hidden: false }, + ], + }, + }, + ], [ "@semantic-release/changelog", { changelogFile: "CHANGELOG.md", + changelogTitle: `# Change Log\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/).\n\n`, }, ], "@semantic-release/github", [ "@semantic-release/git", { - assets: [ - "CHANGELOG.md", - "package.json", - "package-lock.json", - "requirements/**/*", - ], - message: - "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}", + assets: ["CHANGELOG.md", "package.json", "package-lock.json", "requirements/**/*"], + // No CI skip tag: this commit is often the head of the pull request + // from alpha to main, and GitHub would then start no Tests for it, so + // its required checks would never report. + message: "chore(release): ${nextRelease.version}\n\n${nextRelease.notes}", }, ], ], diff --git a/requirements/base.txt b/requirements/base.txt index 28790c7..7b65055 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,72 +2,100 @@ # This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --cert=None --client-cert=None --index-url=None --output-file=requirements/base.txt --pip-args=None requirements/in/base.in +# pip-compile --cert=None --client-cert=None --index-url=None --output-file=requirements/base.txt --pip-args=None --strip-extras requirements/in/base.in # -bleach==6.3.0 +appnope==1.0.0 + # via ipykernel +asttokens==3.0.2 + # via stack-data +attrs==26.1.0 + # via + # jsonschema + # referencing +bleach==6.4.0 # via kaggle -certifi==2025.10.5 +certifi==2026.10.23 # via - # -r requirements/in/constraints.in # kaggle + # kagglesdk # requests -chardet==5.2.0 - # via -r requirements/in/constraints.in -charset-normalizer==3.4.4 +charset-normalizer==3.5.2 + # via requests +cloudpickle==3.1.3 # via - # -r requirements/in/constraints.in - # kaggle - # requests -cloudpickle==3.1.2 - # via shap -colorama==0.4.6 - # via -r requirements/in/constraints.in -contourpy==1.3.3 + # joblib + # shap +comm==0.2.3 + # via ipykernel +contourpy==1.4.0 # via matplotlib cycler==0.12.1 # via matplotlib +debugpy==1.8.22 + # via ipykernel et-xmlfile==2.0.0 # via openpyxl -filelock==3.20.0 - # via -r requirements/in/constraints.in -fonttools==4.60.1 +executing==2.3.0 + # via stack-data +fastjsonschema==2.22.3 + # via nbformat +fonttools==4.68.0 # via matplotlib -idna==3.11 - # via - # -r requirements/in/constraints.in - # kaggle - # requests -iniconfig==2.3.0 - # via -r requirements/in/constraints.in -jmespath==1.0.1 - # via -r requirements/in/constraints.in -joblib==1.5.2 - # via - # -r requirements/in/constraints.in - # scikit-learn -kaggle==1.7.4.5 +idna==3.20 + # via requests +ipykernel==7.4.0 + # via -r requirements/in/base.in +ipython==9.18.1 + # via ipykernel +ipython-pygments-lexers==1.1.1 + # via ipython +jedi==0.20.1 + # via ipython +joblib==1.6.0 + # via scikit-learn +jsonschema==4.26.0 + # via nbformat +jsonschema-specifications==2025.9.1 + # via jsonschema +jupyter-client==8.10.0 + # via ipykernel +jupyter-core==5.9.1 + # via + # ipykernel + # jupyter-client + # nbformat +jupytext==1.19.6 + # via kaggle +kaggle==2.2.5 # via -r requirements/in/base.in -kiwisolver==1.4.9 +kagglesdk==0.1.38 + # via kaggle +kiwisolver==1.5.1 # via matplotlib -llvmlite==0.45.1 - # via numba -matplotlib==3.10.7 +markdown-it-py==4.3.0 + # via + # jupytext + # mdit-py-plugins +matplotlib==3.11.2 # via seaborn -mccabe==0.7.0 - # via -r requirements/in/constraints.in +matplotlib-inline==0.2.2 + # via + # ipykernel + # ipython +mdit-py-plugins==0.7.0 + # via jupytext mdurl==0.1.2 - # via -r requirements/in/constraints.in -more-itertools==10.8.0 - # via -r requirements/in/constraints.in -multidict==6.7.0 - # via -r requirements/in/constraints.in -numba==0.62.1 - # via shap -numpy==2.3.4 + # via markdown-it-py +narwhals==2.29.0 + # via scikit-learn +nbformat==5.11.1 + # via jupytext +nest-asyncio2==1.7.4 + # via ipykernel +numpy==2.5.5 # via # contourpy # matplotlib - # numba # pandas # scikit-learn # scipy @@ -75,128 +103,120 @@ numpy==2.3.4 # shap openpyxl==3.1.5 # via -r requirements/in/base.in -packaging==25.0 +packaging==26.3 # via + # ipykernel + # jupytext + # kaggle # matplotlib # shap -pandas==2.3.3 +pandas==3.1.1 # via # -r requirements/in/base.in # seaborn # shap -pathspec==0.12.1 - # via -r requirements/in/constraints.in -pillow==12.0.0 +parso==0.8.7 + # via jedi +pexpect==4.9.0 + # via ipython +pillow==13.0.0 # via matplotlib -platformdirs==4.5.0 - # via -r requirements/in/constraints.in -pluggy==1.6.0 - # via -r requirements/in/constraints.in -protobuf==6.33.0 - # via kaggle -pyasn1==0.6.1 - # via - # -r requirements/in/constraints.in - # pyasn1-modules - # rsa -pyasn1-modules==0.4.2 - # via -r requirements/in/constraints.in -pycodestyle==2.14.0 - # via -r requirements/in/constraints.in -pycparser==2.23 - # via -r requirements/in/constraints.in -pyflakes==3.4.0 - # via -r requirements/in/constraints.in -pyparsing==3.2.5 - # via - # -r requirements/in/constraints.in - # matplotlib -python-dateutil==2.9.0.post0 +platformdirs==4.14.7 + # via jupyter-core +prompt-toolkit==3.0.53 + # via ipython +protobuf==7.36.2 # via # kaggle + # kagglesdk +psutil==7.2.2 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.4 + # via stack-data +pygments==2.21.0 + # via + # ipython + # ipython-pygments-lexers +pyparsing==3.3.3 + # via matplotlib +python-dateutil==4.0.0 + # via + # jupyter-client + # kaggle # matplotlib # pandas -python-dotenv==1.2.1 - # via -r requirements/in/base.in -python-slugify==8.0.4 - # via kaggle -pytz==2025.2 - # via pandas -rapidfuzz==3.14.3 - # via -r requirements/in/constraints.in -regex==2025.11.3 - # via -r requirements/in/constraints.in -requests==2.32.5 +python-dotenv==1.2.4 + # via + # -r requirements/in/base.in + # kaggle +python-slugify==9.2.0 # via kaggle -rsa==4.9.1 - # via -r requirements/in/constraints.in -scikit-learn==1.7.2 +pyyaml==6.0.3 + # via jupytext +pyzmq==27.3.0 + # via + # ipykernel + # jupyter-client +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.34.2 + # via + # kaggle + # kagglesdk +rpds-py==2026.9.1 + # via + # jsonschema + # referencing +scikit-learn==1.9.1 # via # -r requirements/in/base.in # shap -scipy==1.16.3 +scipy==1.18.1 # via # -r requirements/in/base.in # scikit-learn # shap seaborn==0.13.2 # via -r requirements/in/base.in -shap==0.49.1 +shap==0.54.1 # via -r requirements/in/base.in -six==1.17.0 - # via - # -r requirements/in/constraints.in - # kaggle - # python-dateutil slicer==0.0.8 # via shap -sniffio==1.3.1 - # via -r requirements/in/constraints.in -soupsieve==2.8 - # via -r requirements/in/constraints.in -sqlparse==0.5.3 - # via -r requirements/in/constraints.in -stevedore==5.5.0 - # via -r requirements/in/constraints.in -tenacity==9.1.2 - # via -r requirements/in/constraints.in +stack-data==0.6.3 + # via ipython text-unidecode==1.3 - # via - # kaggle - # python-slugify -threadpoolctl==3.6.0 + # via python-slugify +threadpoolctl==3.8.0 # via scikit-learn -tomlkit==0.13.3 - # via -r requirements/in/constraints.in -tqdm==4.67.1 +tornado==6.6.1 # via - # kaggle - # shap -typing-extensions==4.15.0 + # ipykernel + # jupyter-client +tqdm==4.70.1 # via - # -r requirements/in/constraints.in + # kaggle # shap -tzdata==2025.2 +traitlets==5.16.1 + # via + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbformat +typing-extensions==4.16.0 + # via jupyter-client +tzdata==2026.6 + # via python-dateutil +urllib3==2.8.0 # via - # -r requirements/in/constraints.in - # pandas -urllib3==2.5.0 - # via - # -r requirements/in/constraints.in # kaggle # requests -validators==0.35.0 - # via -r requirements/in/constraints.in -vine==5.1.0 - # via -r requirements/in/constraints.in -wcwidth==0.2.14 - # via -r requirements/in/constraints.in -webencodings==0.5.1 - # via - # bleach - # kaggle -wheel==0.45.1 - # via -r requirements/in/constraints.in - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +wcwidth==0.9.4 + # via prompt-toolkit +webencodings==0.6.1 + # via bleach diff --git a/requirements/in/base.in b/requirements/in/base.in index 1127dc8..07bca03 100644 --- a/requirements/in/base.in +++ b/requirements/in/base.in @@ -9,12 +9,12 @@ -r constraints.in -python-dotenv==1.2.1 # for loading environment variables from .env files -pandas==2.3.3 # for data manipulation -kaggle==1.7.4.5 # for Kaggle api access -seaborn==0.13.2 # for data visualization -scikit-learn==1.7.2 # for machine learning algorithms -scipy==1.16.3 # for scientific computing -shap==0.50.0 # for model interpretability -openpyxl==3.1.5 # for Excel file support from kaggle datasets -ipykernel==7.1.0 # for Jupyter notebook kernel support +python-dotenv # for loading environment variables from .env files +pandas # for data manipulation +kaggle # for Kaggle api access +seaborn # for data visualization +scikit-learn # for machine learning algorithms +scipy # for scientific computing +shap # for model interpretability +openpyxl # for Excel file support from kaggle datasets +ipykernel # for Jupyter notebook kernel support diff --git a/requirements/in/constraints.in b/requirements/in/constraints.in index 81d85ae..1f1bb35 100644 --- a/requirements/in/constraints.in +++ b/requirements/in/constraints.in @@ -5,121 +5,5 @@ # usage: referenced by base.in and local.in to constrain versions # ----------------------------------------------------------------------------- -# Indirect dependency, constrained for compatibility. Update as needed. -certifi>=2026.10.23,<2027.0.0 - -# Indirect dependency, constrained for compatibility. Update as needed. -chardet>=7.7.0,<8.0.0 - -# Indirect dependency, constrained for compatibility. Update as needed. -charset-normalizer>=3.4.0,<4.0.0 - -# Provides cross-platform colored terminal output; often required by CLI tools (e.g., pytest, click) for Windows compatibility. -colorama>=0.4.6,<1.0.0 - -# Used for platform-independent file locking; required by joblib, pytest, and other tools needing concurrency-safe file access. -filelock>=4.0.5,<5.0.0 - -# Handles Internationalized Domain Names; indirect dependency of requests, urllib3, and other networking libraries. -idna>=3.20,<4.0.0 - -# Used for parsing INI files; required by pytest for configuration file support. -iniconfig>=2.3.1,<3.0.0 - -# JSON query language; required by boto3 and other AWS SDKs for querying JSON responses. -jmespath>=1.0.1,<2.0.0 - -# Lightweight pipelining and parallelization; used by scikit-learn and other scientific packages. -joblib>=1.5.2,<2.0.0 - -# URL parsing for Markdown; indirect dependency of markdown-it-py, used by rich and other Markdown rendering tools. -mdurl>=0.1.2,<1.1.0 - -# Complexity checker for Python code; required by flake8 for code quality checks. -mccabe>=0.7.0,<1.0.0 - -# Additional itertools for Python; used by pytest and other testing frameworks for advanced iteration utilities. -more-itertools>=10.8.0,<11.0.0 - -# Multi-value dictionary implementation; required by aiohttp and yarl for async web frameworks. -multidict>=6.6.4,<7.0.0 - -# Pattern matching for file paths; used by black, pre-commit, and other tools for file selection. -pathspec>=0.12.1,<1.0.0 - -# Determines platform-specific directories; used by black, jupyter, and other tools for config/cache locations. -platformdirs>=4.4.0,<5.0.0 - -# Plugin management framework; required by pytest and tox for plugin support. -pluggy>=1.6.0,<2.0.0 - -# ASN.1 types and codecs; required by cryptography, pyasn1-modules, and some authentication libraries. -pyasn1>=0.6.1,<1.0.0 - -# Additional ASN.1 modules; used with pyasn1 for cryptography and network protocols. -pyasn1-modules>=0.4.2,<1.0.0 - -# Python style guide checker; required by flake8 for PEP8 compliance checks. -pycodestyle>=2.14.0,<3.0.0 - -# C parser in Python; required by cffi, which is used by cryptography and other C-extension libraries. -pycparser>=3.12,<4.0.0 - -# Static code analysis for Python; used by flake8 for linting. -pyflakes>=3.4.0,<4.0.0 - -# Parsing library for grammars; required by packaging, matplotlib, and other libraries for parsing expressions. -pyparsing>=3.2.3,<4.0.0 - -# Fast fuzzy string matching; used by rich and other libraries for text processing. -rapidfuzz>=3.14.6,<4.0.0 - -# Enhanced regular expressions; used by black, dateparser, and other tools needing advanced regex features. -regex>=2025.9.1,<2026.0.0 - -# Pure Python RSA implementation; required by google-auth and other authentication libraries. -rsa>=4.9.1,<5.0.0 - -# Python 2 and 3 compatibility utilities; still required by many libraries for legacy support. -six>=1.17.0,<2.0.0 - -# Detects async library context; required by httpx, trio, and other async frameworks. -sniffio>=1.3.1,<2.0.0 - -# CSS selector library for BeautifulSoup; required by beautifulsoup4 for HTML/XML parsing. -soupsieve>=2.8,<3.0.0 - -# SQL parsing library; used by Django and other ORMs for SQL formatting and analysis. -sqlparse>=0.6.0,<1.0.0 - -# Plugin management for Python; used by OpenStack and other extensible frameworks. -stevedore>=5.9.1,<6.0.0 - -# Retry utility for Python; used by celery, requests, and other libraries needing robust retry logic. -tenacity>=9.1.2,<10.0.0 - -# TOML file parser and writer; used by poetry, black, and other tools for pyproject.toml support. -tomlkit>=0.15.1,<1.0.0 - -# Backports of typing features; required by libraries supporting older Python versions. -typing-extensions>=4.15.0,<5.0.0 - -# Time zone database; used by dateutil, pandas, and other libraries for timezone support. -tzdata>=2025.2,<2026.0 - -# HTTP client library; indirect dependency of requests and other networking libraries. -urllib3>=2.5.0,<3.0.0 - -# Data validation for strings, URLs, etc.; used by various web and data libraries. -validators>=0.35.0,<1.0.0 - -# Promises/A+ implementation; required by celery for async task management. -vine>=5.1.0,<6.0.0 - -# Measures printable width of unicode strings; used by rich, prompt-toolkit, and other CLI tools. -wcwidth>=0.9.1,<1.0.0 - -# Built-package format for Python; required for building and distributing Python packages. -wheel>=0.45.1,<1.0.0 - -# URL parsing for async frameworks; required by aiohttp for URL handling. +# No global version constraints today. Add one only for a known incompatibility, +# with a comment saying which package needs it and why. diff --git a/requirements/in/local.in b/requirements/in/local.in index 5ac075e..c6191f3 100644 --- a/requirements/in/local.in +++ b/requirements/in/local.in @@ -11,18 +11,19 @@ -r constraints.in -r base.in -pip-tools==7.5.0 +pip-tools +pre-commit # runs the hooks in .pre-commit-config.yaml; CI installs its pin from local.txt # Code linters, formatters, and security scanners # ------------ -bandit==1.9.2 -black==25.1.0 -codespell==2.4.1 -flake8==7.3.0 -flake8-coding==1.3.2 -isort==6.0.1 -mypy==1.19.1 -mypy_extensions==1.1.0 -pydocstringformatter==0.7.3 -pylint==3.3.7 -tox==4.32.0 +bandit +black +codespell +flake8 +flake8-coding +isort +mypy +mypy_extensions +pydocstringformatter +pylint +tox diff --git a/requirements/local.txt b/requirements/local.txt index 80375a0..1055720 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -2,128 +2,160 @@ # This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --cert=None --client-cert=None --index-url=None --output-file=requirements/local.txt --pip-args=None requirements/in/local.in +# pip-compile --cert=None --client-cert=None --index-url=None --output-file=requirements/local.txt --pip-args=None --strip-extras requirements/in/local.in # -astroid==3.3.11 +appnope==1.0.0 + # via ipykernel +ast-serialize==0.13.1 + # via mypy +astroid==4.3.4 # via pylint -bandit==1.8.5 +asttokens==3.0.2 + # via stack-data +attrs==26.1.0 + # via + # jsonschema + # referencing +bandit==1.9.4 # via -r requirements/in/local.in -black==25.1.0 +black==26.10.1 # via -r requirements/in/local.in -bleach==6.3.0 +bleach==6.4.0 # via kaggle -build==1.3.0 +build==1.6.1 # via pip-tools -cachetools==6.2.1 +cachetools==8.0.1 # via tox -certifi==2025.10.5 +certifi==2026.10.23 # via - # -r requirements/in/constraints.in # kaggle + # kagglesdk # requests -cfgv==3.4.0 +cfgv==3.5.0 # via pre-commit -chardet==5.2.0 - # via - # -r requirements/in/constraints.in - # tox -charset-normalizer==3.4.4 - # via - # -r requirements/in/constraints.in - # kaggle - # requests -click==8.3.0 +charset-normalizer==3.5.2 + # via requests +click==8.5.1 # via # black # pip-tools -cloudpickle==3.1.2 - # via shap -codespell==2.4.1 +cloudpickle==3.1.3 + # via + # joblib + # shap +codespell==2.4.3 # via -r requirements/in/local.in colorama==0.4.6 - # via - # -r requirements/in/constraints.in - # tox -contourpy==1.3.3 + # via tox +comm==0.2.3 + # via ipykernel +contourpy==1.4.0 # via matplotlib cycler==0.12.1 # via matplotlib -dill==0.4.0 +debugpy==1.8.22 + # via ipykernel +dill==0.4.1 # via pylint -distlib==0.4.0 +distlib==0.4.4 # via virtualenv et-xmlfile==2.0.0 # via openpyxl -filelock==3.20.0 +executing==2.3.0 + # via stack-data +fastjsonschema==2.22.3 + # via nbformat +filelock==4.2.12 # via - # -r requirements/in/constraints.in + # python-discovery # tox # virtualenv -flake8==7.3.0 +flake8==7.5.0 # via # -r requirements/in/local.in # flake8-coding flake8-coding==1.3.2 # via -r requirements/in/local.in -fonttools==4.60.1 +fonttools==4.68.0 # via matplotlib -identify==2.6.15 +identify==2.6.20 # via pre-commit -idna==3.11 - # via - # -r requirements/in/constraints.in - # kaggle - # requests -iniconfig==2.3.0 - # via -r requirements/in/constraints.in -isort==6.0.1 +idna==3.20 + # via requests +ipykernel==7.4.0 + # via -r requirements/in/base.in +ipython==9.18.1 + # via ipykernel +ipython-pygments-lexers==1.1.1 + # via ipython +isort==9.0.2 # via # -r requirements/in/local.in # pylint -jmespath==1.0.1 - # via -r requirements/in/constraints.in -joblib==1.5.2 - # via - # -r requirements/in/constraints.in - # scikit-learn -kaggle==1.7.4.5 +jedi==0.20.1 + # via ipython +joblib==1.6.0 + # via scikit-learn +jsonschema==4.26.0 + # via nbformat +jsonschema-specifications==2025.9.1 + # via jsonschema +jupyter-client==8.10.0 + # via ipykernel +jupyter-core==5.9.1 + # via + # ipykernel + # jupyter-client + # nbformat +jupytext==1.19.6 + # via kaggle +kaggle==2.2.5 # via -r requirements/in/base.in -kiwisolver==1.4.9 +kagglesdk==0.1.38 + # via kaggle +kiwisolver==1.5.1 # via matplotlib -llvmlite==0.45.1 - # via numba -markdown-it-py==4.0.0 - # via rich -matplotlib==3.10.7 +librt==0.16.0 + # via mypy +markdown-it-py==4.3.0 + # via + # jupytext + # mdit-py-plugins + # rich +matplotlib==3.11.2 # via seaborn +matplotlib-inline==0.2.2 + # via + # ipykernel + # ipython mccabe==0.7.0 # via - # -r requirements/in/constraints.in # flake8 # pylint +mdit-py-plugins==0.7.0 + # via jupytext mdurl==0.1.2 - # via - # -r requirements/in/constraints.in - # markdown-it-py -more-itertools==10.8.0 - # via -r requirements/in/constraints.in -multidict==6.7.0 - # via -r requirements/in/constraints.in -mypy==1.17.1 + # via markdown-it-py +mypy==2.4.1 # via -r requirements/in/local.in -mypy-extensions==1.1.0 +mypy-extensions==1.2.0 # via + # -r requirements/in/local.in # black + # isort # mypy -nodeenv==1.9.1 +narwhals==2.29.0 + # via scikit-learn +nbformat==5.11.1 + # via jupytext +nest-asyncio2==1.7.4 + # via ipykernel +nodeenv==1.12.0 # via pre-commit -numba==0.62.1 - # via shap -numpy==2.3.4 +numpy==2.5.5 # via # contourpy # matplotlib - # numba # pandas # scikit-learn # scipy @@ -131,181 +163,186 @@ numpy==2.3.4 # shap openpyxl==3.1.5 # via -r requirements/in/base.in -packaging==25.0 +packaging==26.3 # via # black # build + # ipykernel + # jupytext + # kaggle # matplotlib # pyproject-api # shap # tox -pandas==2.3.3 + # virtualenv + # wheel +pandas==3.1.1 # via # -r requirements/in/base.in # seaborn # shap -pathspec==0.12.1 +parso==0.8.7 + # via jedi +pathspec==1.1.1 # via - # -r requirements/in/constraints.in # black # mypy -pillow==12.0.0 +pexpect==4.9.0 + # via ipython +pillow==13.0.0 # via matplotlib -pip-tools==7.5.0 +pip-tools==7.7.0 # via -r requirements/in/local.in -platformdirs==4.5.0 +platformdirs==4.14.7 # via - # -r requirements/in/constraints.in # black + # jupyter-core # pylint # tox # virtualenv -pluggy==1.6.0 - # via - # -r requirements/in/constraints.in - # tox -pre-commit==4.2.0 +pluggy==1.7.0 + # via tox +pre-commit==4.7.0 # via -r requirements/in/local.in -protobuf==6.33.0 - # via kaggle -pyasn1==0.6.1 +prompt-toolkit==3.0.53 + # via ipython +protobuf==7.36.2 # via - # -r requirements/in/constraints.in - # pyasn1-modules - # rsa -pyasn1-modules==0.4.2 - # via -r requirements/in/constraints.in -pycodestyle==2.14.0 - # via - # -r requirements/in/constraints.in - # flake8 -pycparser==2.23 - # via -r requirements/in/constraints.in -pydocstringformatter==0.7.3 + # kaggle + # kagglesdk +psutil==7.2.2 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.4 + # via stack-data +pycodestyle==2.15.0 + # via flake8 +pydocstringformatter==1.0.0 # via -r requirements/in/local.in -pyflakes==3.4.0 - # via - # -r requirements/in/constraints.in - # flake8 -pygments==2.19.2 - # via rich -pylint==3.3.7 +pyflakes==4.1.0 + # via flake8 +pygments==2.21.0 + # via + # ipython + # ipython-pygments-lexers + # rich +pylint==4.1.4 # via -r requirements/in/local.in -pyparsing==3.2.5 - # via - # -r requirements/in/constraints.in - # matplotlib -pyproject-api==1.10.0 +pyparsing==3.3.3 + # via matplotlib +pyproject-api==1.12.6 # via tox -pyproject-hooks==1.2.0 +pyproject-hooks==1.3.3 # via # build # pip-tools -python-dateutil==2.9.0.post0 +python-dateutil==4.0.0 # via + # jupyter-client # kaggle # matplotlib # pandas -python-dotenv==1.2.1 - # via -r requirements/in/base.in -python-slugify==8.0.4 +python-discovery==1.9.6 + # via + # tox + # virtualenv +python-dotenv==1.2.4 + # via + # -r requirements/in/base.in + # kaggle +python-slugify==9.2.0 # via kaggle -pytz==2025.2 - # via pandas +pytokens==0.4.1 + # via black pyyaml==6.0.3 # via # bandit + # jupytext # pre-commit -rapidfuzz==3.14.3 - # via -r requirements/in/constraints.in -regex==2025.11.3 - # via -r requirements/in/constraints.in -requests==2.32.5 - # via kaggle -rich==14.2.0 +pyzmq==27.3.0 + # via + # ipykernel + # jupyter-client +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.34.2 + # via + # kaggle + # kagglesdk +rich==15.0.0 # via bandit -rsa==4.9.1 - # via -r requirements/in/constraints.in -scikit-learn==1.7.2 +rpds-py==2026.9.1 + # via + # jsonschema + # referencing +scikit-learn==1.9.1 # via # -r requirements/in/base.in # shap -scipy==1.16.3 +scipy==1.18.1 # via # -r requirements/in/base.in # scikit-learn # shap seaborn==0.13.2 # via -r requirements/in/base.in -shap==0.49.1 +shap==0.54.1 # via -r requirements/in/base.in -six==1.17.0 - # via - # -r requirements/in/constraints.in - # kaggle - # python-dateutil slicer==0.0.8 # via shap -sniffio==1.3.1 - # via -r requirements/in/constraints.in -soupsieve==2.8 - # via -r requirements/in/constraints.in -sqlparse==0.5.3 - # via -r requirements/in/constraints.in -stevedore==5.5.0 - # via - # -r requirements/in/constraints.in - # bandit -tenacity==9.1.2 - # via -r requirements/in/constraints.in +stack-data==0.6.3 + # via ipython +stevedore==5.10.0 + # via bandit text-unidecode==1.3 - # via - # kaggle - # python-slugify -threadpoolctl==3.6.0 + # via python-slugify +threadpoolctl==3.8.0 # via scikit-learn -tomlkit==0.13.3 +tomli-w==1.2.0 + # via tox +tomlkit==0.15.2 + # via pylint +tornado==6.6.1 # via - # -r requirements/in/constraints.in - # pylint -tox==4.28.4 + # ipykernel + # jupyter-client +tox==4.67.15 # via -r requirements/in/local.in -tqdm==4.67.1 +tqdm==4.70.1 # via # kaggle # shap -typing-extensions==4.15.0 +traitlets==5.16.1 # via - # -r requirements/in/constraints.in - # mypy - # shap -tzdata==2025.2 + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbformat +typing-extensions==4.16.0 # via - # -r requirements/in/constraints.in - # pandas -urllib3==2.5.0 + # jupyter-client + # mypy +tzdata==2026.6 + # via python-dateutil +urllib3==2.8.0 # via - # -r requirements/in/constraints.in # kaggle # requests -validators==0.35.0 - # via -r requirements/in/constraints.in -vine==5.1.0 - # via -r requirements/in/constraints.in -virtualenv==20.35.4 +virtualenv==21.15.8 # via # pre-commit # tox -wcwidth==0.2.14 - # via -r requirements/in/constraints.in -webencodings==0.5.1 - # via - # bleach - # kaggle -wheel==0.45.1 - # via - # -r requirements/in/constraints.in - # pip-tools +wcwidth==0.9.4 + # via prompt-toolkit +webencodings==0.6.1 + # via bleach +wheel==0.48.0 + # via pip-tools # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 0000000..1af2db3 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,66 @@ +# ai-algorithms Agent Skills + +This folder holds the [Agent Skills](https://agentskills.io/specification) for +working on this repository with an AI coding agent such as Claude Code. This +README is for people; the `SKILL.md` files are for the agent. + +A skill is a folder of written know-how that an agent loads only when a task +needs it. Each contains a `SKILL.md`, with a `name` and a `description` in its +front matter that tell the agent when to load it. They are developer tooling, +like the `Makefile` and `.pre-commit-config.yaml`; no code here loads them. + +## What's here + +Start with `ai-algorithms-development`. It sets the rules for every change, and +its table says which narrower skill to load. + +| Skill | Use it when | +| ---------------------------------- | -------------------------------------------------------------------- | +| `ai-algorithms-development` | You're making any change. Start here. | +| `ai-algorithms-notebooks` | You're editing, running or reviewing a notebook, or touching Kaggle. | +| `ai-algorithms-python-environment` | The venv, Python 3.13, requirements or make targets. | +| `ai-algorithms-code-quality` | You're finishing a change and need the pre-commit hooks. | +| `ai-algorithms-ci` | Workflows, semantic-release, Dependabot, or a failed run. | +| `ai-algorithms-git-commits` | You've been asked to write a commit message. | +| `ai-algorithms-pull-requests` | You're creating or describing a pull request. | + +The repository's [CLAUDE.md](../CLAUDE.md) holds the few rules that apply to +nearly every task, and points here for the rest. + +## How agents find them + +Claude Code reads project skills from `.claude/skills/`, which is a committed +symlink to this folder. It loads each skill's description at the start of a +session and the body when a task matches; you can also call one directly, e.g. +`/ai-algorithms-git-commits`. If the symlink breaks (e.g. a Windows checkout +without symlink support), Claude Code silently stops seeing the skills. + +Other agents that support the Agent Skills format read `SKILL.md` folders from +their own configuration location. Agents without skill support can be told to +read `skills/ai-algorithms-development/SKILL.md` at the start of a session. + +## Security considerations + +A skill is a set of instructions an agent follows, often without asking. +**Review a change to this folder with the same care as a change to CI.** + +- **The local environment holds live credentials.** `~/.kaggle/kaggle.json` + is a real Kaggle key, and one notebook reaches a paid Azure ML workspace. + `ai-algorithms-notebooks` exists so that an agent doesn't run those cells + on its own. Don't weaken it. +- **A rule applies to every task that triggers the skill.** State limits + rather than absolutes that become harmful in some case. +- **Broad descriptions have a wide reach.** A skill meant for notebooks that + says "use when changing Python" will load, and be followed, for unrelated + work. +- **Some rules forbid things on purpose.** `ai-algorithms-git-commits` tells + the agent never to run `git commit`, `git add` or `git push`, and + `ai-algorithms-ci` never to trigger workflows or releases. Don't remove that + unless you mean to give agents that power. +- **Watch for prompt injection.** Look for instructions unrelated to a skill's + purpose, new scripts or network calls, text that tells the agent to skip + confirmations or hooks, and links it is told to fetch and follow. +- **Stale skills mislead.** When a convention changes (a make target, the + Python version, a workflow), update the skill in the same pull request. +- **Keep secrets out.** This repository is public. Use placeholders, and tell + the agent where to find the real value at run time. diff --git a/skills/ai-algorithms-ci/SKILL.md b/skills/ai-algorithms-ci/SKILL.md new file mode 100644 index 0000000..3e2291b --- /dev/null +++ b/skills/ai-algorithms-ci/SKILL.md @@ -0,0 +1,85 @@ +--- +name: ai-algorithms-ci +description: Use when changing or troubleshooting ai-algorithms' GitHub Actions workflows (test.yml, release.yml, precommitVersionBumps.yml, auto-assign.yml), semantic-release (release.config.js, package.json, CHANGELOG.md, tags), Dependabot (.github/dependabot.yml) or Mergify (.mergify.yml), or when asked why a workflow run failed. Covers what each workflow does and needs, how to read a failed run with gh, the release branches, the PAT secret, the CI skip trap, and what never to trigger yourself. +--- + +# CI and Releases + +## The workflows + +| Workflow | Trigger | What it does | +| --------------------------- | ---------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `test.yml` | every pull request; pushes to main, next | Runs every pre-commit hook on every file (Python 3.13, `requirements/local.txt`). Job `pre-commit` | +| `release.yml` | push to `main`, `workflow_dispatch` | semantic-release: version, `CHANGELOG.md`, tag, GitHub release; then pushes the major branch `vN` | +| `precommitVersionBumps.yml` | weekly cron (Wed 00:00 UTC), dispatch | `pre-commit autoupdate` on `alpha`; commits and pushes `.pre-commit-config.yaml` if it changed | +| `auto-assign.yml` | issue or pull request opened | Assigns it to the maintainer | + +There's no Python package, so there's no `__version__.py` and no version-bump +workflow for alpha and beta (azureml-example has one because it has a +package). There are no unit tests either; `test.yml` is the lint check. + +Conventions shared with azureml-example (keep them): + +- A header comment says what the workflow does and why. +- `permissions: contents: read` at the top; writes use the `PAT` secret. +- Every action is pinned to a commit SHA with its version in a comment + (`actions/checkout@ # v7.0.1`). Dependabot's `github-actions` updates + bump them. Copy SHAs from azureml-example's workflows when adding a step. +- `timeout-minutes` on every job; `concurrency` where runs could overlap. +- Step outputs go to `"$GITHUB_OUTPUT"`, never `::set-output` (removed). +- `actions/setup-node` without `node-version: latest`: "latest" can be a + pre-release Node that semantic-release refuses to run on. +- No `pull_request_target`. A fork's pull request runs its own code with + `pull_request`, without the repository's secrets. Point out to the user any + fork pull request that changes `.github/`: once merged, it runs with them. + +## semantic-release + +- `release.config.js` is the one place that lists the release branches + (`main`, `next`, prereleases `beta` and `alpha`) and plugins. Workflows + don't repeat the branch list. +- `feat` makes a minor release, `fix` and `perf` a patch, `BREAKING CHANGE` + a major; `docs`, `test`, `style` and `refactor` make none (`releaseRules`). +- The release commit is `chore(release): ` with **no CI skip + instruction**, so that the pull request from alpha to main still runs its + checks; `release.yml` skips its own commit with an `if:` instead. +- The `@semantic-release/git` plugin commits `CHANGELOG.md`, `package.json`, + `package-lock.json` and `requirements/**/*`. +- `package.json` holds only semantic-release, its plugins, + `conventional-changelog-conventionalcommits` (the preset it uses) and + Prettier. + +## Dependabot and Mergify + +Dependabot opens monthly pull requests for `requirements/` (pip), npm and +GitHub Actions. Mergify approves and merges Dependabot's pull requests into +`main` once checks pass, and each merge is a release (`chore(deps)` commits +release nothing, a `fix(deps)` would). A Dependabot pull request that only +edits a `.in` constraint leaves the compiled `.txt` stale; recompile +(`ai-algorithms-python-environment`). + +## Troubleshooting a failed run + +Read-only commands, fine to run: + +```bash +gh run list --status failure --limit 20 +gh run view # jobs and the failing step +gh run view --log-failed | cut -f3- | tail -40 +``` + +Then find the cause in the repository, not in the runner: most failures here +have been template leftovers (a file the workflow expects that this +repository doesn't have). Check every path a workflow names +(`requirements/*.txt`, a package folder, a branch) against `git ls-files`. +A scheduled workflow runs the default branch's copy of the file (`main`), so a +fix on `alpha` doesn't reach the cron run until it's merged. + +## What needs the user + +Never run these yourself; give the user the command: +`gh workflow run`, `gh run rerun`, `gh run cancel`, `gh release create|delete`, +`make release`, `npx semantic-release` without `--dry-run`, `git push`, +deleting branches or tags, and changes to repository settings, secrets or +rulesets. `npx semantic-release --dry-run --no-ci` needs a GitHub token and +pushes nothing; still ask before running it. diff --git a/skills/ai-algorithms-code-quality/SKILL.md b/skills/ai-algorithms-code-quality/SKILL.md new file mode 100644 index 0000000..25a2251 --- /dev/null +++ b/skills/ai-algorithms-code-quality/SKILL.md @@ -0,0 +1,65 @@ +--- +name: ai-algorithms-code-quality +description: Use before finishing any ai-algorithms change, to lint and format Markdown, JSON, YAML, notebooks and any Python with the repository's pre-commit hooks (pyupgrade, codespell, Prettier, black, flake8, isort, autoflake, bandit, pre-commit-hooks, commitlint), or when changing .pre-commit-config.yaml, pyproject.toml, .flake8, .codespellrc or codespell.txt. Explains what each hook enforces, how to run the hooks on changed files under zsh, and the style conventions. +--- + +# Code Quality + +`.pre-commit-config.yaml` is the standard. If a hook passes, the change meets +it; if it fails, fix the change. Never bypass a hook (`--no-verify`, `SKIP=`, a +blanket `# noqa`). + +## What the hooks enforce + +| Hook | Configuration | Notes | +| ---------------- | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| pyupgrade | default | Python files only (there are none today) | +| codespell | `.codespellrc`, `--ignore-words=codespell.txt` | Skips `requirements/` and `jupyter/`. Add real project words to `codespell.txt` | +| prettier | `.prettierrc.json`, `.prettierignore` | Markdown, JSON, YAML. Let it reformat | +| black | `pyproject.toml`, line length 120, `py313` | Formatter | +| flake8 | `.flake8` | `select = C101` only (coding magic comment) | +| isort | `pyproject.toml` (`profile = black`) | The hook passes `--settings-path=pyproject.toml`, so that file must exist | +| autoflake | `--remove-all-unused-imports` | | +| bandit | `-ll` | Medium and high findings fail | +| pre-commit-hooks | v6 | JSON (notebooks too)/TOML/XML/YAML syntax, merge markers, symlinks, private keys, AWS keys, trailing whitespace, end-of-file | +| commitlint | `commitlint.config.js`, commit-msg stage | See `ai-algorithms-git-commits` | + +`default_install_hook_types: [pre-commit, commit-msg]` makes `pre-commit +install` install the commitlint hook too. + +## How to run + +```bash +source venv/bin/activate +git diff --name-only | xargs pre-commit run --files +git ls-files --others --exclude-standard | xargs pre-commit run --files # new files +``` + +- Use `xargs`: under zsh `pre-commit run --files $FILES` passes one argument + and checks nothing. +- Hooks that rewrite files (black, isort, Prettier, end-of-file-fixer) fail + the first run and pass the second. Run twice, and read the diff. +- `--all-files` skips untracked files. +- `make pre-commit` runs `pre-commit autoupdate`, which edits + `.pre-commit-config.yaml`. Don't run it as part of another change; the + weekly `precommitVersionBumps.yml` workflow does it. +- The hooks bring their own environments, but the Python ones need a + `python3.13` on the PATH. + +## Changing the hook configuration + +- Keep it in step with azureml-example's `.pre-commit-config.yaml`, apart from + the Python version and the `exclude` paths that fit this repository. +- A hook that references a settings file (isort's `--settings-path`) needs + that file committed, or it fails on every run. +- Every tool a hook needs in CI must be installable from + `requirements/local.txt` (`pre-commit` itself is pinned there). + +## Style + +- Simple over compact: about one operation per line. Students read this code + line by line. +- Markdown docs: weave new material into the guide it belongs to rather than + adding a "what's new" section. +- Comments explain what a student wouldn't guess, not what the next line does. +- Don't reformat files you didn't change. diff --git a/skills/ai-algorithms-development/SKILL.md b/skills/ai-algorithms-development/SKILL.md new file mode 100644 index 0000000..bc06491 --- /dev/null +++ b/skills/ai-algorithms-development/SKILL.md @@ -0,0 +1,113 @@ +--- +name: ai-algorithms-development +description: Start here for any change to the ai-algorithms repository (the Jupyter notebooks in jupyter/, the Markdown docs, the requirements, the Makefile, pre-commit, semantic-release, the GitHub workflows, commits and pull requests). Gives the non-negotiable rules, the pattern-first workflow, known oddities left over from the template the repository was made from, and an index of the narrower ai-algorithms-* skills to load for each kind of work. +--- + +# ai-algorithms Development + +ai-algorithms is a teaching repository from Full Stack With Lawrence: Jupyter +notebooks that demonstrate basic AI algorithms in Python (anomaly detection, +regression, k-means clustering), for university courses and YouTube +tutorials. There is no Python package and no unit test suite: **the notebooks +are the product**, and everything else (requirements, Makefile, pre-commit, +semantic-release, workflows) exists to keep them runnable and versioned. + +Its sister repository, `FullStackWithLawrence/azureml-example`, was made from +the same template and has been modernized first. When this repository's +tooling and azureml-example's differ for no reason, azureml-example's version +is the reference. + +## Repository map + +| Path | What lives there | +| ------------------------------------------- | ------------------------------------------------------------------------------------------- | +| `jupyter/*.ipynb` | The course notebooks. See `ai-algorithms-notebooks` | +| `jupyter/data/` | Kaggle downloads the notebooks read. Git-ignored; never commit it | +| `docs/*.md` | Guides. Several are Azure guides copied from azureml-example (see oddities) | +| `requirements/` | `in/*.in` sources, compiled by pip-compile into `base.txt` and `local.txt` | +| `Makefile` | `init`, `python-requirements`, `lint`, `pre-commit`. See `ai-algorithms-python-environment` | +| `pyproject.toml`, `.flake8`, `.codespellrc` | Tool settings the pre-commit hooks read. See `ai-algorithms-code-quality` | +| `.pre-commit-config.yaml` | The lint and format standard | +| `package.json` | semantic-release and its plugins, Prettier | +| `commitlint.config.js`, `release.config.js` | Commit message rules and semantic-release (branches main, next, beta, alpha) | +| `.github/workflows/` | Lint on pull requests, semantic-release on `main`, pre-commit version bumps, auto-assign | +| `.github/dependabot.yml`, `.mergify.yml` | Monthly dependency pull requests; Mergify approves and merges Dependabot's into `main` | +| `skills/` (`.claude/skills` is a symlink) | These Agent Skills | + +## Non-negotiable rules + +1. **Never execute git write commands.** "Write a git commit" means output the + command text. See `ai-algorithms-git-commits`. +2. **Don't run notebook cells that reach Kaggle or Azure** without the user's + OK. `~/.kaggle/kaggle.json` is the user's real key, and + `anomaly_detection_creditcard.ipynb` registers a model in a real Azure ML + workspace. See `ai-algorithms-notebooks`. +3. **Python 3.13 everywhere.** The Makefile, `.pre-commit-config.yaml`, + `pyproject.toml`, the workflows and the notebook kernels must agree. Regenerate + `requirements/*.txt` with `make python-requirements`; never edit them by + hand. See `ai-algorithms-python-environment`. +4. **The pre-commit hooks are the lint standard.** See + `ai-algorithms-code-quality`. +5. **The repository is public.** No Kaggle keys, Azure subscription ids, + personal paths or other personal data in notebooks (outputs included), + docs, workflows or skills. `.env`, `config.json` and `kaggle.json` are + git-ignored and stay that way. +6. **Notebooks are course material.** Edit cell by cell, keep the teaching + voice, and don't strip or regenerate outputs a lesson relies on. +7. **Don't trigger workflows, releases or pushes.** `gh workflow run`, + `gh run rerun`, `make release`, `git push` and anything that publishes are + the user's to run. See `ai-algorithms-ci`. + +## Pattern first + +Before writing anything, find the closest existing example and copy its shape: +an existing notebook's step-by-step structure (a Markdown cell explaining each +`Step N`, then the code), an existing `.in` file's commented pins, an existing +workflow's header comment and pinned action SHAs. For tooling, check how +azureml-example does it (`/Users/mcdaniel/Desktop/gh/fswl/azureml-example`, if +it is checked out next to this one). + +## Known oddities (don't copy, don't "fix" unasked) + +These came from the template. Leave them alone unless the task is about them, +and don't spread them to new files. When one bites the task at hand, fix it as +its own change and say so. + +- `docs/AZURE_*.md`, `docs/12-FACTOR.md` and `docs/REST_BASICS.md` are + azureml-example and template guides; `docs/CONTRIBUTING.md` mentions + Smarter, Docker and gulp; `.github/CONTRIBUTING.md` describes a Terraform + and ReactJS project. None of that exists here. +- `conda.yaml` describes an `automl-env` with onnxruntime. Nothing uses it. +- `jupyter/anomaly_detection_creditcard.ipynb` uses the retired Azure ML SDK + v1 (`azureml.core`) with a Python 3.9 kernel. It can't run in the project's + 3.13 venv. azureml-example has the maintained v2 version of this notebook. +- The README's algorithm list is out of date: k-means uses the UCI Iris data + (not Kaggle), and `k-means-it-support-tickets.ipynb` isn't listed. +- `.github/actions/merge-branch/action.yml` is a composite action no workflow + calls. +- The committed outputs of `k-means.ipynb`'s first cell include `pip` output + with the maintainer's home-directory path. + +## Which skill to load + +| Task | Skill | +| ---------------------------------------------------------------------------- | ---------------------------------- | +| Reading, editing, running or reviewing a notebook; Kaggle or Azure access | `ai-algorithms-notebooks` | +| The venv, Python version, requirements, Makefile targets | `ai-algorithms-python-environment` | +| Lint, format, pre-commit, codespell | `ai-algorithms-code-quality` | +| GitHub workflows, semantic-release, Dependabot, troubleshooting a failed run | `ai-algorithms-ci` | +| Writing a commit message | `ai-algorithms-git-commits` | +| Creating or describing a pull request | `ai-algorithms-pull-requests` | + +## Definition of done + +- The change follows an existing pattern and reads well to a student. +- Notebooks you changed still parse (`python -m json.tool > /dev/null`), + and any local-only cells you changed were run in the project venv, or you + said why not. +- The pre-commit hooks pass on the changed files. +- Tooling changes are consistent across the Makefile, pre-commit, pyproject, + the workflows and these skills. +- Nothing was pushed, released or run against Kaggle or Azure without the + user's say-so. +- Your summary says what changed, what you verified, and what you skipped. diff --git a/skills/ai-algorithms-git-commits/SKILL.md b/skills/ai-algorithms-git-commits/SKILL.md new file mode 100644 index 0000000..aa113ba --- /dev/null +++ b/skills/ai-algorithms-git-commits/SKILL.md @@ -0,0 +1,96 @@ +--- +name: ai-algorithms-git-commits +description: Use whenever the user asks you to "write a git commit" (or a commit message, or the git add) in the ai-algorithms repository. Covers the never-execute rule, authorship, the multiple -m command format, conventional-commit types and scopes as commitlint and semantic-release read them, the 72-character header limit, zsh quoting traps, the CI skip trap, and how thorough the body must be. +--- + +# Git Commits + +## The rule: write, never execute + +"Write a git commit", in any wording or tone, means **output the command as +text**. It never means run it. Emphatic or frustrated phrasing still asks for +the command text. + +- Never execute `git commit`, `git add`, `git reset`, `git restore --staged`, + `git push`, `git tag`, `git stash`, or any other git command that writes. + The user stages, commits and pushes. (`.claude/settings.local.json` denies + most of these too; don't look for a way around it.) +- Read-only commands (`git status`, `git diff`, `git log`) are fine. +- Don't volunteer commit messages. Write one only when asked. + +## Authorship + +You are the author; Lawrence McDaniel is the co-author. + +- `--author="Claude "` +- The last `-m` is `Co-authored-by: Lawrence McDaniel `. + +## Command format + +Exactly one `git commit` command in one code block: + +```console +git commit --author="Claude " \ + -m "(): " \ + -m "" \ + -m "" \ + -m "Co-authored-by: Lawrence McDaniel " +``` + +- One `-m` per paragraph; git joins them with blank lines. +- No heredoc, no `-F`, no `git add` in the same block. When the user asks for + the `git add`, write it as a separate block that lists paths explicitly + (and `git rm` for deleted files). +- No backticks, `$` or `!` inside the double-quoted messages. In zsh a `!` in + double quotes is history expansion and aborts the command with + `event not found`. Reword instead. +- **Never write a CI skip instruction** anywhere in the message, not even to + describe one. GitHub then starts no workflows for the push. Write "a CI skip + instruction". + +## Subject line + +`(): `, checked by commitlint +(`@commitlint/config-conventional` + `@commitlint/config-angular`) and read by +semantic-release (`release.config.js`), which versions the release from it. + +- **type:** `feat`, `fix`, `docs`, `refactor`, `perf`, `test`, `style`, + `build`, `ci`, `chore`. + - `feat` (minor release) is for a truly new capability, such as a new + notebook demonstrating a new algorithm. When in doubt, it is `fix`. + - `fix` (patch release) covers bugs and shortcomings of something that + exists, a broken notebook included. + - Notebook prose and docs are `docs` (no release); dependency pins are + `build` or `chore(deps)`; workflow changes are `ci`. + - `chore(release): ...` belongs to semantic-release. Never write it. +- **scope:** the area, in lower case: `notebooks`, `kmeans`, `anomaly`, + `regression`, `docs`, `deps`, `ci`, `pre-commit`, `make`, `skills`. Omit it + for repository-wide changes. +- **subject:** imperative mood, lower case, no trailing period. +- **length:** the whole header must be **72 characters or fewer**. Measure it, + aim for 70: + + ```console + printf %s "ci(release): pin node to an lts release for semantic-release" | wc -c + ``` + +## Body + +Write for a reader with no context, such as a student reading the history six +months from now. + +- Start with the problem or motivation: what was wrong or missing, and how it + showed up (for CI, name the failing workflow and its error). +- Then area by area, a short heading paragraph per area, then one bullet per + change, each saying what changed and why. Name files precisely. +- Say what you verified: hooks run, notebooks parsed or run, workflows + checked with a dry run or not at all. +- End with the status: what's unverified, follow-ups. + +## Before writing + +1. Run `git status` and `git diff --stat` (and `git diff` where unclear), so the + message describes the working tree, not your memory. +2. Ask which files belong to the commit if that's ambiguous. +3. Measure the header with `wc -c`. +4. Start the response with a one-line `Written for:` audience note. diff --git a/skills/ai-algorithms-notebooks/SKILL.md b/skills/ai-algorithms-notebooks/SKILL.md new file mode 100644 index 0000000..1cfc4c4 --- /dev/null +++ b/skills/ai-algorithms-notebooks/SKILL.md @@ -0,0 +1,85 @@ +--- +name: ai-algorithms-notebooks +description: Use when reading, editing, adding, reviewing or running the Jupyter notebooks in ai-algorithms (jupyter/*.ipynb - anomaly_detection_creditcard, House_Prices_Regression_Demo, k-means, k-means-it-support-tickets), or when a task touches Kaggle downloads, jupyter/data/, or the Azure ML workspace a notebook uses. Covers which notebooks reach Kaggle or Azure, their kernels, editing cell by cell, outputs and what may be committed, data files, and reviewing a student's notebook pull request. +--- + +# Notebooks + +The notebooks are course material: students read them top to bottom, often in +front of a class. Accuracy of the prose matters as much as the code. + +## What's there + +| Notebook | Kernel | Data source | Reaches | +| -------------------------------------------- | ----------------------------- | ------------------------------------------------------------------ | --------------------------------- | +| `jupyter/k-means.ipynb` | `venv (3.13.x)`, project venv | UCI Iris over HTTPS (first cell also runs a Kaggle download) | UCI, Kaggle | +| `jupyter/k-means-it-support-tickets.ipynb` | `venv (3.13.x)` | Kaggle dataset `tobiasbueck/multilingual-customer-support-tickets` | Kaggle (skips if already local) | +| `jupyter/House_Prices_Regression_Demo.ipynb` | Google Colab (`Python 3`) | Kaggle competition `house-prices-advanced-regression-techniques` | Kaggle, `google.colab` uploads | +| `jupyter/anomaly_detection_creditcard.ipynb` | `venv (3.9.6)`, a legacy venv | Azure ML data asset `creditcard_fraud`, via the v1 SDK | Azure ML (reads, registers model) | + +- The project venv (`ai-algorithms-python-environment`) has pandas, + scikit-learn, scipy, seaborn, shap, kaggle, openpyxl and ipykernel. +- `anomaly_detection_creditcard.ipynb` imports `azureml.core` (Azure ML SDK + v1, retired, Python < 3.12). It doesn't run in the project venv, and its + last steps register a model in the user's workspace. Don't add `azureml-*` + to the requirements to "fix" it; that's a rewrite to the v2 SDK, which + azureml-example already has. Raise it with the user instead. +- `House_Prices_Regression_Demo.ipynb` runs in Colab: it calls + `files.upload()` for `kaggle.json` and reads `jupyter/house_prices/train.csv` + relative to Colab's working directory. +- `jupyter/data/` (house prices, support tickets) holds Kaggle downloads. It + is git-ignored, as is `*.zip`. Kaggle's terms don't allow redistributing + the data: never commit it. + +## Rules + +1. **Don't run cells that reach Kaggle or Azure** (`!kaggle ...`, + `kaggle.api`, `Workspace`, `Dataset.get_by_name`, `Model.register`, + `files.upload()`) without the user's OK. `~/.kaggle/kaggle.json` is the + user's real key; Kaggle downloads are rate limited, and competitions need + their rules accepted on kaggle.com first. `Model.register` writes to a real, + paid Azure workspace. Local pandas, scikit-learn, matplotlib and shap cells + are fine to run in the matching kernel, once the data is on disk. +2. **Don't run `%pip install` cells.** They change the user's venv outside the + requirements files. Install from `requirements/local.txt` instead. +3. **Edit cell by cell** with the notebook editing tool. Don't rewrite or + re-serialize a whole notebook: it churns metadata and makes the diff + unreadable. The tool itself causes churn: it saves an edited cell's + `source` as one string instead of a list of lines, and clears an edited + code cell's `outputs` and `execution_count`. After editing, normalize every + cell's `source` back to a list of lines (`str.splitlines(keepends=True)`) + and write the notebook with `json.dumps(nb, indent=1, ensure_ascii=False)` + plus a trailing newline. Then check that `git diff` shows only the real + changes. Edit kernel metadata as a text replacement in the file. +4. **Outputs.** The two k-means notebooks are committed with outputs (plots a + lesson shows); the other two without. Keep each as it is unless asked. + Before any notebook is committed, check its outputs for anything personal: + home-directory paths (`/Users//...`, as in `k-means.ipynb`'s pip + output today), Kaggle usernames, Azure subscription or workspace details. +5. **Keep the teaching voice.** Markdown cells explain why, step by step + (`## Step N: ...`). Correct wrong statements (metrics, what a plot shows, + which dataset is used), but don't flatten the explanations into terse + comments. +6. **No secrets in cells.** Credentials come from `~/.kaggle/kaggle.json`, + `config.json` or `.env`, never literals. Placeholders look like + `[REPLACE-THIS-WITH-YOUR-USERNAME]`. +7. **Kernels match the venv.** A notebook meant for the project venv names + `venv (3.13.x)` in its kernelspec. When the Python version changes, update + the kernel display names too. + +## Checking a notebook without running it + +```bash +python -m json.tool jupyter/.ipynb > /dev/null # it parses +git diff --stat jupyter/ # only the cells you meant +``` + +pre-commit's `check-json` also parses notebooks. codespell skips `jupyter/` +(`.pre-commit-config.yaml`), and black and isort don't touch `.ipynb`. + +## Reviewing a student's notebook pull request + +Check that it runs in its stated kernel (by reading it, or running local-only +cells), that the prose matches the code and the results, that no outputs leak +anything (rule 4), that no data files, zips or generated HTML were added, and +that it's in `jupyter/`. Report findings; don't push changes to their branch. diff --git a/skills/ai-algorithms-pull-requests/SKILL.md b/skills/ai-algorithms-pull-requests/SKILL.md new file mode 100644 index 0000000..4063607 --- /dev/null +++ b/skills/ai-algorithms-pull-requests/SKILL.md @@ -0,0 +1,85 @@ +--- +name: ai-algorithms-pull-requests +description: Use when asked to create, open, write or update a GitHub pull request in FullStackWithLawrence/ai-algorithms, or to review or describe a contributor's pull request (often a notebook change from a fork, or a Dependabot update). Covers the branches, the title, filling in every section of .github/PULL_REQUEST_TEMPLATE.md from the commits' own messages, creating it with gh pr create --body-file, the checks that run on pull requests, and what never to do (push, merge, leave placeholders in). +--- + +# Pull Requests + +The repository is `FullStackWithLawrence/ai-algorithms`. Every pull request +uses [.github/PULL_REQUEST_TEMPLATE.md](../../.github/PULL_REQUEST_TEMPLATE.md). +`gh pr create --body-file` doesn't pre-fill it, so fill in every section from +the commits themselves. + +## Rules + +- **Create one only when asked.** A pull request is public and notifies + people. Writing the description as text is fine when that's all the user + asked for. +- **Never push, merge, or change a branch.** If the head branch isn't on + GitHub, or is behind the local one, say so and stop. +- **Never leave a placeholder** (`Fixes #[Add issue number here.]`, + `_Describe what this Pull Request does_`). Each section gets content or a + plain "None.". +- **The repository is public.** No keys, subscription ids or personal details. +- **No bare `@name`** unless you mean to notify that GitHub user. +- **No CI skip instruction** anywhere (see `ai-algorithms-git-commits`). + +## Branches and title + +- Work lands on `main` (the default) and every merge to `main` can release. + `alpha` is where the maintainer works and where the weekly pre-commit + version bumps land; `beta` and `next` are semantic-release branches too. + Ask which base the user wants if it isn't obvious. +- The title is the main commit's subject, a conventional commit. +- Check for an open one first: + `gh pr list --repo FullStackWithLawrence/ai-algorithms --head `. + If there is one, update it with `gh pr edit --body-file ...`. + +## Gather the material + +```bash +git fetch origin +git status -sb +git log --oneline origin/main.. +git log --format='### %h %s%n%n%b' origin/main.. +git diff --stat origin/main... | tail -1 +``` + +Leave out `chore(release)` and merge commits. Read every commit body. + +## Fill in the template + +Keep its headings and order, drop the `` hints. + +- **Type of Change:** check every box a commit earns: `feat` = New feature, + `fix` = Bug fix, `docs` (and notebook prose) = Documentation, `refactor` = + Refactor, `ci`/`build`/`chore`/`test` = Chore. +- **Resolves:** one `- Fixes #` per issue a commit names, or `- None.` +- **Changes:** a sentence or two on the whole, then a `###` subsection per + commit (subject and short hash) with the problem and bullets naming files. +- **Testing:** what ran and how: pre-commit hooks, notebooks run by hand (which + cells, which kernel), workflows observed on GitHub. Never claim a check + nobody ran. +- **Screenshots:** for notebook plots; otherwise `None: no visible change.` +- **Dependencies:** changed pins in `requirements/`, new secrets (CI uses + `PAT`), new datasets a reader must download. Otherwise `None.` +- **Breaking Changes:** renamed or removed notebooks, a new Python version, + removed make targets. Otherwise `None.` + +End with the attribution line from the conversation's system context, if any. + +## What runs on a pull request + +- `test.yml` (`pull_request`): job `pre-commit` runs every hook on every file. +- `auto-assign.yml`; Mergify approves and merges Dependabot pull requests into + `main`. + +## Create it + +```bash +gh pr create --repo FullStackWithLawrence/ai-algorithms --base main --head \ + --title "(): " --body-file /pr-body.md +``` + +Write the body to a file in your scratchpad first. Give the user the url, and +say what you couldn't verify. diff --git a/skills/ai-algorithms-python-environment/SKILL.md b/skills/ai-algorithms-python-environment/SKILL.md new file mode 100644 index 0000000..0012c90 --- /dev/null +++ b/skills/ai-algorithms-python-environment/SKILL.md @@ -0,0 +1,70 @@ +--- +name: ai-algorithms-python-environment +description: Use when ai-algorithms' notebooks or tools must run locally, when the virtual environment is missing or broken (e.g. ModuleNotFoundError in a notebook kernel), or when changing the Python version, requirements (requirements/in/*.in and the compiled requirements/*.txt), Dependabot's pip updates, or Makefile targets. Covers the Python 3.13 target, the venv folder that is also the notebooks' kernel, compiling requirements with pip-compile, and the Makefile's destructive targets. +--- + +# Python Environment + +## Python 3.13 + +The repository runs on Python 3.13: `PYTHON := python3.13` in the Makefile, +`default_language_version: python: python3.13` in `.pre-commit-config.yaml`, +`requires-python = ">=3.13"` and black's `py313` in `pyproject.toml`, +`python-version: "3.13"` in the workflows, and `venv (3.13.x)` in the notebook +kernels. Change them together, or not at all. (azureml-example has moved to +3.14; moving this repository too is a deliberate change of its own, after +checking that every package in `base.in` has 3.14 wheels.) + +Before adding any package, check PyPI's `requires_python` and wheels for 3.13 +(`curl -s https://pypi.org/pypi//json`). + +## The folders on disk + +| Folder | What it is | +| --------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `venv/` | The project venv from `make init`: Python 3.13 + `requirements/local.txt`. It is also the Jupyter kernel `venv (3.13.x)` | +| `node_modules/` | semantic-release, Prettier (from `npm install`) | + +If `venv/` is missing or broken, don't rebuild it with `make init` unasked +(see below). `pip install -r requirements/local.txt` into the existing venv is +fine. Report which environment you actually ran in. + +## Requirements files + +The `.txt` files are compiled from `requirements/in/*.in` with pip-compile +(from `pip-tools`, in `local.in`), on Python 3.13. + +| File | Contents | +| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------- | +| `requirements/in/constraints.in` | Global version constraints, included by every `.in` file. None today | +| `requirements/in/base.in` | What the notebooks import: pandas, scikit-learn, scipy, seaborn, shap, kaggle, openpyxl, ipykernel, python-dotenv | +| `requirements/in/local.in` | `base.in` + pip-tools, pre-commit and the linters | +| `requirements/base.txt`, `local.txt` | The compiled, fully pinned output. Don't edit by hand. `make init` installs `local.txt` | + +- `.in` files never carry version pins (`==`): list the bare package with a + short trailing comment saying what it is for, and let pip-compile pin it. + A range goes in `constraints.in` only for a known incompatibility, with a + comment naming the package that needs it. Then recompile with `make python-requirements`, or by hand + `pip-compile --strip-extras requirements/in/base.in -o requirements/base.txt` + (and `local`). Add `--upgrade` only when upgrading is the task. When an old + pin in a `.txt` file blocks the resolve, upgrade just that package + (`pip-compile -P ...`) rather than pinning in a `.in` file. +- A package a notebook imports goes in `base.in`; a dev tool in `local.in`. +- Dependabot watches `requirements/` (the compiled `.txt` files), so its pull + requests change real pins. +- `release.config.js` commits `requirements/**/*` on release; don't fight it. + +## Makefile + +| Target | Effect | +| -------------------------- | ----------------------------------------------------------------------------------------- | +| `make` (any target) | Creates `.env` from `.env.example` if it's missing | +| `make init`, `python-init` | **`python-init` runs `python-clean` first: deletes `venv/`**. Ask first | +| `make python-requirements` | Compiles `base.in` and `local.in` into `requirements/*.txt` | +| `make lint` | Runs every pre-commit hook on every file | +| `make pre-commit` | `pre-commit install`, **`pre-commit autoupdate`** (edits the config), then runs the hooks | +| `make analyze` | `cloc` report | +| `make release` | Pushes an empty commit to force a release. Never run it | + +The host shell is zsh. It doesn't word-split variables, so pass file lists with +`xargs` or an array, not `$FILES`.