diff --git a/docker/requirements-full.txt b/docker/requirements-full.txt index 68e332d..5340f2c 100644 --- a/docker/requirements-full.txt +++ b/docker/requirements-full.txt @@ -10,6 +10,7 @@ numpy==2.5.4 openpyxl==3.1.5 pandas==3.1.0 Pillow==13.0.0 +plot-serializer==0.2.3 PuLP==4.2.0 Pygments==2.21.0 pytest==9.2.0 diff --git a/docker/run.sh b/docker/run.sh index 2fbc49a..9d7aa4a 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -10,14 +10,15 @@ if [ ! -d /grade ]; then fi # the parent directory containing everything about this grading job -export JOB_DIR='/grade' +JOB_DIR='/grade' # the job subdirectories STUDENT_DIR=$JOB_DIR'/student' TEST_DIR=$JOB_DIR'/tests' OUT_DIR=$JOB_DIR'/results' +DATA_DIR=$JOB_DIR'/data' # where we will copy everything -export MERGE_DIR=$JOB_DIR'/run' +MERGE_DIR=$JOB_DIR'/run' # now set up the stuff so that our run.sh can work mkdir $MERGE_DIR @@ -33,6 +34,21 @@ mv "$STUDENT_DIR" "$MERGE_DIR/test_student/student" mv $TEST_DIR/* $MERGE_DIR/test_student/ mv $MERGE_DIR/test_student/test_student.py $MERGE_DIR/test_student.py +# PrairieLearn writes data.json to the data directory, and pytest is told to read it from there. +# It can hold correct_answers, so it's for the grader only. It must be owned by root with +# owner-only modes: student code keeps root's supplementary groups, and its uid may match the owner +# of files mounted from the host. +protect_data_json() { + chown -R root:root "$DATA_DIR" || return 1 + chmod 700 "$DATA_DIR" && chmod 600 "$DATA_DIR/data.json" +} + +if [ -f "$DATA_DIR/data.json" ] && ! protect_data_json; then + echo '{"succeeded": false, "score": 0.0, "message": "The autograder could not protect its grading data. Please contact course staff and have them check the logs for this submission."}' > "$OUT_DIR/results.json" + echo "ERROR: could not protect $DATA_DIR/data.json" + exit 1 +fi + # Do not allow ag user to modify, rename, or delete any existing files #chmod -R 755 "$MERGE_DIR" #chmod 1777 "$MERGE_DIR" @@ -45,7 +61,9 @@ echo "[run] starting autograder" # run the autograder as a limited user called ag # Use --worker-username to drop privileges to the 'ag' user in the student code sandbox -uv run pytest -p prairielearn-grader --color=no --log-cli-level=DEBUG --worker-username ag "$MERGE_DIR" +# Pass plugin options with '=': pytest's first pass over the arguments, before plugin options are +# registered, would treat a separate value as a test path. +uv run pytest -p prairielearn-grader --color=no --log-cli-level=DEBUG --worker-username=ag --data-json-path="$DATA_DIR/data.json" "$MERGE_DIR" # TODO change the default output name mv "$MERGE_DIR/autograder_results.json" "$OUT_DIR/results.json" diff --git a/quick_start.md b/quick_start.md index d045a30..d4e6d29 100644 --- a/quick_start.md +++ b/quick_start.md @@ -264,7 +264,7 @@ result = sandbox.query_function("slow_computation", data, query_timeout=2.0) **Note**: Student code must define the queried symbols, and return values must be JSON-serializable. Supported types include: `int`, `float`, `str`, `list`, `dict`, `bool`, `None`, numpy arrays, pandas DataFrames, -and matplotlib figures. +and plots built with `MatplotlibSerializer` (see [Testing Matplotlib Plots](#testing-matplotlib-plots)). ## Partial Credit and Feedback @@ -603,6 +603,8 @@ def generate(data): These parameters are automatically passed to the autograder via `/grade/data/data.json` and enforced during student code execution. +In the grader images, `run.sh` passes this path to pytest with `--data-json-path` and makes the file readable only by root, so student code can't open it. The file can contain `correct_answers`. Without that option, for example when running locally, the fixture reads `data.json` next to your test file. The sandbox still receives `params` through `__data_params`, so don't rely on values in `params` staying hidden from student code. + #### Testing Import/Builtin Restrictions Locally To test your restrictions in the test scenarios: @@ -950,9 +952,27 @@ since students can learn from reading full tracebacks. ### Testing Matplotlib Plots -The grader supports automatic serialization and deserialization of matplotlib figures: +To grade a plot, student code builds it with `MatplotlibSerializer` from the `plot-serializer` package and returns the serializer. The test receives a matplotlib `Figure`. + +`plot-serializer` is included in the `:latest` image but not in `:lite`. Returning a plain matplotlib `Figure` from student code isn't supported. + +```python +# Student code +from plot_serializer.matplotlib.serializer import MatplotlibSerializer + + +def create_plot(data): + serializer = MatplotlibSerializer() + fig, ax = serializer.subplots() + ax.plot(data) + ax.set_xlabel("Time") + ax.set_ylabel("Value") + ax.set_title("Data Visualization") + return serializer +``` ```python +# Test code import matplotlib matplotlib.use("Agg") # Non-GUI backend for server environments @@ -965,7 +985,7 @@ from pytest_prairielearn_grader.fixture import StudentFixture @pytest.mark.grading_data(name="Plot Test", points=5) def test_student_plot(sandbox: StudentFixture) -> None: """Test that student creates a correct plot.""" - # Student function returns a matplotlib figure + # Student function returns a MatplotlibSerializer, which arrives here as a Figure plot = sandbox.query_function("create_plot", data) assert isinstance(plot, Figure) @@ -984,7 +1004,7 @@ def test_student_plot(sandbox: StudentFixture) -> None: The grader automatically serializes/deserializes: -- Matplotlib figures +- Plots built with `MatplotlibSerializer` (received as a `Figure`) - NumPy arrays - Pandas DataFrames - Standard Python types (int, float, str, list, dict, bool, None) @@ -1019,7 +1039,7 @@ Two image variants are published. Pick the smallest one that includes what your | Tag | Size (approx.) | Includes | Best for | | --------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| `:latest` | ~400–500 MB | Full scientific stack: `numpy`, `pandas`, `scipy`, `matplotlib` + `matplotcheck2`, `seaborn`, `scikit-learn`, `sympy`, `networkx`, `nltk`, `nbformat`, and more | Any course that needs scientific computing, data science, plotting, or Jupyter notebook grading | +| `:latest` | ~400–500 MB | Full scientific stack: `numpy`, `pandas`, `scipy`, `matplotlib` + `matplotcheck2` + `plot-serializer`, `seaborn`, `scikit-learn`, `sympy`, `networkx`, `nltk`, `nbformat`, and more | Any course that needs scientific computing, data science, plotting, or Jupyter notebook grading | | `:lite` | ~60–80 MB | Grader core only: `pytest-prairielearn-grader`, `Pygments`, `beautifulsoup4`, `defusedxml`, `requests` | Introductory CS courses where student code uses only the Python standard library | To use the lite image, change the `image` field: diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 00b15c5..81b96da 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -67,19 +67,30 @@ def get_datadir(test_module: ModuleType) -> Path | None: return data_dir +def _find_data_json(request: pytest.FixtureRequest) -> Path | None: + """Prefer the file named by --data-json-path, and otherwise use data.json next to the test file.""" + data_json_path = request.config.getoption("--data-json-path") + if data_json_path is not None and Path(data_json_path).is_file(): + return Path(data_json_path) + + datadir = get_datadir(request.module) + return None if datadir is None else datadir / "data.json" + + @pytest.fixture(scope="module") def data_json(request: pytest.FixtureRequest) -> dict[str, Any] | None: try: - datadir = get_datadir(request.module) - assert datadir is not None - data_file = datadir / "data.json" + data_file = _find_data_json(request) + if data_file is None: + logger.debug("No location to load data.json from") + return None logger.debug(f"Loading data.json from {data_file}") return json.loads(data_file.read_text(encoding="utf-8")) - except Exception as e: - logger.debug(f"No data.json found or failed to load: {e}") - - # If the data file is not found or cannot be read, return None - return None + except FileNotFoundError: + logger.debug("No data.json found") + return None + except (OSError, ValueError) as e: + raise RuntimeError(f"Could not load data.json: {e}") from e # if datadir is None or not datadir.is_dir(): # raise ValueError(f"Data directory '{datadir}' not found or is not a directory.") @@ -677,6 +688,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: """ group = parser.getgroup("pytest_pl_autograder", "Prairielearn Python Autograder Options") + group.addoption( + "--data-json-path", + action="store", + default=None, + help="Path to the data.json file for this grading job. If it doesn't exist, data.json next to the test file is used.", + ) group.addoption( "--worker-username", action="store", diff --git a/tests/scenario_root/test_invalid_data_json/__init__.py b/tests/scenario_root/test_invalid_data_json/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_invalid_data_json/data.json b/tests/scenario_root/test_invalid_data_json/data.json new file mode 100644 index 0000000..42e2ff7 --- /dev/null +++ b/tests/scenario_root/test_invalid_data_json/data.json @@ -0,0 +1 @@ +{"params": diff --git a/tests/scenario_root/test_invalid_data_json/expected_outcome.json b/tests/scenario_root/test_invalid_data_json/expected_outcome.json new file mode 100644 index 0000000..2762715 --- /dev/null +++ b/tests/scenario_root/test_invalid_data_json/expected_outcome.json @@ -0,0 +1,15 @@ +{ + "expected_data_object": { + "score": 0.0, + "tests": [ + { + "max_points": 1, + "points_frac": 0.0, + "points": 0.0, + "test_id": "test_invalid_data_json.py::test_query[student_code]", + "outcome": "error", + "message": "Could not load data.json: Expecting value" + } + ] + } +} diff --git a/tests/scenario_root/test_invalid_data_json/scenario.py b/tests/scenario_root/test_invalid_data_json/scenario.py new file mode 100644 index 0000000..1950ffa --- /dev/null +++ b/tests/scenario_root/test_invalid_data_json/scenario.py @@ -0,0 +1,5 @@ +from pytest_prairielearn_grader.fixture import StudentFixture + + +def test_query(sandbox: StudentFixture) -> None: + assert sandbox.query("ANSWER") == 42 diff --git a/tests/scenario_root/test_invalid_data_json/student_code.py b/tests/scenario_root/test_invalid_data_json/student_code.py new file mode 100644 index 0000000..b0f8b2d --- /dev/null +++ b/tests/scenario_root/test_invalid_data_json/student_code.py @@ -0,0 +1 @@ +ANSWER = 42