Report missing student functions as NOT_FOUND

quick_start.md

@@ -171,6 +171,7 @@ Calls a function defined in the student code with the given arguments and **retu
 
 - **On success**: Returns the function's return value
 - **On error**: Raises a `RuntimeError` with details about the exception
+- **If the function doesn't exist**: Raises a `NameError` saying so
 
 **Example:**
 
@@ -228,7 +229,7 @@ def test_raises_value_error(sandbox: StudentFixture, feedback: FeedbackFixture)
 
 The `query_function_raw` response contains:
 
-- `status`: One of `"success"`, `"exception"`, `"timeout"`, `"not_found"`, or `"not_serializable"`. `"not_serializable"` means the function finished, but its return value can't be sent back to the grader. `exception_message` explains this, and later queries on the sandbox still work.
+- `status`: One of `"success"`, `"exception"`, `"timeout"`, `"not_found"`, or `"not_serializable"`. `"not_found"` means the function doesn't exist, and only the lookup can cause it: an error raised while the function runs, even a `KeyError` or `ImportError`, is `"exception"`. `"not_serializable"` means the function finished, but its return value can't be sent back to the grader. `exception_message` explains this, and later queries on the sandbox still work.
 - `value`: The return value (when status is `"success"`)
 - `stdout` / `stderr`: Captured output from the function call
 - `exception_name`: The exception class name (e.g., `"ValueError"`)

src/pytest_prairielearn_grader/_student_code_runner.py

@@ -90,6 +90,36 @@ def _resolve_dotted_name(name: str, student_code_vars: dict[str, Any]) -> Any:
     return student_code_vars[name]
 
 
+class _FunctionNotFoundError(Exception):
+    """The requested function doesn't exist. Only the lookup raises this, never the student's call."""
+
+
+def _look_up_function(name: str, student_code_vars: dict[str, Any], workspace_mode: bool) -> Any:
+    """
+    Find a student function by name, raising _FunctionNotFoundError if it doesn't exist.
+
+    In workspace mode, importing a dotted name's module runs student code. If that code fails, the
+    error isn't caught here unless the failure is that the requested module itself doesn't exist,
+    so a student module that exists but has a bad import is reported as an exception.
+    """
+    not_found = _FunctionNotFoundError(f"Function '{name}' wasn't found in the student code")
+    if workspace_mode and "." in name:
+        module_path, attr_name = name.rsplit(".", 1)
+        try:
+            module = importlib.import_module(module_path)
+        except ModuleNotFoundError as e:
+            if e.name is not None and (module_path == e.name or module_path.startswith(f"{e.name}.")):
+                raise not_found from e
+            raise
+        try:
+            return getattr(module, attr_name)
+        except AttributeError as e:
+            raise not_found from e
+    if name not in student_code_vars:
+        raise not_found
+    return student_code_vars[name]
+
+
 async def student_function_runner(
     student_code_vars: dict[str, Any],
     func_name: str,
@@ -105,14 +135,12 @@ async def student_function_runner(
     exception_traceback: str | None = None
     result = None
     timed_out = False
+    function_not_found = False
 
     try:
 
         def student_function_temp() -> Any:
-            if workspace_mode:
-                student_function = _resolve_dotted_name(func_name, student_code_vars)
-            else:
-                student_function = student_code_vars[func_name]
+            student_function = _look_up_function(func_name, student_code_vars, workspace_mode)
             return student_function(*args_tup, **kwargs_dict)
 
         with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture):
@@ -123,6 +151,9 @@ def student_function_temp() -> Any:
             else:
                 execution_error = asyncio.TimeoutError(f"Function timed out after {timeout} seconds")
                 timed_out = True
+    except _FunctionNotFoundError as e:
+        execution_error = NameError(str(e))
+        function_not_found = True
     except Exception as e:
         execution_error = e
         exception_traceback = traceback.format_exc(limit=-1)
@@ -133,6 +164,8 @@ def student_function_temp() -> Any:
             if execution_error is None
             else FunctionStatusCode.TIMEOUT
             if timed_out
+            else FunctionStatusCode.NOT_FOUND
+            if function_not_found
             else FunctionStatusCode.EXCEPTION
         ),
         "value": None,

src/pytest_prairielearn_grader/fixture.py

@@ -320,7 +320,7 @@ def query_function(self, function_name: str, *args: Any, query_timeout: float =
             case FunctionStatusCode.TIMEOUT:
                 raise TimeoutError(f"Query for function '{function_name}' timed out after {query_timeout} seconds.")
             case FunctionStatusCode.NOT_FOUND:
-                raise NameError(f"Query for function '{function_name}' failed: {response['exception_message']}")
+                raise NameError(response["exception_message"])
             case FunctionStatusCode.NOT_SERIALIZABLE:
                 raise RuntimeError(response["exception_message"])
         return from_json(response["value"])

tests/scenario_root/test_function_not_found/expected_outcome.json

@@ -0,0 +1,105 @@
+{
+  "expected_data_object": {
+    "score": 0.8333333333333334,
+    "tests": [
+      {
+        "max_points": 1,
+        "name": "raw_response",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_raw_response_has_all_fields[student_code]"
+      },
+      {
+        "max_points": 1,
+        "name": "query_function",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_query_function_raises_clear_error[student_code]"
+      },
+      {
+        "max_points": 1,
+        "name": "student_errors",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_errors_inside_function_stay_exceptions[student_code-raises_key_error-KeyError-'missing_key']"
+      },
+      {
+        "max_points": 1,
+        "name": "student_errors",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_errors_inside_function_stay_exceptions[student_code-raises_key_error_with_function_name-KeyError-'missing_function']"
+      },
+      {
+        "max_points": 1,
+        "name": "student_errors",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_errors_inside_function_stay_exceptions[student_code-raises_attribute_error-AttributeError-missing_attribute]"
+      },
+      {
+        "max_points": 1,
+        "name": "student_errors",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_errors_inside_function_stay_exceptions[student_code-raises_import_error-ImportError-student import problem]"
+      },
+      {
+        "max_points": 1,
+        "name": "student_errors",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_errors_inside_function_stay_exceptions[student_code-raises_name_error-NameError-undefined_variable]"
+      },
+      {
+        "max_points": 1,
+        "name": "output_kept",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_output_kept_for_student_errors[student_code]"
+      },
+      {
+        "max_points": 1,
+        "name": "not_callable",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_name_that_is_not_callable_stays_an_exception[student_code]"
+      },
+      {
+        "max_points": 1,
+        "name": "later_queries",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "outcome": "passed",
+        "test_id": "test_function_not_found.py::test_later_queries_still_work[student_code]"
+      },
+      {
+        "max_points": 1,
+        "name": "assert_fn_equal",
+        "points": 0.0,
+        "points_frac": 0.0,
+        "outcome": "failed",
+        "test_id": "test_function_not_found.py::test_assert_fn_equal_shows_clear_message[student_code]",
+        "message": "Function 'missing_function' wasn't found in the student code"
+      },
+      {
+        "max_points": 1,
+        "name": "assert_fn_approx_equal",
+        "points": 0.0,
+        "points_frac": 0.0,
+        "outcome": "failed",
+        "test_id": "test_function_not_found.py::test_assert_fn_approx_equal_shows_clear_message[student_code]",
+        "message": "Function 'missing_function' wasn't found in the student code"
+      }
+    ]
+  }
+}

tests/scenario_root/test_function_not_found/scenario.py

@@ -0,0 +1,82 @@
+import pytest
+
+from pytest_prairielearn_grader.assertions import assert_fn_approx_equal
+from pytest_prairielearn_grader.assertions import assert_fn_equal
+from pytest_prairielearn_grader.fixture import StudentFixture
+from pytest_prairielearn_grader.utils import FunctionStatusCode
+from pytest_prairielearn_grader.utils import GradingOutputLevel
+
+EXPECTED_MESSAGE = "Function 'missing_function' wasn't found in the student code"
+
+
+@pytest.mark.grading_data(name="raw_response", points=1)
+def test_raw_response_has_all_fields(sandbox: StudentFixture) -> None:
+    assert sandbox.query_function_raw("missing_function") == {
+        "status": FunctionStatusCode.NOT_FOUND,
+        "value": None,
+        "stdout": "",
+        "stderr": "",
+        "exception_name": "NameError",
+        "exception_message": EXPECTED_MESSAGE,
+        "traceback": None,
+    }
+
+
+@pytest.mark.grading_data(name="query_function", points=1)
+def test_query_function_raises_clear_error(sandbox: StudentFixture) -> None:
+    with pytest.raises(NameError) as error:
+        sandbox.query_function("missing_function")
+    assert str(error.value) == EXPECTED_MESSAGE
+
+
+@pytest.mark.grading_data(name="student_errors", points=1)
+@pytest.mark.parametrize(
+    ("function_name", "exception_name", "message"),
+    [
+        ("raises_key_error", "KeyError", "'missing_key'"),
+        ("raises_key_error_with_function_name", "KeyError", "'missing_function'"),
+        ("raises_attribute_error", "AttributeError", "missing_attribute"),
+        ("raises_import_error", "ImportError", "student import problem"),
+        ("raises_name_error", "NameError", "undefined_variable"),
+    ],
+)
+def test_errors_inside_function_stay_exceptions(sandbox: StudentFixture, function_name: str, exception_name: str, message: str) -> None:
+    response = sandbox.query_function_raw(function_name)
+
+    assert response["status"] == FunctionStatusCode.EXCEPTION
+    assert response["exception_name"] == exception_name
+    assert response["exception_message"] is not None
+    assert message in response["exception_message"]
+    assert response["traceback"] is not None
+    assert "student_code" in response["traceback"]
+
+
+@pytest.mark.grading_data(name="output_kept", points=1)
+def test_output_kept_for_student_errors(sandbox: StudentFixture) -> None:
+    assert sandbox.query_function_raw("raises_key_error")["stdout"] == "before the error\n"
+
+
+@pytest.mark.grading_data(name="not_callable", points=1)
+def test_name_that_is_not_callable_stays_an_exception(sandbox: StudentFixture) -> None:
+    response = sandbox.query_function_raw("not_callable")
+
+    assert response["status"] == FunctionStatusCode.EXCEPTION
+    assert response["exception_name"] == "TypeError"
+
+
+@pytest.mark.grading_data(name="later_queries", points=1)
+def test_later_queries_still_work(sandbox: StudentFixture) -> None:
+    assert sandbox.query_function_raw("missing_function")["status"] == FunctionStatusCode.NOT_FOUND
+    assert sandbox.query_function("add_one", 1) == 2
+
+
+@pytest.mark.output(level=GradingOutputLevel.Friendly)
+@pytest.mark.grading_data(name="assert_fn_equal", points=1)
+def test_assert_fn_equal_shows_clear_message(sandbox: StudentFixture) -> None:
+    assert_fn_equal(sandbox, "missing_function", args=(1,), expected=2)
+
+
+@pytest.mark.output(level=GradingOutputLevel.Friendly)
+@pytest.mark.grading_data(name="assert_fn_approx_equal", points=1)
+def test_assert_fn_approx_equal_shows_clear_message(sandbox: StudentFixture) -> None:
+    assert_fn_approx_equal(sandbox, "missing_function", args=(1,), expected=2.0)

tests/scenario_root/test_function_not_found/student_code.py

@@ -0,0 +1,26 @@
+not_callable = 5
+
+
+def add_one(x):
+    return x + 1
+
+
+def raises_key_error():
+    print("before the error")
+    return {}["missing_key"]
+
+
+def raises_key_error_with_function_name():
+    raise KeyError("missing_function")
+
+
+def raises_attribute_error():
+    return add_one.missing_attribute
+
+
+def raises_import_error():
+    raise ImportError("student import problem")
+
+
+def raises_name_error():
+    return undefined_variable  # noqa: F821

tests/scenario_root/test_quiz_2_1/expected_outcome.json

@@ -4,7 +4,7 @@
     "tests": [
       {
         "max_points": 6,
-        "message": "Function 'analyze_vibration' raised an exception",
+        "message": "Function 'analyze_vibration' wasn't found in the student code",
         "name": "Test analyze_vibration function",
         "outcome": "failed",
         "points_frac": 0.0,
@@ -13,7 +13,7 @@
       },
       {
         "max_points": 6,
-        "message": "Function 'analyze_vibration' raised an exception",
+        "message": "Function 'analyze_vibration' wasn't found in the student code",
         "name": "Test analyze_vibration function",
         "outcome": "failed",
         "points_frac": 0.0,

tests/scenario_root/test_stdout_feature/expected_outcome.json

@@ -60,7 +60,7 @@
       },
       {
         "test_id": "test_stdout_feature.py::test_global_scope_stdout[student_code]",
-        "message": "Student code grading failed with an exception: RuntimeError\nException Message: Function 'get_global_message' raised an exception KeyError: 'get_global_message'",
+        "message": "Student code grading failed with an exception: NameError\nException Message: Function 'get_global_message' wasn't found in the student code",
         "max_points": 2,
         "points_frac": 0.0,
         "points": 0.0,