diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py index 0f7bab799..e053e533b 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py @@ -663,7 +663,16 @@ async def _process_terminating_job( result.job_update_map["status"] = _get_job_termination_status(job_model) return result - if job_model.remove_at is None: + if _is_aborted(job_model): + # An aborted job is not asked to stop, so it keeps running and the runner never runs out + # of logs to hand over. Collect what it has buffered so far, but do not wait for more. + if _has_logs_to_drain(job_model): + await _drain_job_logs( + job_model=job_model, + instance_model=instance_model, + job_update_map=result.job_update_map, + ) + elif job_model.remove_at is None: # The first terminating pass. `graceful_termination_attempts` decides whether the runner # is asked to stop the job; `remove_at` decides when the container is killed regardless. # The two are independent: a job that has already finished on its own is not asked to @@ -741,6 +750,19 @@ async def _process_terminating_job( return result +def _is_aborted(job_model: JobModel) -> bool: + """ + Whether the job is to be terminated without waiting for it to finish. + + The run's reason is checked too: the run pipeline does not update jobs that are already + terminating, so a job of a run aborted after it was stopped keeps its graceful reason. + """ + return ( + job_model.termination_reason == JobTerminationReason.ABORTED_BY_USER + or job_model.run.termination_reason == RunTerminationReason.ABORTED_BY_USER + ) + + def _has_logs_to_drain(job_model: JobModel) -> bool: """ Whether the runner may still be holding logs for this job. diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py index 5a64fd62a..081f33385 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py @@ -12,7 +12,12 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import TaskConfiguration from dstack._internal.core.models.instances import InstanceStatus -from dstack._internal.core.models.runs import JobStatus, JobTerminationReason +from dstack._internal.core.models.runs import ( + JobStatus, + JobTerminationReason, + RunStatus, + RunTerminationReason, +) from dstack._internal.core.models.volumes import VolumeStatus from dstack._internal.core.services.ssh.tunnel import SSHTunnel from dstack._internal.server.background.pipeline_tasks.jobs_terminating import ( @@ -573,6 +578,92 @@ async def test_collects_last_logs_before_terminating_container( # Advanced, so the next pass does not collect the same logs again assert job.runner_timestamp == 3 + @pytest.mark.parametrize( + ("stopped_before_abort", "expected_status"), + [ + pytest.param(False, JobStatus.ABORTED, id="aborted"), + # The run pipeline leaves jobs that are already terminating alone, so the job keeps + # its graceful reason and is already waiting for the runner to hand over its logs + pytest.param(True, JobStatus.TERMINATED, id="aborted_after_stop"), + ], + ) + async def test_does_not_wait_for_logs_of_aborted_job( + self, + test_db, + session: AsyncSession, + worker: JobTerminatingWorker, + ssh_tunnel_mock: Mock, + runner_client_mock: Mock, + stopped_before_abort: bool, + expected_status: JobStatus, + ): + """ + An aborted job is not asked to stop, so the runner never runs out of logs. What it has + buffered is collected, but the container is stopped without waiting for more. + """ + project = await create_project(session=session) + user = await create_user(session=session) + instance = await create_instance( + session=session, + project=project, + status=InstanceStatus.BUSY, + ) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + status=RunStatus.TERMINATING, + termination_reason=RunTerminationReason.ABORTED_BY_USER, + ) + job = await create_job( + session=session, + run=run, + status=JobStatus.TERMINATING, + termination_reason=( + JobTerminationReason.TERMINATED_BY_USER + if stopped_before_abort + else JobTerminationReason.ABORTED_BY_USER + ), + job_provisioning_data=get_job_provisioning_data(dockerized=True), + instance=instance, + running_at=get_current_datetime() - timedelta(minutes=1), + ) + if stopped_before_abort: + job.graceful_termination_attempts = 1 + job.remove_at = get_current_datetime() + timedelta(seconds=30) + job.runner_timestamp = 1 + _lock_job(job) + await session.commit() + + runner_client_mock.pull.return_value = PullResponse( + job_states=[], + job_logs=[LogEvent(timestamp=2, message=b"the tail")], + runner_logs=[], + last_updated=2, + has_more=True, + ) + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.logs_services.write_logs" + ) as write_logs, + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating._stop_container", + new=AsyncMock(return_value=True), + ) as stop_container, + ): + await worker.process(_job_to_pipeline_item(job)) + + runner_client_mock.stop.assert_not_called() + runner_client_mock.pull.assert_called_once_with(job.id, 1) + write_logs.assert_called_once() + stop_container.assert_awaited_once() + + await session.refresh(job) + assert job.status == expected_status + assert job.runner_timestamp == 2 + async def test_does_not_wait_for_logs_if_runner_no_longer_runs_job( self, test_db,