diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index b01c63f62b..a3d5638c45 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -33,8 +33,8 @@ trials: 5 min_context_length: 1048576 max_ttft: 675 -# The number of simultaneous requests every benchmark uses -concurrency: 1 +# A list of simultaneous request counts every benchmark uses. +concurrency: [4, 8, 16] # The request shape every benchmark uses (defaults to 1024 and 1024) input_tokens: 10000 @@ -55,7 +55,7 @@ To create the preset, pass the configuration to the `dstack apply` command: $ dstack apply -f preset.dstack.yml --fleet b200-fleet Create the preset dsv4-flash? [y/n]: y [2026-08-04 11:38:34] Starting preset creation for deepseek-ai/DeepSeek-V4-Flash. Allowed fleets: b200-fleet. -[2026-08-04 12:31:19] Trial 3 switched from vLLM to SGLang: 319 tok/s per user, 2.2x the baseline. +[2026-08-04 12:31:19] Trial 3 switched from vLLM to SGLang: 319/495/570 tok/s per user at concurrency 4/8/16, 2.1x/1.9x/1.7x the baseline. [2026-08-04 13:04:52] Final service dsv4-flash-c83375b4-4 verified with context length 1048576. [2026-08-04 13:12:07] Benchmark via sglang.bench_serving: 32/32 requests succeeded. ``` @@ -255,9 +255,9 @@ A pulled preset works like any other, and is named `/`: ```shell $ dstack preset list -a - NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED - main/dsv4-flash-b200 8f065dde deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tps/user=309 ttft=213ms ctx=1M pulled 2 min ago - qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tps/user=19.6 ttft=3.43s ctx=32K verified (7) 3 days ago + NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED + main/dsv4-flash-b200 8f065dde deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=4..16 tps/user=319..570 ttft=213ms..465ms ctx=1M pulled 2 min ago + qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tps/user=19.6 ttft=3.43s ctx=32K verified (7) 3 days ago ``` @@ -336,8 +336,8 @@ Use `dstack preset` to list presets: ```shell $ dstack preset list - NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED - dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tps/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED + dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=4..16 tps/user=319..570 ttft=213ms..465ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago ``` @@ -348,20 +348,36 @@ By default, `dstack preset` shows creations that are still running, or the most ```shell $ dstack preset list -a - NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED - dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tps/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago - qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tps/user=19.6 ttft=3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago - qwen36-pro4500 9ab0fa65 Qwen/Qwen3.6-27B io=1K/1K c=8 tps/user=57.1 ttft=499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago - qwen3-32b-5090 f91d6b60 Qwen/Qwen3-32B io=1K/512 c=8 tps/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago + NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED + dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=4..16 tps/user=319..570 ttft=213ms..465ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=16..64 tps/user=19.6..41.2 ttft=1.21s..3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago + qwen36-pro4500 9ab0fa65 Qwen/Qwen3.6-27B io=1K/1K c=2..8 tps/user=57.1..94.3 ttft=312ms..499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago + qwen3-32b-5090 f91d6b60 Qwen/Qwen3-32B io=1K/512 c=8 tps/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago ``` The `CONSTRAINTS` column is what the creation was asked for, and `BENCHMARK` is the best trial so far. `tps/user` is the steady decode rate, measured as one second divided by the mean time per output token, so it excludes the time to the first token. -The glyphs after the benchmark are one per trial: height is throughput, a yellow bar is a trial whose benchmark broke a constraint, and a red `·` is one that produced no benchmark at all. The shape shows whether a run converged or wandered. +When `concurrency` is a list, `CONSTRAINTS` shows it as `c=min..max`, and the `BENCHMARK` metrics become ranges over the concurrencies that met the constraints, e.g. `tps/user=min..max` and `ttft=min..max`. -Pass `-w` to watch in realtime, `-v` for more detail, or `--json` for complete preset objects. Filter with `--base` or `--repo`. + +!!! info "Sparkline legend" + + Each bar in the sparklines represents one trial. The bar height is the trial's highest tok/s across concurrency values that met the constraints. + + | Bar color | Semantics | + | --- | --- | + | Red | A trial that broke constraints at every concurrency. | + | Green | The best trial that met the constraints. | + | Gray | A trial that met the constraints but had lower tok/s than the best trial. | + | Yellow | The best trial, if it broke constraints at every concurrency. | + + A red `·` represets a trial that produced no benchmark. + +For detailed benchmark data at each concurrency, pass `--json` to `dstack preset get`. + +To watch in realtime, use `-w` with `dstack preset`. Filter with `--base` or `--repo`. Also, use `-v` for verbose output. ### Delete presets @@ -390,7 +406,6 @@ At the same time, it's recommended to create presets using your own agent — ei ## Limitations * Currently, the agent doesn't upload compiled binaries anywhere; patches compile at runtime -* Doesn't support ranges for `concurrency` > Report bugs and request features on [GitHub](https://github.com/dstackai/dstack/issues), and ask questions on [Discord](https://discord.gg/u8SmfwPpMd). diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py index 28ba5e75ef..bae363f538 100644 --- a/src/dstack/_internal/cli/commands/preset.py +++ b/src/dstack/_internal/cli/commands/preset.py @@ -14,6 +14,7 @@ from dstack._internal.cli.models.presets import ( AnyStoredPreset, PresetListOutput, + PresetSessionEntry, UnverifiedPreset, ) from dstack._internal.cli.services.completion import ProjectNameCompleter @@ -284,7 +285,7 @@ def _list(self, args: argparse.Namespace) -> None: def _list_presets_and_sessions( self, *, base: str | None, repo: str | None - ) -> tuple[list[AnyStoredPreset], list[dict]]: + ) -> tuple[list[AnyStoredPreset], list[PresetSessionEntry]]: self._reconcile() presets = PresetStore().list() sessions = list_preset_sessions() @@ -552,15 +553,13 @@ def _filter_presets( def _session_matches_model( - session: dict, + session: PresetSessionEntry, *, base: str | None, repo: str | None, repo_to_base: dict[str, str], ) -> bool: - model = str(session.get("model") or "") - if not model: - return False + model = session.state.model if repo is not None: return model == repo return model == base or repo_to_base.get(model) == base diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 812ddce17e..33d6128121 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -1,6 +1,6 @@ import uuid from datetime import datetime -from typing import Annotated, Any, Dict, Literal, Optional, Union +from typing import Annotated, Any, Dict, Literal, Optional, Self, Union import yaml from pydantic import ( @@ -9,11 +9,12 @@ ValidationError, ValidatorFunctionWrapHandler, WrapValidator, + model_validator, ) from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.configurations import PresetAgentProvider, ServiceConfiguration -from dstack._internal.core.models.presets import PresetBenchmark +from dstack._internal.core.models.presets import PresetBenchmark, PresetResult class PresetAgentInvalidService(CoreModel): @@ -52,8 +53,7 @@ class PresetAgentSuccess(CoreModel): trial: PositiveInt base: Annotated[str, Field(min_length=1)] model: Annotated[str, Field(min_length=1)] - context_length: PositiveInt - benchmark: PresetBenchmark + result: Annotated[list[PresetResult], Field(min_length=1)] class PresetAgentFailure(CoreModel): @@ -66,6 +66,80 @@ class PresetAgentFailure(CoreModel): AnyPresetAgentResult = Union[PresetAgentSuccess, PresetAgentFailure] +class PresetTrialBenchmark(PresetBenchmark): + """A trial's benchmark, which the agent writes into `trials//trial.json` + after benchmarking the trial at a concurrency, read by the CLI for + `dstack preset list`. `PresetBenchmark.validate_metrics` is overridden + because its rule is for the final report; applied to `trial.json` it would + drop the whole trial from the listing.""" + + @model_validator(mode="after") + def validate_metrics(self) -> Self: + return self + + +class PresetTrialResult(CoreModel): + """One `result` entry of `trials//trial.json`, as the system prompt + defines it: the agent writes one per concurrency in `constraints.json` once + the trial is benchmarked, and the CLI reads them for `dstack preset list`. + The entry's `learned` field is left out: the CLI does not use it, because + the agent writes it for its own reading in later sessions.""" + + concurrency: Optional[PositiveInt] = None + context_length: Optional[PositiveInt] = None + benchmark: Optional[PresetTrialBenchmark] = None + failed: bool = False + + @model_validator(mode="after") + def fail_without_benchmark(self) -> Self: + """An entry without a benchmark is one the agent could not measure, which + the prompt defines as failed; the flag is set in case the agent left it out.""" + if self.benchmark is None: + self.failed = True + return self + + +def _entry_or_failed(value: Any, handler: ValidatorFunctionWrapHandler) -> Any: + try: + return handler(value) + except ValidationError: + # A broken entry counts as a failed one, so the other entries of the + # trial are still listed. + return PresetTrialResult(failed=True) + + +class PresetTrial(CoreModel): + """The `trials//trial.json` record as defined in the system_prompt.md. + The agent writes it once trial `` is benchmarked, and its presence is + what marks the trial completed; the CLI reads it for `dstack preset list`.""" + + resources: Optional[dict[str, Any]] = None + groups: Optional[list[list[dict[str, Any]]]] = None + # Required and non-empty: `system_prompt.md` states that `trial.json` has + # one `result` entry per concurrency value. + result: Annotated[ + list[Annotated[PresetTrialResult, WrapValidator(_entry_or_failed)]], + Field(min_length=1), + ] + + @model_validator(mode="before") + @classmethod + def convert_to_result_list(cls, data: Any) -> Any: + # An old `trial.json` has no `result` list. Its `benchmark`, + # `context_length` and `failed` are at the top level, and a failed one + # may have no `benchmark` at all. + if isinstance(data, dict) and "result" not in data: + data = dict(data) + data["result"] = [ + { + key: data.pop(key) + for key in ("benchmark", "context_length", "failed") + if key in data + } + ] + return data + + PresetSessionStatus = Literal["running", "interrupted", "success", "failed"] # The session models carry no defaults: `session.json` is this CLI's own state diff --git a/src/dstack/_internal/cli/models/presets.py b/src/dstack/_internal/cli/models/presets.py index d197df273c..92ece018f2 100644 --- a/src/dstack/_internal/cli/models/presets.py +++ b/src/dstack/_internal/cli/models/presets.py @@ -3,9 +3,14 @@ from pydantic import Field, PositiveInt +from dstack._internal.cli.models.preset_agent import ( + PresetSessionState, + PresetSessionStatus, + PresetTrial, +) from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.configurations import PresetConfiguration -from dstack._internal.core.models.presets import PortablePreset +from dstack._internal.core.models.presets import AnyPresetConstraints, PortablePreset class BasePreset(CoreModel): @@ -44,3 +49,34 @@ class PulledPreset(BasePreset, PortablePreset): class PresetListOutput(CoreModel): presets: List[AnyStoredPreset] + + +class PresetTrialsSummary(CoreModel): + """Built by the CLI for `dstack preset list`.""" + + count: int + # The trial with the highest tok/s that met the constraints. + best: Optional[PresetTrial] = None + # The trial with the highest tok/s but broke the constraints. + best_failed: Optional[PresetTrial] = None + # One value per trial in order, for the sparkline; None for a trial that + # produced no benchmark. + series: list[Optional[float]] + # Parallel to `series`: True where the trial broke the constraints. + failed: list[bool] + # The hardware of the shown trial, `best` or else `best_failed`; when neither + # exists, of the last trial that recorded it, so the session still shows what + # it ran on. + gpu: Optional[str] = None + + +class PresetSessionEntry(CoreModel): + """Built by the CLI for `dstack preset list`.""" + + id: str + state: PresetSessionState + name: Optional[str] + status: PresetSessionStatus + trials: PresetTrialsSummary + verification: Optional[str] + constraints: Optional[AnyPresetConstraints] diff --git a/src/dstack/_internal/cli/services/presets/build.py b/src/dstack/_internal/cli/services/presets/build.py index 1c3137f7aa..cf2d4b183e 100644 --- a/src/dstack/_internal/cli/services/presets/build.py +++ b/src/dstack/_internal/cli/services/presets/build.py @@ -9,7 +9,7 @@ from dstack._internal.core.models.instances import Resources from dstack._internal.core.models.presets import ( PRESET_EXCLUDED_FIELDS, - PresetBenchmark, + PresetResult, PresetVerificationReplicaGroup, ) from dstack._internal.core.models.resources import ( @@ -31,8 +31,7 @@ def build_preset( verification_replica_groups: list[PresetVerificationReplicaGroup], base_model: str, repo: str, - context_length: int, - benchmark: PresetBenchmark, + result: list[PresetResult], configuration: PresetConfiguration, best_trial: int, preset_id: str, @@ -48,12 +47,11 @@ def build_preset( base=base_model, id=preset_id, repo=repo, - context_length=context_length, best_trial=best_trial, configuration=configuration, created_at=created_at, service=service, - benchmark=benchmark, + result=result, verified_on=verification_replica_groups, ) diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 86f39d54c7..9e667b6529 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -1,13 +1,22 @@ from collections import defaultdict -from typing import Any, Optional, Sequence +from typing import Any, Callable, Iterable, Optional, Sequence, Union from rich.table import Table -from dstack._internal.cli.models.presets import AnyStoredPreset, VerifiedPreset +from dstack._internal.cli.models.preset_agent import PresetTrialResult +from dstack._internal.cli.models.presets import ( + AnyStoredPreset, + PresetSessionEntry, + VerifiedPreset, +) from dstack._internal.cli.utils.common import add_row_from_dict, console from dstack._internal.core.models.configurations import PresetConfiguration -from dstack._internal.core.models.presets import PresetWorkload -from dstack._internal.utils.common import parse_datetime, pretty_date, pretty_resources +from dstack._internal.core.models.presets import ( + PresetDatasetConstraints, + PresetResult, + PresetWorkload, +) +from dstack._internal.utils.common import pretty_date, pretty_resources _STATUS_DISPLAY = { "verified": ("verified", "secondary"), @@ -24,32 +33,27 @@ def _format_status(status: str) -> str: return f"[{style}]{text}[/]" if style else text -def _verifying(session: dict[str, Any]) -> bool: - return isinstance(session.get("verification"), dict) +def _verifying(session: PresetSessionEntry) -> bool: + return session.verification is not None _SPARK_BLOCKS = "▁▂▃▄▅▆▇█" -def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: - if not isinstance(session, dict): +def _format_trial_spark(session: PresetSessionEntry) -> str: + series = session.trials.series + failed = session.trials.failed + if not series: return "" - trials = session.get("trials") - series = trials.get("series") if isinstance(trials, dict) else None - if not isinstance(series, list) or not series: - return "" - failed = trials.get("failed") - if not isinstance(failed, list) or len(failed) != len(series): - failed = [False] * len(series) - values = [v for v in series if isinstance(v, (int, float))] + values = [v for v in series if v is not None] if not values: return "·" * len(series) high = max(values) - passed = [v for v, f in zip(series, failed) if isinstance(v, (int, float)) and not f] + passed = [v for v, f in zip(series, failed) if v is not None and not f] best = max(passed) if passed else max(values) out = [] for value, is_failed in zip(series, failed): - if not isinstance(value, (int, float)): + if value is None: out.append("[indian_red1]·[/]") continue glyph = ( @@ -65,25 +69,22 @@ def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: return "".join(out) -def _format_trial_progress(session: Optional[dict[str, Any]], *, in_flight: bool = False) -> str: - if not isinstance(session, dict): - return "" - trials = session.get("trials") - trials_num = session.get("trials_num") - if not isinstance(trials, dict) or not (trials.get("count") or isinstance(trials_num, int)): +def _format_trial_progress(session: PresetSessionEntry, *, in_flight: bool = False) -> str: + count = session.trials.count + trials_num = session.state.trials_num + if not count and trials_num is None: return "" - count = trials.get("count") or 0 if in_flight: - count = min(count + 1, trials_num) if isinstance(trials_num, int) else count + 1 + count = min(count + 1, trials_num) if trials_num is not None else count + 1 progress = str(count) - if isinstance(trials_num, int): + if trials_num is not None: progress += f"/{trials_num}" return f" [secondary]({progress})[/]" def print_presets( presets: Sequence[AnyStoredPreset], - sessions: Optional[list[dict[str, Any]]] = None, + sessions: Optional[list[PresetSessionEntry]] = None, verbose: bool = False, all_presets: bool = False, limit: Optional[int] = None, @@ -98,7 +99,7 @@ def print_presets( def get_presets_table( presets: Sequence[AnyStoredPreset], - sessions: Optional[list[dict[str, Any]]] = None, + sessions: Optional[list[PresetSessionEntry]] = None, verbose: bool = False, all_presets: bool = False, limit: Optional[int] = None, @@ -120,16 +121,15 @@ def get_presets_table( for preset in presets: presets_by_base[preset.base].append(preset) repo_to_base[preset.repo] = preset.base - sessions_by_model: dict[str, list[dict[str, Any]]] = defaultdict(list) - creations_by_id: dict[str, dict[str, Any]] = {} + sessions_by_model: dict[str, list[PresetSessionEntry]] = defaultdict(list) + creations_by_id: dict[str, PresetSessionEntry] = {} for session in sessions or []: - session_id = str(session.get("id")) - if session_id in preset_ids: + if session.id in preset_ids: # The stored artifact is the completion record. Cleanup can be # interrupted before session.json advances to `success`. - creations_by_id[session_id] = session + creations_by_id[session.id] = session continue - model = str(session.get("model") or "unknown") + model = session.state.model sessions_by_model[repo_to_base.get(model, model)].append(session) # Same contract as `dstack ps`: one flat list, newest first (base is a column, @@ -139,12 +139,12 @@ def get_presets_table( rows += [(preset.created_at.isoformat(), preset, True) for preset in preset_list] for session_list in sessions_by_model.values(): rows += [ - (str(session.get("created_at") or ""), session, False) for session in session_list + (session.state.created_at.isoformat(), session, False) for session in session_list ] rows.sort(key=lambda r: r[0], reverse=True) only_active = not all_presets and limit is None if only_active: - active = [r for r in rows if not r[2] and str(r[1].get("status")) == "running"] + active = [r for r in rows if not r[2] and r[1].status == "running"] rows = active or rows[:1] for _, item, is_preset in rows[:limit] if limit is not None else rows: @@ -155,93 +155,62 @@ def get_presets_table( return table -def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False) -> None: - created = "" - created_at = session.get("created_at") - if isinstance(created_at, str): - try: - created = pretty_date(parse_datetime(created_at)) - except ValueError: - created = created_at - benchmark = "" - gpu = "" - status_key = str(session.get("status", "")) +def _add_session(table: Table, session: PresetSessionEntry, *, verbose: bool = False) -> None: + status_key = session.status if status_key == "running" and _verifying(session): status_key = "verifying" status = _format_status(status_key) + _format_trial_progress( session, in_flight=status_key == "running" ) - trials = session.get("trials") - best = trials.get("best") if isinstance(trials, dict) else None + trials = session.trials # Nothing passed: fall back to the fastest attempt that did not. - breached = not isinstance(best, dict) - if breached and isinstance(trials, dict): - best = trials.get("best_failed") - if isinstance(best, dict): - gpu = best.get("gpu") or "" - if not gpu and isinstance(trials, dict): - # Trials can record hardware without ever producing a benchmark. - gpu = trials.get("gpu") or "" - constraints = session.get("constraints") or {} - parts = [] + breached = trials.best is None + best = trials.best or trials.best_failed + constraints = session.constraints objective = [] - if dataset := constraints.get("dataset"): - objective.append(f"data={dataset}") - if constraints.get("input_tokens") and constraints.get("output_tokens"): + if isinstance(constraints, PresetDatasetConstraints): + objective.append(f"data={constraints.dataset}") + elif constraints is not None: objective.append( - f"io={_format_token_count(constraints['input_tokens'])}" - f"/{_format_token_count(constraints['output_tokens'])}" + f"io={_format_token_count(constraints.input_tokens)}" + f"/{_format_token_count(constraints.output_tokens)}" ) - input_tokens = constraints.get("input_tokens") - if input_tokens: - shared_prefix_tokens = constraints.get("shared_prefix_tokens") or 0 - share = round(100 * shared_prefix_tokens / input_tokens) + share = round(100 * constraints.shared_prefix_tokens / constraints.input_tokens) if share: objective.append(f"prefix={share}%") - concurrency = (best or {}).get("concurrency") or constraints.get("concurrency") + # The shown trial's sweep when it has one, else what was requested. + concurrency = _result_concurrencies(best.result) if best is not None else [] + if not concurrency and constraints is not None: + concurrency = constraints.concurrency if concurrency: - objective.append(f"c={concurrency}") - min_context_length = constraints.get("min_context_length") - if verbose and isinstance(min_context_length, int): - objective.append(f"ctx>={_format_token_count(min_context_length)}") - max_ttft = constraints.get("max_ttft") - if verbose and isinstance(max_ttft, (int, float)): - objective.append(f"ttft<={_format_duration_ms(max_ttft)}") - if isinstance(best, dict): - tps = _format_number(best["tok_s"]) + objective.append(f"c={_format_concurrency(concurrency)}") + if verbose and constraints is not None: + objective.append(f"ctx>={_format_token_count(constraints.min_context_length)}") + objective.append(f"ttft<={_format_duration_ms(constraints.max_ttft)}") + benchmark = "" + if best is not None: if objective: - # Lead with per-user tps: comparable across rows regardless of concurrency. - tpot_ms = best.get("tpot_ms") - if isinstance(tpot_ms, (int, float)) and tpot_ms > 0: - parts.append(f"tps/user={_format_number(1000 / tpot_ms)}") - if verbose: - parts.append(f"tps={tps}") - ttft_ms = best.get("ttft_ms") - if isinstance(ttft_ms, (int, float)): - parts.append(f"ttft={_format_duration_ms(ttft_ms)}") - context_length = best.get("context_length") - if isinstance(context_length, int): - parts.append(f"ctx={_format_token_count(context_length)}") - benchmark = " ".join(parts) + benchmark = _format_result_benchmark(best.result, verbose=verbose) else: - benchmark = f"c={best.get('concurrency')} tps={tps}" - benchmark = benchmark.strip() + benchmarks = [r.benchmark for r in best.result if r.benchmark is not None] + tps = _format_range([b.effective_output_tok_per_s for b in benchmarks], _format_number) + benchmark = f"c={_format_concurrency(concurrency)} tps={tps}" if benchmark and breached: # Marked, not only dimmed: colour alone is not a signal. benchmark = f"[secondary]*{benchmark}[/]" add_row_from_dict( table, { - "ID": str(session.get("id", "")), - "BASE": str(session.get("model") or ""), - "NAME": str(session.get("name") or ""), - "GPU": gpu, - "RESOURCES": gpu, + "ID": session.id, + "BASE": session.state.model, + "NAME": session.name or "", + "GPU": trials.gpu or "", + "RESOURCES": trials.gpu or "", "": _format_trial_spark(session), "CONSTRAINTS": f"[secondary]{' '.join(objective)}[/]" if objective else "", "BENCHMARK": benchmark, "STATUS": status, - "SUBMITTED": created, + "SUBMITTED": pretty_date(session.state.created_at), }, ) @@ -251,7 +220,7 @@ def _add_preset( preset: AnyStoredPreset, *, verbose: bool, - creation: Optional[dict[str, Any]] = None, + creation: Optional[PresetSessionEntry] = None, ) -> None: groups = preset.service.replica_groups row = { @@ -259,8 +228,9 @@ def _add_preset( "NAME": preset.name or "", "RESOURCES": _format_resources(groups[0].resources, verbose=verbose), "BASE": preset.base, - "STATUS": _format_status(preset.status) + _format_trial_progress(creation), - "": _format_trial_spark(creation), + "STATUS": _format_status(preset.status) + + (_format_trial_progress(creation) if creation is not None else ""), + "": _format_trial_spark(creation) if creation is not None else "", "CONSTRAINTS": format_preset_objective( preset, verbose=verbose, @@ -288,7 +258,7 @@ def format_preset_objective( *, verbose: bool = False, ) -> str: - workload = preset.benchmark.workload + workload = _measured_workload_from_preset(preset) if isinstance(preset, VerifiedPreset): return _format_creation_objective(preset.configuration, workload, verbose=verbose) # A pulled preset carries no creation context; the measured workload is its @@ -304,10 +274,25 @@ def format_preset_objective( share = round(100 * workload.shared_prefix_tokens / workload.input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={workload.concurrency}") + parts.append(f"c={_format_concurrency([r.concurrency for r in preset.result])}") return f"[secondary]{' '.join(parts)}[/]" +def _measured_workload_from_preset(preset: AnyStoredPreset) -> PresetWorkload: + for r in preset.result: + if r.benchmark is not None: + return r.benchmark.workload + # Unreachable for a stored preset: verification requires at least one + # completed benchmark. Kept so the function returns on every path. + raise ValueError(f"preset {preset.id} has no measured workload") + + +def _format_concurrency(values: list[int]) -> str: + if len(values) == 1: + return str(values[0]) + return f"{min(values)}..{max(values)}" + + def _format_creation_objective( configuration: PresetConfiguration, workload: PresetWorkload, *, verbose: bool ) -> str: @@ -323,7 +308,7 @@ def _format_creation_objective( share = round(100 * (configuration.shared_prefix_tokens or 0) / input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={configuration.concurrency or workload.concurrency}") + parts.append(f"c={_format_concurrency(configuration.concurrency or [workload.concurrency])}") if verbose and configuration.min_context_length is not None: parts.append(f"ctx>={_format_token_count(configuration.min_context_length)}") if verbose and configuration.max_ttft is not None: @@ -331,35 +316,78 @@ def _format_creation_objective( return f"[secondary]{' '.join(parts)}[/]" -def _breaches_constraints(preset: AnyStoredPreset) -> bool: +def _passing_result(preset: AnyStoredPreset) -> list[PresetResult]: + """The measured entries that met the requested constraints.""" + measured = [r for r in preset.result if r.benchmark is not None] if not isinstance(preset, VerifiedPreset): # A pulled preset carries no requested constraints to breach. - return False - configuration = preset.configuration - metrics = preset.benchmark.metrics - if configuration.max_ttft is not None and metrics.ttft_ms.p50 > configuration.max_ttft: + return measured + return [r for r in measured if not _breaches_constraints(r, preset.configuration)] + + +def _breaches_constraints(r: PresetResult, configuration: PresetConfiguration) -> bool: + assert r.benchmark is not None + if ( + configuration.max_ttft is not None + and r.benchmark.metrics.ttft_ms.p50 > configuration.max_ttft + ): return True - return configuration.min_context_length is not None and ( - preset.context_length < configuration.min_context_length + return ( + configuration.min_context_length is not None + and r.context_length is not None + and r.context_length < configuration.min_context_length ) def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) -> str: - benchmark = preset.benchmark - metrics = benchmark.metrics + passing = _passing_result(preset) + if passing: + return _format_result_benchmark(passing, verbose=verbose) + text = _format_result_benchmark(preset.result, verbose=verbose) + return f"[secondary]*{text}[/]" + + +# Entries of a stored preset and of a session's trial share the fields read here. +AnyResult = Union[PresetResult, PresetTrialResult] + + +def _format_result_benchmark(result: Sequence[AnyResult], *, verbose: bool) -> str: + """Each metric as its range over the entries that have a benchmark.""" + benchmarks = [r.benchmark for r in result if r.benchmark is not None] + context_lengths = [r.context_length for r in result if r.context_length is not None] + # Lead with per-user tps: comparable across rows regardless of concurrency. parts = [ - f"tps/user={_format_number(benchmark.effective_per_user_tok_per_s)}", + f"tps/user={_format_range([b.effective_per_user_tok_per_s for b in benchmarks], _format_number)}" ] if verbose: - parts.append(f"tps={_format_number(benchmark.effective_output_tok_per_s)}") - parts += [ - f"ttft={_format_duration_ms(metrics.ttft_ms.p50)}", - f"ctx={_format_token_count(preset.context_length)}", - ] - text = " ".join(parts) - if _breaches_constraints(preset): - return f"[secondary]*{text}[/]" - return text + parts.append( + f"tps={_format_range([b.effective_output_tok_per_s for b in benchmarks], _format_number)}" + ) + parts.append( + f"ttft={_format_range([b.metrics.ttft_ms.p50 for b in benchmarks], _format_duration_ms)}" + ) + if context_lengths: + parts.append(f"ctx={_format_range(context_lengths, _format_token_count)}") + return " ".join(parts) + + +def _result_concurrencies(result: Sequence[AnyResult]) -> list[int]: + """Old `trial.json` entries carry the concurrency only inside the benchmark.""" + values = [] + for r in result: + if r.concurrency is not None: + values.append(r.concurrency) + elif r.benchmark is not None: + values.append(r.benchmark.workload.concurrency) + return values + + +def _format_range(values: Iterable[float], format_value: Callable[[Any], str]) -> str: + values = list(values) + low, high = min(values), max(values) + if low == high: + return format_value(low) + return f"{format_value(low)}..{format_value(high)}" def _format_duration_ms(value: float) -> str: diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 691df34b33..a010ad78bf 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -6,8 +6,8 @@ hardware, through sequential experimental trials. ## Performance Performance is `output_tok_per_s`, the output token throughput of a benchmark -run at the `concurrency` from `constraints.json` (see `## Benchmark`). Benchmark -tools report several throughput numbers; performance is the output one. +run at each `concurrency` value from `constraints.json` (see `## Benchmark`). +Benchmark tools report several throughput numbers; performance is the output one. Significantly better performance goes beyond what conventional configurations and published benchmarks reach; they are not the ceiling. Estimate the @@ -35,10 +35,10 @@ Field semantics: - `max_ttft`: the maximum p50 time to first token, in milliseconds, that any benchmark in this session may report. - `trials_num`: the number of trials in this session. -- `concurrency`: the number of simultaneous requests for every benchmark in - this session. It is fixed so that benchmark results are comparable. - +- `concurrency`: a list of simultaneous request counts used for every + benchmark in this session. For example, `[2, 4, 8]` means measuring + performance at each of 2, 4, and 8 simultaneous requests. The list is + fixed so that benchmark results are comparable across trials. - `dataset`: the benchmark dataset for every benchmark in this session; see `## Benchmark`. @@ -167,6 +167,10 @@ continuing to improve a previous trial's idea, but when that risks settling into a local optimum, search for a substantially different approach rather than tweaking parameters further. + + Before starting trials, analyze the previous sessions' results provided in `previous/` (sessions: {previous}; see `## Previous Sessions`). The @@ -243,13 +247,16 @@ Docker images and other dependencies.--> Once `trials//task.dstack.yml` is written, write the corresponding benchmark results (see `## Benchmark` for the structure) to `trials//trial.json`: the presence of `trials//trial.json` is what -marks trial `` completed. The benchmark -may be skipped in one case only: you failed to make the configuration run -at all — a failed trial. `trials//task.dstack.yml` may be skipped in -one case only: you failed to get benchmark results at all. A trial is -also failed when its benchmark does not meet the constraints (see -`# Constraints`). When a trial that changed several things fails, be -mindful of which specific change was the root cause. +marks trial `` completed. The benchmark at a concurrency may be skipped +only if you failed to make the configuration run at that concurrency. +`trials//task.dstack.yml` may be skipped only if the benchmark produced no +result at any concurrency value in `constraints.json` — a failed trial. A +trial is also failed when its benchmark at all concurrency values fails to +meet the constraints (see `# Constraints`). When a trial that changed +several things fails, be mindful of which specific change was the root +cause. + +## Trial Report `trials//trial.json` is a JSON object. @@ -257,7 +264,7 @@ mindful of which specific change was the root cause. others: ``` -{"resources": {...}, "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} +{"resources": {...}, "result": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` 2. In case the task is using node groups (a router group plus worker @@ -265,7 +272,7 @@ mindful of which specific change was the root cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} +{"groups": [[{...}], [{...}, {...}], [{...}]], "result": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -293,15 +300,20 @@ mindful of which specific change was the root cause. [{"cpu": "192", "memory": "2048GB", "disk": "1000GB", "gpu": {"name": "H200", "memory": "141GB", "count": 8}}] ] ``` -- `context_length`: the largest context the trial's configuration handles, - found as described in `## Benchmark`; `null` only when the benchmark couldn't - be done at all. -- `benchmark`: the trial benchmark (see `## Benchmark` for the structure); - `null` only when the benchmark couldn't be done at all. -- `learned`: the major things this trial taught you that you did not know - before it ran. Required for every trial, including a failed one. -- `failed`: `true` if the configuration never ran or the benchmark broke a - constraint such as `max_ttft` or `min_context_length`, absent otherwise. +- `result`: a list of trial result objects, one result object for each value of `concurrency` from `constraints.json`. Each object has fields no other than below: + - `concurrency`: the concurrency value for this result. + - `context_length`: the largest context the trial's configuration handles + at this concurrency, found as described in `## Benchmark`; `null` only + when the benchmark couldn't be done at this concurrency. + - `benchmark`: the trial benchmark at this concurrency (see `## Benchmark` + for the structure); `null` only when the benchmark couldn't be done at this + concurrency. + - `learned`: the major things this trial taught you at this concurrency that + you did not know before it ran. Required for every `result` entry, + including a failed one. + - `failed`: `true` if the configuration did not run or the benchmark broke + a constraint such as `max_ttft` or `min_context_length` at this + concurrency; `false` otherwise. You're expected to do exactly `trials_num` trials (see `# Constraints`). @@ -375,9 +387,16 @@ never part of the measured metrics. All verification and benchmark requests must succeed. -Record every benchmark using the following structure and field names — -trial benchmarks in `trials//trial.json`, the final benchmark as -`final_report.json.benchmark` (values are illustrative): +Use the format below for the `benchmark` field of each `result` entry in: + +- `trials//trial.json`, as defined in `## Trial Report`. +- `final_report.json`, as defined in `# Final Report`. + +The benchmark's `workload.concurrency` must equal the entry's `concurrency` +value from `constraints.json`. + +The JSON below is one `benchmark` object, not the whole file. Use its exact +structure and field names; values are illustrative. ```json { @@ -407,17 +426,16 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free -invocation. For the final benchmark, run it with streaming responses, set -`workload.concurrency` to `concurrency` from `constraints.json`, produce -every field of the structure, and calculate all metrics from the -`num_requests` measured requests only — exclude setup, health-check, and -warmup requests. Never invent missing values. +invocation. Use streaming responses for the final benchmark. -After each benchmark, find the largest context the configuration handles by -sending real requests, and record it: for a trial, as the `context_length` -field of `trials//trial.json`; for the final benchmark, as -`final_report.json.context_length`. Stopping at the required minimum is not -enough. +Include every field in the benchmark format. Calculate all metrics from the +`num_requests` measured requests only; exclude setup, health-check, and warmup +requests. Never invent missing values. + +After each benchmark, send real requests at the same concurrency to find the +largest context the configuration can handle. Record it in the `context_length` +field of the same `result` entry as that benchmark (see `## Trial Report` and +`# Final Report`). Do not stop at the required minimum. @@ -575,10 +593,7 @@ In case the service is using a router (see `## Cluster Placement`), run the final benchmark inside the router replica, directly against the router engine. Attach to it via `dstack attach --replica --job `. -If the service or its benchmark cannot be completed, stop that service, -pick the next-best trial, and repeat, until a service is verified or there -are no unverified trials left. Report the result accordingly (see -`# Final Report`). +If the service cannot be completed or none of its benchmarks at the concurrency values from `constraints.json` can be completed, stop that service, pick the next-best trial, and repeat, until a service is verified or there are no unverified trials left. Report the result accordingly (see `# Final Report`). When you report success, leave the final `dstack` service running: the caller verifies the live run and stops it afterwards. @@ -591,16 +606,14 @@ print, copy, or summarize tokens, secrets, or environment variable values. Do not expose the value of `DSTACK_TOKEN` or the value of any environment variable listed under `env` in `constraints.json`. -Do not put secret values in `final_report.json`, or print -them. Use env -references in `final_report.json.service_yaml`; use environment variable names or redacted values in -`final_report.json.benchmark.command`. +Do not put secret values in `final_report.json` or print them. Use env +references in `service_yaml` and environment variable names or redacted values +in each `result` entry's `benchmark.command` field (see `# Final Report`). # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmark`, -and `failure_summary`. +`service_yaml`, `trial`, `base`, `model`, `result`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed trial remained), include exactly: @@ -612,9 +625,13 @@ trial remained), include exactly: - `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `context_length`: the largest context verified for the final service, as - described in `## Benchmark` -- `benchmark`: the final service benchmark described in `## Benchmark` +- `result`: list of final service result objects, one object for each value of `concurrency` from `constraints.json`. Each object has only the fields below: + - `concurrency`: the concurrency value for this result + - `context_length`: the largest context verified for the final service at + this concurrency, as described in `## Benchmark`. `null` only when the benchmark couldn't be done at this concurrency. + - `benchmark`: the final service benchmark at this concurrency, as + described in `## Benchmark`. `null` only when the benchmark couldn't be done at this + concurrency. Set `final_report.json.base` as follows: diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index 9ce77213e3..a1f4c27674 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -25,12 +25,16 @@ PresetSessionState, PresetSessionStatus, PresetSessionWorkspace, + PresetTrial, + PresetTrialResult, ) +from dstack._internal.cli.models.presets import PresetSessionEntry, PresetTrialsSummary from dstack._internal.cli.utils.common import console, error_console from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.errors import CLIError from dstack._internal.core.models.common import validate_extra_ignore, validate_json_extra_ignore from dstack._internal.core.models.configurations import PresetAgentProvider, PresetConfiguration +from dstack._internal.core.models.presets import AnyPresetConstraints from dstack._internal.utils.common import get_dstack_dir _PROGRESS_FILENAME = "progress.jsonl" @@ -515,7 +519,7 @@ def resolve_session_ref(ref: str) -> str: return ref -def list_preset_sessions() -> list[dict[str, Any]]: +def list_preset_sessions() -> list[PresetSessionEntry]: entries = [] for session in iter_preset_sessions(): path = session.path @@ -525,18 +529,21 @@ def list_preset_sessions() -> list[dict[str, Any]]: status = state.status if status == "running" and not session_process_alive(state): status = "interrupted" - entry = state.model_dump(mode="json") - entry["id"] = path.name - entry["name"] = claimed_session_name(state) - entry["status"] = status - entry["trials"] = _summarize_session_trials(path / TRIALS_DIRNAME) - entry["verification"] = _read_last_session_verification(path / SERVICE_DIRNAME) - entry["constraints"] = _read_session_constraints(path) - entries.append(entry) + entries.append( + PresetSessionEntry( + state=state, + id=path.name, + name=claimed_session_name(state), + status=status, + trials=_summarize_session_trials(path / TRIALS_DIRNAME), + verification=_read_last_session_verification(path / SERVICE_DIRNAME), + constraints=_read_session_constraints(path), + ) + ) return entries -def _read_session_constraints(path: Path) -> dict[str, Any]: +def _read_session_constraints(path: Path) -> Optional[AnyPresetConstraints]: """Reads the session's own copy first: it outlives the agent workspace (removed once the session finishes). The workspace copy is a backward-compat fallback for sessions recorded before the session-level copy existed.""" @@ -545,12 +552,12 @@ def _read_session_constraints(path: Path) -> dict[str, Any]: path / "workspace" / "w" / _CONSTRAINTS_FILENAME, ): try: - data = json.loads(candidate.read_text(encoding="utf-8")) - except (OSError, ValueError): + return validate_json_extra_ignore( + AnyPresetConstraints, candidate.read_text(encoding="utf-8") + ) + except (OSError, ValueError, ValidationError): continue - if isinstance(data, dict): - return data - return {} + return None def _numbered_subdirs(path: Path) -> list[Path]: @@ -571,101 +578,116 @@ def _read_record(path: Path) -> Optional[dict[str, Any]]: return record if isinstance(record, dict) else None -def _read_last_session_verification(path: Path) -> Optional[dict[str, Any]]: - """An attempt whose result file has not appeared yet is still in flight - (reported as verifying).""" +def _read_last_session_verification(path: Path) -> Optional[str]: + """The last attempt's status; an attempt whose result file has not appeared + yet is still in flight (reported as verifying).""" attempts = _numbered_subdirs(path) if not attempts: return None - last = attempts[-1] - record = _read_record(last / VERIFICATION_RESULT_FILENAME) + record = _read_record(attempts[-1] / VERIFICATION_RESULT_FILENAME) if record is not None and isinstance(record.get("status"), str): - return record - return {"status": "verifying"} + return record["status"] + return "verifying" -def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: - # TODO: Refactor this crap - must be explicit what is this and where is it used; also dicts are prohibited in dstack repo +def _summarize_session_trials(path: Path) -> PresetTrialsSummary: """A trial directory without `trial.json` is still in flight and is not counted.""" - records = [] + trials = [] for trial_dir in _numbered_subdirs(path): - record = _read_record(trial_dir / TRIAL_RESULT_FILENAME) - if record is not None: - records.append(record) - count = 0 - best: Optional[dict[str, Any]] = None - # The fastest trial that broke a constraint, shown only when nothing passed. - best_failed: Optional[dict[str, Any]] = None - # One entry per trial in order, `None` for a trial that produced no benchmark. - series: list[Optional[float]] = [] - # Parallel to `series`: True where the trial broke a constraint. - failed: list[bool] = [] - # Kept outside `best` so a run where nothing passed still shows what it ran on. + trial = _read_trial(trial_dir / TRIAL_RESULT_FILENAME) + if trial is not None: + trials.append(trial) + # Select the trial with the highest tok/s that meets the constraints, and + # show its sweep. When no trial meets them, fallback to the fastest + # failed one, marked in the listing. + best, best_failed = _select_best_and_best_failed(trials) + shown = best or best_failed + gpu = _format_trial_gpu(shown) if shown is not None else None + if gpu is None: + gpu = _last_trial_gpu(trials) + return PresetTrialsSummary( + count=len(trials), + best=best, + best_failed=best_failed, + series=[_max_tok_s(_scored_trial(trial)) for trial in trials], + failed=[_trial_failed(trial) for trial in trials], + gpu=gpu, + ) + + +def _last_trial_gpu(trials: list[PresetTrial]) -> Optional[str]: + """Trials can record hardware without ever producing a benchmark.""" gpu: Optional[str] = None - for record in records: - count += 1 - benchmark = record.get("benchmark") - failed.append(bool(record.get("failed"))) - record_gpu = _format_trial_gpu(record) - if record_gpu: - gpu = record_gpu - if not isinstance(benchmark, dict): - series.append(None) - continue - metrics = benchmark.get("metrics") or {} - workload = benchmark.get("workload") or {} - duration = metrics.get("duration_seconds") - tokens = metrics.get("total_output_tokens") - if not isinstance(duration, (int, float)) or duration <= 0: - series.append(None) - continue - if not isinstance(tokens, (int, float)): - series.append(None) - continue - tok_s = tokens / duration - series.append(tok_s) - # A failed trial keeps its benchmark — it is what the next trial learns - # from — but it is not a candidate for best, and promoting one would put - # a configuration that broke a constraint at the top of the listing. - if record.get("failed"): - if best_failed is None or tok_s > best_failed["tok_s"]: - best_failed = _trial_entry(tok_s, record, metrics, workload, record_gpu) + for trial in trials: + gpu = _format_trial_gpu(trial) or gpu + return gpu + + +def _read_trial(path: Path) -> Optional[PresetTrial]: + """None if `trial.json` is missing, half-written, or not a trial record.""" + record = _read_record(path) + if record is None: + return None + try: + return validate_extra_ignore(PresetTrial, record) + except ValidationError: + return None + + +def _trial_failed(trial: PresetTrial) -> bool: + """A trial failed when it broke the constraints at every concurrency.""" + return all(entry.failed for entry in trial.result) + + +def _passing_trial_results(trial: PresetTrial) -> list[PresetTrialResult]: + """The `result` where the trial met the constraints.""" + return [entry for entry in trial.result if not entry.failed] + + +def _max_tok_s(trial: PresetTrial) -> Optional[float]: + """The highest tok/s over the trial's entries that have a benchmark; None + when no entry has one.""" + values = [ + entry.benchmark.effective_output_tok_per_s + for entry in trial.result + if entry.benchmark is not None + ] + if not values: + return None + return max(values) + + +def _scored_trial(trial: PresetTrial) -> PresetTrial: + """Returns trial with only the entries that met the constraints, or the original trial if none did.""" + if _trial_failed(trial): + return trial + return trial.model_copy(update={"result": _passing_trial_results(trial)}) + + +def _select_best_and_best_failed( + trials: list[PresetTrial], +) -> tuple[Optional[PresetTrial], Optional[PresetTrial]]: + """Selects the trial with the highest tok/s that meets the constraints, and + the trail with highest tok/s but failed to meet the constraints.""" + best: Optional[PresetTrial] = None + best_tok_s = 0.0 + best_failed: Optional[PresetTrial] = None + best_failed_tok_s = 0.0 + for trial in trials: + kept = _scored_trial(trial) + tok_s = _max_tok_s(kept) + if tok_s is None: continue - if best is None or tok_s > best["tok_s"]: - best = _trial_entry(tok_s, record, metrics, workload, record_gpu) - return { - "count": count, - "best": best, - "best_failed": best_failed, - "series": series, - "failed": failed, - "gpu": gpu, - } - - -def _trial_entry( - tok_s: float, - record: dict[str, Any], - metrics: dict[str, Any], - workload: dict[str, Any], - gpu: Optional[str], -) -> dict[str, Any]: - ttft = (metrics.get("ttft_ms") or {}).get("p50") - # Mean, not p50: MTP skews TPOT right, and mean is the delivered decode rate. - tpot = (metrics.get("tpot_ms") or {}).get("mean") - context_length = record.get("context_length") - return { - "tok_s": tok_s, - "tpot_ms": tpot if isinstance(tpot, (int, float)) and tpot > 0 else None, - "ttft_ms": ttft if isinstance(ttft, (int, float)) else None, - "context_length": context_length if isinstance(context_length, int) else None, - "concurrency": workload.get("concurrency"), - "gpu": gpu, - } - - -def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: + if _trial_failed(trial): + if best_failed is None or tok_s > best_failed_tok_s: + best_failed, best_failed_tok_s = kept, tok_s + elif best is None or tok_s > best_tok_s: + best, best_tok_s = kept, tok_s + return best, best_failed + + +def _format_trial_gpu(trial: PresetTrial) -> Optional[str]: """A trial records its hardware in one of two formats, as described in `system_prompt.md`: `{"resources": {...}}` without node groups, and `{"groups": [[...], ...]}` with them. @@ -679,7 +701,7 @@ def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: `dstack preset list -v`. """ counts: dict[str, int] = {} - for node in _trial_nodes(record): + for node in _trial_nodes(trial): spec = _format_gpu(node.get("gpu")) if spec: # Insertion order is group order, so the roles read in the order they ran. @@ -689,18 +711,10 @@ def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: return " + ".join(spec if n == 1 else f"{spec} x{n}" for spec, n in counts.items()) -def _trial_nodes(record: dict[str, Any]) -> list[dict[str, Any]]: - groups = record.get("groups") - if isinstance(groups, list): - return [ - node - for group in groups - if isinstance(group, list) - for node in group - if isinstance(node, dict) - ] - resources = record.get("resources") - return [resources] if isinstance(resources, dict) else [] +def _trial_nodes(trial: PresetTrial) -> list[dict[str, Any]]: + if trial.groups is not None: + return [node for group in trial.groups for node in group] + return [trial.resources] if trial.resources is not None else [] def _format_gpu(gpu: Any) -> Optional[str]: diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 643522c503..e7467180a6 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -29,6 +29,7 @@ from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.presets import ( PresetBenchmark, + PresetResult, PresetVerificationReplicaGroup, PresetWorkload, ) @@ -113,8 +114,7 @@ def build_verified_preset( # prompt has always used; the preset document calls it `repo`, so that # it cannot be confused with the service's client-facing model name. repo=report.model, - context_length=report.context_length, - benchmark=_normalized_benchmark(report.benchmark, preset_configuration), + result=_normalized_result(report.result, preset_configuration), best_trial=report.trial, configuration=preset_configuration, preset_id=preset_id, @@ -122,6 +122,17 @@ def build_verified_preset( ) +def _normalized_result( + result: list[PresetResult], configuration: PresetConfiguration +) -> list[PresetResult]: + return [ + r.model_copy(update={"benchmark": _normalized_benchmark(r.benchmark, configuration)}) + if r.benchmark is not None + else r + for r in result + ] + + def _verified_run_service(run: Run, report: PresetAgentSuccess) -> ServiceConfiguration: """The service the server actually runs, after proving the report talks about this run and the run is a live model service.""" @@ -141,7 +152,7 @@ def _check_report_answers_request( """The report must answer what the configuration asked: the same benchmark workload, and the requested model — exactly when it was exact, any variant of the base otherwise.""" - _check_workload_answers_request(report.benchmark.workload, configuration) + _check_result_answers_request(report.result, configuration) if configuration.model.allows_variant_selection: if report.base != configuration.model.api_model_name: raise CLIError("The agent's final report base does not match the requested model") @@ -149,6 +160,40 @@ def _check_report_answers_request( raise CLIError("The agent changed an exact model request") +def _check_result_answers_request( + result: list[PresetResult], configuration: PresetConfiguration +) -> None: + if all(r.benchmark is None for r in result): + raise CLIError("The agent's final report has no completed benchmark") + concurrencies = [r.concurrency for r in result] + if len(set(concurrencies)) != len(concurrencies): + raise CLIError("The agent's final report has more than one entry for the same concurrency") + for r in result: + if ( + configuration.concurrency is not None + and r.concurrency not in configuration.concurrency + ): + raise CLIError( + f"The agent's final benchmark concurrency of {r.concurrency} does not match the" + f" requested concurrency of {configuration.concurrency}" + ) + # The agent could not measure this concurrency. That is allowed, so + # there is nothing more to check for it. + if r.benchmark is None: + continue + if r.context_length is None: + raise CLIError( + f"The agent's final report has a benchmark at concurrency {r.concurrency}" + " but no context length" + ) + if r.benchmark.workload.concurrency != r.concurrency: + raise CLIError( + f"The agent's final benchmark concurrency {r.concurrency} does not" + f" match the benchmark workload concurrency {r.benchmark.workload.concurrency}" + ) + _check_workload_answers_request(r.benchmark.workload, configuration) + + def _check_workload_answers_request( workload: PresetWorkload, configuration: PresetConfiguration ) -> None: @@ -172,11 +217,6 @@ def _check_workload_answers_request( f"The agent's final benchmark shared prefix of {workload.shared_prefix_tokens}" f" tokens does not match the requested {shared_prefix_tokens}" ) - if configuration.concurrency is not None and workload.concurrency != configuration.concurrency: - raise CLIError( - f"The agent's final benchmark concurrency of {workload.concurrency} does not match the" - f" requested concurrency of {configuration.concurrency}" - ) def _normalized_benchmark( diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 93f32be414..87aba357d9 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -1773,12 +1773,13 @@ class PresetConfiguration( ), ] = None concurrency: Annotated[ - Optional[PositiveInt], + Optional[list[PositiveInt]], Field( + min_length=1, description=( - "The number of simultaneous requests used for benchmarks during preset" + "List of simultaneous request counts used for benchmarks during preset" " creation. Required for creation" - ) + ), ), ] = None input_tokens: Annotated[ @@ -1944,6 +1945,26 @@ def validate_prompt(cls, value: Any) -> Any: raise ValueError(f"Prompt must be at most {MAX_PROMPT_LENGTH} characters") return value + @field_validator( + "concurrency", mode="before", json_schema_input_type=Union[PositiveInt, list[PositiveInt]] + ) + @classmethod + def convert_single_concurrency_to_list(cls, value: Any) -> Any: + if isinstance(value, bool): + raise ValueError( + "`concurrency` must be a positive integer or a list of positive integers" + ) + if isinstance(value, int): + return [value] + return value + + @field_validator("concurrency") + @classmethod + def sort_concurrency(cls, value: Optional[list[PositiveInt]]) -> Optional[list[PositiveInt]]: + if value is None: + return None + return sorted(set(value)) + def _validate_model(value: Any, *, field: str) -> str: if not isinstance(value, str) or not value.strip(): diff --git a/src/dstack/_internal/core/models/presets.py b/src/dstack/_internal/core/models/presets.py index 7600651ffa..8ede419dd5 100644 --- a/src/dstack/_internal/core/models/presets.py +++ b/src/dstack/_internal/core/models/presets.py @@ -1,5 +1,5 @@ import re -from typing import Annotated, Any, List, Literal, Optional, Self, Sequence +from typing import Annotated, Any, List, Literal, Optional, Self, Sequence, Union from pydantic import Field, PositiveFloat, PositiveInt, field_validator, model_validator @@ -127,6 +127,15 @@ def validate_metrics(self) -> Self: return self +class PresetResult(CoreModel): + concurrency: PositiveInt + # The largest context the service was verified to serve at this concurrency. + # A preset needs a completed benchmark at only one of the concurrencies from constraints. + # This is why context_length and benchmark are optional. + context_length: Optional[PositiveInt] = None + benchmark: Optional[PresetBenchmark] = None + + class PresetVerificationReplicaGroup(CoreModel): # The service replica group this was measured for. name: str @@ -143,8 +152,6 @@ class PortablePreset(CoreModel): # The exact repo/path the service loads, which `base` is a variant of. Not # the client-facing API model name — that is `service.model`. repo: Annotated[str, Field(min_length=1)] - # The largest context the service was verified to serve. - context_length: PositiveInt # The verified run's spec configuration. The validator below keeps `name`, # `gateway`, and profile params unset (the deployer's choices) and requires # `model` and resources. Env keys the user declared as passthroughs hold @@ -152,11 +159,26 @@ class PortablePreset(CoreModel): # stored as-is. `files` paths are stored relative to the preset directory, # absolute after load. service: ServiceConfiguration - benchmark: PresetBenchmark + result: Annotated[List[PresetResult], Field(min_length=1)] # The hardware it was verified on: the actual resources of every running # replica, by service replica group. verified_on: List[PresetVerificationReplicaGroup] + @model_validator(mode="before") + @classmethod + def convert_to_result_list(cls, data: Any) -> Any: + if isinstance(data, dict) and "result" not in data and "benchmark" in data: + data = dict(data) + benchmark = data.pop("benchmark") + data["result"] = [ + { + "concurrency": benchmark["workload"]["concurrency"], + "context_length": data.pop("context_length", None), + "benchmark": benchmark, + } + ] + return data + @model_validator(mode="after") def validate_artifact(self) -> Self: service = self.service @@ -303,11 +325,20 @@ class PresetConstraints(CoreModel): min_context_length: PositiveInt max_ttft: PositiveInt trials_num: PositiveInt - concurrency: PositiveInt + concurrency: Annotated[list[PositiveInt], Field(min_length=1)] baseline: bool fleets: Annotated[list[str], Field(min_length=1)] env: list[str] + @field_validator("concurrency", mode="before") + @classmethod + def convert_single_concurrency_to_list(cls, value: Any) -> Any: + # `constraints.json` of a session created before the concurrency sweep + # holds a single value. + if isinstance(value, int) and not isinstance(value, bool): + return [value] + return value + class PresetRandomConstraints(PresetConstraints): """Constraints for the synthetic `random` dataset, which the request shape defines.""" @@ -323,6 +354,11 @@ class PresetDatasetConstraints(PresetConstraints): dataset: str +# `constraints.json` is one of the two; each has a required field the other lacks, +# so a parse picks the right one. +AnyPresetConstraints = Union[PresetDatasetConstraints, PresetRandomConstraints] + + def _validate_model(value: Any, *, field: str) -> str: if not isinstance(value, str) or not value.strip(): raise ValueError(f"Preset model {field} must be a non-empty string") diff --git a/src/tests/_internal/cli/commands/test_preset.py b/src/tests/_internal/cli/commands/test_preset.py index 5c36acd506..8bf3699dd5 100644 --- a/src/tests/_internal/cli/commands/test_preset.py +++ b/src/tests/_internal/cli/commands/test_preset.py @@ -368,8 +368,8 @@ def test_gets_complete_preset_as_json_without_api_client(self, tmp_path, capsys) data = json.loads(capsys.readouterr().out) assert data["id"] == preset.id assert data["created_at"] == render_datetime_as_api(preset.created_at) - assert data["context_length"] == 32768 - assert data["benchmark"]["metrics"]["total_output_tokens"] == 2048 + assert data["result"][0]["context_length"] == 32768 + assert data["result"][0]["benchmark"]["metrics"]["total_output_tokens"] == 2048 @pytest.mark.parametrize( "args", @@ -389,8 +389,8 @@ def test_lists_complete_presets_as_json(self, tmp_path, capsys, args): data = output["presets"][0] assert data["id"] == preset.id assert data["created_at"] == render_datetime_as_api(preset.created_at) - assert data["context_length"] == 32768 - assert data["benchmark"]["metrics"]["total_output_tokens"] == 2048 + assert data["result"][0]["context_length"] == 32768 + assert data["result"][0]["benchmark"]["metrics"]["total_output_tokens"] == 2048 @pytest.mark.parametrize("flag_attribute", [("--base", "base"), ("--repo", "repo")]) def test_deletes_all_presets_of_model_keeping_others_without_api_client( diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index add3afc533..6815c7c963 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -16,11 +16,18 @@ PresetSessionRun, PresetSessionState, PresetSessionWorkspace, + PresetTrialBenchmark, + PresetTrialResult, +) +from dstack._internal.cli.models.presets import ( + PresetSessionEntry, + PresetTrialsSummary, + VerifiedPreset, ) -from dstack._internal.cli.models.presets import VerifiedPreset from dstack._internal.cli.services.presets.agents.base import PresetAgentSpec from dstack._internal.cli.services.presets.store import PresetStore from dstack._internal.compat import IS_WINDOWS +from dstack._internal.core.models.common import validate_extra_ignore from dstack._internal.core.models.configurations import ( DEFAULT_REPLICA_GROUP_NAME, PresetConfiguration, @@ -28,7 +35,9 @@ ) from dstack._internal.core.models.instances import Disk, Gpu, Resources from dstack._internal.core.models.presets import ( + AnyPresetConstraints, PresetBenchmark, + PresetResult, PresetVerificationReplicaGroup, ) from dstack._internal.core.models.resources import ResourcesSpec @@ -169,9 +178,14 @@ def get_preset( "env": ["HF_TOKEN"], } ), - context_length=context_length, best_trial=1, - benchmark=get_preset_benchmark(), + result=[ + PresetResult( + concurrency=1, + context_length=context_length, + benchmark=get_preset_benchmark(), + ) + ], verified_on=[ PresetVerificationReplicaGroup(name=DEFAULT_REPLICA_GROUP_NAME, replicas=[resources]) ], @@ -267,6 +281,82 @@ def get_session_run(**overrides: Any) -> PresetSessionRun: return PresetSessionRun(**fields) +def get_preset_constraints(**overrides: Any) -> AnyPresetConstraints: + """`constraints.json` as a session records it: synthetic by default, a + dataset one when `dataset` is given.""" + fields: dict[str, Any] = { + "run_name_prefix": "qwen-ab12cd34", + "model": {"base": "Qwen/Qwen3.5-27B"}, + "min_context_length": 8192, + "max_ttft": 5000, + "trials_num": 3, + "concurrency": [1], + "baseline": True, + "fleets": ["gpu-fleet"], + "env": [], + } + if "dataset" not in overrides: + fields.update({"input_tokens": 1024, "output_tokens": 1024, "shared_prefix_tokens": 0}) + fields.update(overrides) + return validate_extra_ignore(AnyPresetConstraints, fields) + + +def get_trial_result( + *, + tok_s: float = 42.0, + tpot_ms: Optional[float] = None, + ttft_ms: Optional[float] = None, + concurrency: int = 1, + context_length: Optional[int] = None, + failed: bool = False, +) -> PresetTrialResult: + """One `result` entry of a `trial.json`, with a benchmark that measured + `tok_s` over one second, so it must be a whole number of tokens. `tpot_ms` + sets the mean and `ttft_ms` the p50, the statistics the listing shows.""" + benchmark = get_preset_benchmark().model_dump() + benchmark["workload"]["concurrency"] = concurrency + benchmark["metrics"].update({"duration_seconds": 1.0, "total_output_tokens": tok_s}) + if tpot_ms is not None: + benchmark["metrics"]["tpot_ms"]["mean"] = tpot_ms + if ttft_ms is not None: + benchmark["metrics"]["ttft_ms"]["p50"] = ttft_ms + return PresetTrialResult( + concurrency=concurrency, + context_length=context_length, + benchmark=PresetTrialBenchmark.model_validate(benchmark), + failed=failed, + ) + + +def get_trials_summary(count: int = 0, **overrides: Any) -> PresetTrialsSummary: + """`count` trials that produced no benchmark, unless `series` and `failed` + say otherwise.""" + fields: dict[str, Any] = {"count": count, "series": [None] * count, "failed": [False] * count} + fields.update(overrides) + return PresetTrialsSummary(**fields) + + +def get_session_entry(**overrides: Any) -> PresetSessionEntry: + """A session as `dstack preset list` reads it. `model`, `trials_num` and + `created_at` go to the stored state; the rest are entry fields.""" + state_overrides = { + key: overrides.pop(key) + for key in ("model", "trials_num", "created_at") + if key in overrides + } + fields: dict[str, Any] = { + "id": "ab12cd34", + "name": None, + "status": "running", + "trials": get_trials_summary(), + "verification": None, + "constraints": None, + } + fields.update(overrides) + state = get_session_state(id=fields["id"], status=fields["status"], **state_overrides) + return PresetSessionEntry(state=state, **fields) + + def get_successful_preset_report(run: Run) -> PresetAgentSuccess: return PresetAgentSuccess.model_validate( { @@ -277,8 +367,13 @@ def get_successful_preset_report(run: Run) -> PresetAgentSuccess: "trial": 1, "base": "Qwen/Qwen3.5-27B", "model": "community/Qwen3.5-27B-GPTQ-Int4", - "context_length": 32768, - "benchmark": get_preset_benchmark(), + "result": [ + { + "concurrency": 1, + "context_length": 32768, + "benchmark": get_preset_benchmark(), + } + ], } ) diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index e55c3fd013..7728b209a0 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -14,7 +14,12 @@ import pytest import yaml -from dstack._internal.cli.models.preset_agent import PresetAgentInfo, PresetSessionProcess +from dstack._internal.cli.models.preset_agent import ( + PresetAgentInfo, + PresetSessionProcess, + PresetTrial, + PresetTrialResult, +) from dstack._internal.cli.services.presets.agent import ( _prepare_subprocess_command, _terminate_process, @@ -68,6 +73,7 @@ get_fake_api, get_session_run, get_session_state, + get_trial_result, ) @@ -1610,89 +1616,107 @@ def test_refusals(self, tmp_path, monkeypatch, state, match): load_resumable_session(preset_id) -def _write_trials(tmp_path, records): +def _write_trials(tmp_path, trials: list[PresetTrial]): trials_dir = tmp_path / "trials" - for number, record in enumerate(records, 1): + for number, trial in enumerate(trials, 1): (trials_dir / str(number)).mkdir(parents=True) - (trials_dir / str(number) / "trial.json").write_text(json.dumps(record)) + (trials_dir / str(number) / "trial.json").write_text(trial.model_dump_json()) return trials_dir class TestSummarizeSessionTrials: def test_counts_records_even_when_trials_share_a_task(self, tmp_path): - record = { - "resources": {"gpu": {"name": "A40", "memory": "48GB", "count": 1}}, - "benchmark": { - "workload": {"concurrency": 8}, - "metrics": {"duration_seconds": 10.0, "total_output_tokens": 20000}, - }, - } - rerun = json.loads(json.dumps(record)) - rerun["benchmark"]["metrics"]["total_output_tokens"] = 23000 - nameless = {"benchmark": {"workload": {}, "metrics": {}}} - trials_dir = _write_trials(tmp_path, [record, rerun, record, nameless]) + trial = PresetTrial( + resources={"gpu": {"name": "A40", "memory": "48GB", "count": 1}}, + result=[get_trial_result(tok_s=2000.0, concurrency=8)], + ) + rerun = trial.model_copy( + update={"result": [get_trial_result(tok_s=2300.0, concurrency=8)]} + ) + # A trial whose benchmark could not be done, as the agent records it. + unmeasured = PresetTrial(result=[PresetTrialResult(concurrency=8, failed=True)]) + trials_dir = _write_trials(tmp_path, [trial, rerun, trial, unmeasured]) summary = _summarize_session_trials(trials_dir) - assert summary is not None # 4 records = 4 trials: one long-lived task commonly hosts several # trials, so identical records must not collapse the count. - assert summary["count"] == 4 - assert summary["best"] == { - "tok_s": 2300.0, - "tpot_ms": None, - "ttft_ms": None, - "context_length": None, - "concurrency": 8, - "gpu": "A40:48GB:1", - } + assert summary.count == 4 + assert summary.best == rerun + assert summary.gpu == "A40:48GB:1" def test_trials_are_ordered_numerically_beyond_nine(self, tmp_path): # Twelve trials: a string sort would chart 1, 10, 11, 12, 2, ... - records = [ - { - "benchmark": { - "metrics": {"total_output_tokens": float(i * 100), "duration_seconds": 1.0}, - "workload": {"concurrency": 1}, - } - } - for i in range(1, 13) + trials = [ + PresetTrial(result=[get_trial_result(tok_s=float(i * 100))]) for i in range(1, 13) ] - trials_dir = _write_trials(tmp_path, records) + trials_dir = _write_trials(tmp_path, trials) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["series"] == [float(i * 100) for i in range(1, 13)] + assert summary.series == [float(i * 100) for i in range(1, 13)] def test_a_trial_still_in_flight_is_not_counted(self, tmp_path): - record = { - "resources": {}, - "benchmark": { - "workload": {"concurrency": 8}, - "metrics": {"duration_seconds": 10.0, "total_output_tokens": 20000}, - }, - } - trials_dir = _write_trials(tmp_path, [record]) + trial = PresetTrial(resources={}, result=[get_trial_result(tok_s=2000.0, concurrency=8)]) + trials_dir = _write_trials(tmp_path, [trial]) # Trial 2 has started but has no result yet. (trials_dir / "2").mkdir() (trials_dir / "2" / "task.dstack.yml").write_text("type: task\n") summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["count"] == 1 - assert summary["series"] == [2000.0] + assert summary.count == 1 + assert summary.series == [2000.0] def test_a_torn_result_copy_reads_as_still_in_flight(self, tmp_path): trials_dir = _write_trials(tmp_path, []) (trials_dir / "1").mkdir(parents=True) - (trials_dir / "1" / "trial.json").write_text('{"benchmark": {"met') + (trials_dir / "1" / "trial.json").write_text('{"result": [{"benchmark": {"met') + + summary = _summarize_session_trials(trials_dir) + + assert summary.count == 0 + + def test_an_old_failed_trial_without_a_benchmark_is_counted(self, tmp_path): + # Before concurrency sweeps, a failed trial was written with no + # `benchmark` key and no `result` list. + trials_dir = _write_trials(tmp_path, []) + (trials_dir / "1").mkdir(parents=True) + (trials_dir / "1" / "trial.json").write_text( + json.dumps( + { + "resources": {"gpu": {"name": "A40", "memory": "48GB", "count": 1}}, + "failed": True, + } + ) + ) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["count"] == 0 + assert summary.count == 1 + assert summary.series == [None] + assert summary.failed == [True] + assert summary.gpu == "A40:48GB:1" + + def test_a_broken_entry_does_not_drop_the_other_entries(self, tmp_path): + trial = PresetTrial( + result=[ + get_trial_result(tok_s=100.0, concurrency=48), + get_trial_result(tok_s=180.0, concurrency=96), + ] + ) + record = json.loads(trial.model_dump_json()) + record["result"].append({"concurrency": 192, "benchmark": {"metrics": {}}}) + trials_dir = _write_trials(tmp_path, []) + (trials_dir / "1").mkdir(parents=True) + (trials_dir / "1" / "trial.json").write_text(json.dumps(record)) + + summary = _summarize_session_trials(trials_dir) + + assert summary.count == 1 + assert summary.series == [180.0] + assert summary.failed == [False] + assert [r.failed for r in get_or_error(summary.best).result] == [False, False] class TestReadLastSessionVerification: @@ -1710,7 +1734,7 @@ def test_newest_attempt_wins_and_a_missing_directory_is_not_verifying(self, tmp_ (service_dir / "2" / "service.dstack.yml").write_text("type: service\n") # Attempt 2 has no result yet, so it is the one in flight. - assert _read_last_session_verification(service_dir) == {"status": "verifying"} + assert _read_last_session_verification(service_dir) == "verifying" def test_the_newest_result_is_returned(self, tmp_path): # Attempts 9 and 10: a string sort would pick "9" as the newest. @@ -1721,11 +1745,7 @@ def test_the_newest_result_is_returned(self, tmp_path): json.dumps({"trial": number, "run_name": f"p-{number}", "status": status}) ) - assert _read_last_session_verification(service_dir) == { - "trial": 10, - "run_name": "p-10", - "status": "verified", - } + assert _read_last_session_verification(service_dir) == "verified" def test_a_torn_result_copy_reads_as_verifying(self, tmp_path): # The mirror copies whole files, but a reader can still catch one @@ -1734,7 +1754,7 @@ def test_a_torn_result_copy_reads_as_verifying(self, tmp_path): (service_dir / "1").mkdir(parents=True) (service_dir / "1" / "verification.json").write_text('{"trial": 1, "run_na') - assert _read_last_session_verification(service_dir) == {"status": "verifying"} + assert _read_last_session_verification(service_dir) == "verifying" class TestFileLineReader: diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index c6203023f4..2289f0e137 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -826,7 +826,7 @@ def test_max_ttft_reaches_the_constraints(self): def test_throughput_is_derived_not_read(self): # A miscomputed field must not become the number we rank on. preset = get_preset() - benchmark = preset.benchmark + benchmark = get_or_error(preset.result[0].benchmark) benchmark.metrics.output_tok_per_s = 999999.0 benchmark.metrics.per_user_tok_per_s = 999999.0 @@ -863,7 +863,7 @@ def test_renders_defaults_for_the_optional_fields(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": 8, + "concurrency": [8], "input_tokens": 1024, "output_tokens": 1024, "shared_prefix_tokens": 0, @@ -897,7 +897,7 @@ def test_renders_custom_dataset_without_request_shape(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": 8, + "concurrency": [8], "dataset": "spec_bench", "baseline": True, "fleets": ["gpu-fleet"], @@ -927,7 +927,7 @@ def test_renders_configured_values(self): assert data["model"] == {"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"} assert data["min_context_length"] == 32768 assert data["trials_num"] == 10 - assert data["concurrency"] == 16 + assert data["concurrency"] == [16] assert data["fleets"] == ["a", "b"] @@ -1234,7 +1234,7 @@ def test_lists_a_failed_session(self, tmp_path, monkeypatch): lambda: tmp_path, ) - listed = {entry["id"]: entry["status"] for entry in list_preset_sessions()} + listed = {entry.id: entry.status for entry in list_preset_sessions()} assert listed == {"dead0000": "failed", "beef0000": "success"} diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index e5e963388b..93428705e3 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -1,15 +1,23 @@ -import json from datetime import datetime, timedelta, timezone from io import StringIO import pytest from rich.table import Table -from dstack._internal.cli.models.presets import PulledPreset +from dstack._internal.cli.models.preset_agent import PresetTrial, PresetTrialResult +from dstack._internal.cli.models.presets import PresetSessionEntry, PulledPreset from dstack._internal.cli.services.presets import output as output_module from dstack._internal.cli.services.presets.output import _add_session, _format_number from dstack._internal.core.models.presets import PortablePreset, PresetWorkload -from tests._internal.cli.common import get_preset, plain_console +from dstack._internal.utils.common import get_or_error +from tests._internal.cli.common import ( + get_preset, + get_preset_constraints, + get_session_entry, + get_trial_result, + get_trials_summary, + plain_console, +) pytestmark = pytest.mark.windows @@ -28,7 +36,7 @@ def test_small_values_keep_three_significant_digits(self): class TestFormatPresetBenchmark: def test_formats_second_scale_ttft_without_scientific_notation(self): preset = get_preset() - ttft = preset.benchmark.metrics.ttft_ms + ttft = get_or_error(preset.result[0].benchmark).metrics.ttft_ms ttft.mean = 8148.3 ttft.p50 = 8151.4 ttft.p99 = 8334.2 @@ -71,7 +79,7 @@ def test_renders_the_requested_workload_not_the_measured_one(self): # They diverge with a dataset, where the benchmark records measured means. preset = get_preset() preset.configuration.dataset = "spec_bench" - benchmark = preset.benchmark + benchmark = get_or_error(preset.result[0].benchmark) benchmark.workload = PresetWorkload( api=benchmark.workload.api, num_requests=benchmark.workload.num_requests, @@ -140,7 +148,7 @@ def test_prints_submitted_column(self, monkeypatch): assert "2 months ago" in output.getvalue() -def _session_row(session: dict) -> dict: +def _session_row(session: PresetSessionEntry) -> dict: table = Table(box=None) for column in ( "BASE", @@ -163,20 +171,17 @@ def _session_row(session: dict) -> dict: class TestSessionRow: def test_shows_progress_after_status_and_best_benchmark(self): row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "trials_num": 3, - "trials": { - "count": 2, - "best": { - "tok_s": 2339.0, - "tpot_ms": 3.42, - "concurrency": 8, - "gpu": "A40:48GB:1", - }, - }, - } + get_session_entry( + id="c7e18d52", + trials_num=3, + trials=get_trials_summary( + 2, + best=PresetTrial( + result=[get_trial_result(tok_s=2339.0, tpot_ms=3.42, concurrency=8)] + ), + gpu="A40:48GB:1", + ), + ) ) # 2 completed, so the 3rd is the one being trialed. @@ -188,27 +193,25 @@ def test_shows_progress_after_status_and_best_benchmark(self): def test_shows_the_shared_prefix_when_the_workload_has_one(self): row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "constraints": { - "input_tokens": 8192, - "output_tokens": 1024, - "concurrency": 162, - "shared_prefix_tokens": 7373, - }, - } + get_session_entry( + id="c7e18d52", + constraints=get_preset_constraints( + input_tokens=8192, + output_tokens=1024, + concurrency=[162], + shared_prefix_tokens=7373, + ), + ) ) assert row["CONSTRAINTS"] == ("[secondary]io=8K/1K prefix=90% c=162[/]") def test_shows_the_dataset_for_a_session_with_a_custom_dataset(self): row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "constraints": {"dataset": "spec_bench", "concurrency": 4}, - } + get_session_entry( + id="c7e18d52", + constraints=get_preset_constraints(dataset="spec_bench", concurrency=[4]), + ) ) assert row["CONSTRAINTS"] == ("[secondary]data=spec_bench c=4[/]") @@ -217,47 +220,38 @@ def test_omits_the_shared_prefix_when_requests_are_fully_unique(self): # A zero prefix is the default; only a non-zero share is information. # can serve from cache, so a row without it cannot be compared to one with. row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "constraints": { - "input_tokens": 8192, - "output_tokens": 1024, - "concurrency": 162, - "shared_prefix_tokens": 0, - }, - } + get_session_entry( + id="c7e18d52", + constraints=get_preset_constraints( + input_tokens=8192, + output_tokens=1024, + concurrency=[162], + shared_prefix_tokens=0, + ), + ) ) assert "prefix" not in row["CONSTRAINTS"] def test_shows_zero_progress_without_benchmark(self): - row = _session_row( - {"id": "ab12cd34", "status": "running", "trials_num": 3, "trials": {"count": 0}} - ) + row = _session_row(get_session_entry(trials_num=3)) assert row["STATUS"] == "[bold sea_green3]trialing[/] [secondary](1/3)[/]" def test_verifying_keeps_the_completed_count(self): row = _session_row( - { - "id": "ab12cd34", - "status": "running", - "trials_num": 3, - "trials": {"count": 3}, - "verification": {"status": "verifying"}, - } + get_session_entry(trials_num=3, trials=get_trials_summary(3), verification="verifying") ) assert row["STATUS"] == "[bold deep_sky_blue1]verifying[/] [secondary](3/3)[/]" def test_omits_progress_without_trials_data(self): - row = _session_row({"id": "ab12cd34", "status": "interrupted"}) + row = _session_row(get_session_entry(status="interrupted")) assert row["STATUS"] == "[secondary]interrupted[/]" def test_counts_without_trials_num(self): - row = _session_row({"id": "ab12cd34", "status": "interrupted", "trials": {"count": 2}}) + row = _session_row(get_session_entry(status="interrupted", trials=get_trials_summary(2))) assert row["STATUS"] == "[secondary]interrupted[/] [secondary](2)[/]" @@ -271,18 +265,18 @@ def test_sorts_all_rows_newest_first_without_grouping(self, monkeypatch): update={"id": "11aa22bb", "created_at": old.created_at + timedelta(days=2)} ) sessions = [ - { - "id": "aaaaaaaa", - "status": "interrupted", - "model": old.base, - "created_at": "2026-07-01T00:00:00Z", - }, - { - "id": "bbbbbbbb", - "status": "interrupted", - "model": old.base, - "created_at": "2026-07-02T00:00:00Z", - }, + get_session_entry( + id="aaaaaaaa", + status="interrupted", + model=old.base, + created_at=datetime(2026, 7, 1, tzinfo=timezone.utc), + ), + get_session_entry( + id="bbbbbbbb", + status="interrupted", + model=old.base, + created_at=datetime(2026, 7, 2, tzinfo=timezone.utc), + ), ] output_module.print_presets([old, new], sessions=sessions) @@ -311,13 +305,13 @@ def test_saved_preset_owns_row_regardless_of_session_status(self, monkeypatch, s monkeypatch.setattr(output_module, "console", plain_console(buffer, width=200)) preset = get_preset() sessions = [ - { - "id": preset.id, - "status": session_status, - "model": preset.base, - "trials_num": 4, - "trials": {"count": 3}, - } + get_session_entry( + id=preset.id, + status=session_status, + model=preset.base, + trials_num=4, + trials=get_trials_summary(3), + ) ] output_module.print_presets([preset], sessions=sessions) @@ -331,13 +325,9 @@ def test_saved_preset_owns_row_regardless_of_session_status(self, monkeypatch, s class TestVerifyingStatus: def test_running_session_with_a_verification_record_shows_verifying(self): row = _session_row( - { - "id": "ab12cd34", - "status": "running", - "trials_num": 10, - "trials": {"count": 6}, - "verification": {"trial": 3, "run_name": "p-2", "status": "verifying"}, - } + get_session_entry( + trials_num=10, trials=get_trials_summary(6), verification="verifying" + ) ) # The trial budget is not spent, which the old inference required. @@ -345,30 +335,22 @@ def test_running_session_with_a_verification_record_shows_verifying(self): def test_a_failed_attempt_still_counts_as_verifying(self): row = _session_row( - { - "id": "ab12cd34", - "status": "running", - "trials_num": 2, - "trials": {"count": 2}, - "verification": {"trial": 3, "status": "failed", "reason": "probe never passed"}, - } + get_session_entry(trials_num=2, trials=get_trials_summary(2), verification="failed") ) assert row["STATUS"].startswith("[bold deep_sky_blue1]verifying[/]") def test_a_spent_trial_budget_alone_stays_trialing(self): - row = _session_row( - {"id": "ab12cd34", "status": "running", "trials_num": 2, "trials": {"count": 2}} - ) + row = _session_row(get_session_entry(trials_num=2, trials=get_trials_summary(2))) assert row["STATUS"].startswith("[bold sea_green3]trialing[/]") -def _write_trials(tmp_path, records): +def _write_trials(tmp_path, trials: list[PresetTrial]): trials_dir = tmp_path / "trials" - for number, record in enumerate(records, 1): + for number, trial in enumerate(trials, 1): (trials_dir / str(number)).mkdir(parents=True) - (trials_dir / str(number) / "trial.json").write_text(json.dumps(record)) + (trials_dir / str(number) / "trial.json").write_text(trial.model_dump_json()) return trials_dir @@ -378,28 +360,19 @@ def test_a_failed_trial_keeps_its_benchmark_but_never_becomes_best(self, tmp_pat # The failed trial is the fastest. It broke `max_ttft`, so promoting it # would put a non-compliant configuration at the top of the listing. - trials_dir = _write_trials( - tmp_path, - [ - { - "benchmark": { - "metrics": {"total_output_tokens": tokens, "duration_seconds": 1.0}, - "workload": {"concurrency": 8}, - }, - **({"failed": True} if failed else {}), - } - for tokens, failed in ((100.0, False), (900.0, True), (300.0, False)) - ], - ) + trials = [ + PresetTrial(result=[get_trial_result(tok_s=tokens, concurrency=8, failed=failed)]) + for tokens, failed in ((100.0, False), (900.0, True), (300.0, False)) + ] + trials_dir = _write_trials(tmp_path, trials) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["count"] == 3 + assert summary.count == 3 # Still charted: the trial happened and its number is real. - assert summary["series"] == [100.0, 900.0, 300.0] - assert summary["best"]["tok_s"] == 300.0 - assert summary["best_failed"]["tok_s"] == 900.0 + assert summary.series == [100.0, 900.0, 300.0] + assert summary.best == trials[2] + assert summary.best_failed == trials[1] def test_reports_the_gpu_when_no_trial_produced_a_benchmark(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials @@ -408,28 +381,36 @@ def test_reports_the_gpu_when_no_trial_produced_a_benchmark(self, tmp_path): # the only thing left that knows what the run was on. trials_dir = _write_trials( tmp_path, - [{"resources": {"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}}], + [ + PresetTrial( + resources={"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}, + result=[PresetTrialResult(concurrency=4, failed=True)], + ) + ], ) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["best"] is None - assert summary["best_failed"] is None - assert summary["gpu"] == "MI300X:192GB:1" + assert summary.best is None + assert summary.best_failed is None + assert summary.gpu == "MI300X:192GB:1" def test_reports_the_gpu_of_a_single_node_group(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials trials_dir = _write_trials( tmp_path, - [{"groups": [[{"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}]]}], + [ + PresetTrial( + groups=[[{"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}]], + result=[get_trial_result()], + ) + ], ) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["gpu"] == "MI300X:192GB:1" + assert summary.gpu == "MI300X:192GB:1" def test_counts_the_worker_nodes_of_a_disaggregated_trial(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials @@ -438,13 +419,17 @@ def test_counts_the_worker_nodes_of_a_disaggregated_trial(self, tmp_path): h200 = {"gpu": {"name": "H200", "memory": "141GB", "count": 8}} trials_dir = _write_trials( tmp_path, - [{"groups": [[{"cpu": "16"}], [h200], [h200, h200]]}], + [ + PresetTrial( + groups=[[{"cpu": "16"}], [h200], [h200, h200]], + result=[get_trial_result()], + ) + ], ) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["gpu"] == "H200:141GB:8 x3" + assert summary.gpu == "H200:141GB:8 x3" def test_shows_the_split_when_roles_ran_different_gpus(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials @@ -453,72 +438,64 @@ def test_shows_the_split_when_roles_ran_different_gpus(self, tmp_path): trials_dir = _write_trials( tmp_path, [ - { - "groups": [ + PresetTrial( + groups=[ [{"gpu": {"name": "H200", "memory": "141GB", "count": 8}}], [h100, h100], - ] - } + ], + result=[get_trial_result()], + ) ], ) summary = _summarize_session_trials(trials_dir) - assert summary is not None # Group order, not sorted: the roles read in the order they ran. - assert summary["gpu"] == "H200:141GB:8 + H100:80GB:8 x2" + assert summary.gpu == "H200:141GB:8 + H100:80GB:8 x2" def test_the_fastest_failed_trial_is_kept_when_nothing_passed(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials - trials_dir = _write_trials( - tmp_path, - [ - { - "benchmark": { - "metrics": { - "total_output_tokens": tokens, - "duration_seconds": 1.0, - "tpot_ms": {"mean": 41.7, "p50": 34.4}, - "ttft_ms": {"p50": 4300.0}, - }, - "workload": {"concurrency": 4}, - }, - "failed": True, - } - for tokens in (100.0, 300.0, 200.0) - ], - ) + trials = [ + PresetTrial( + result=[ + get_trial_result( + tok_s=tokens, tpot_ms=41.7, ttft_ms=4300.0, concurrency=4, failed=True + ) + ] + ) + for tokens in (100.0, 300.0, 200.0) + ] + trials_dir = _write_trials(tmp_path, trials) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["best"] is None - assert summary["best_failed"]["tok_s"] == 300.0 - # Mean, not p50: MTP skews TPOT right, and mean is the delivered rate. - assert summary["best_failed"]["tpot_ms"] == 41.7 + assert summary.best is None + assert summary.best_failed == trials[1] def test_a_run_that_met_nothing_still_shows_what_it_measured(self): row = _session_row( - { - "id": "ab12cd34", - "status": "running", - "constraints": {"input_tokens": 10000, "output_tokens": 1500}, - "trials": { - "count": 1, - "series": [100.0], - "failed": [True], - "best": None, - "best_failed": { - "tok_s": 100.0, - "tpot_ms": 34.4, - "ttft_ms": 4300.0, - "context_length": 1048576, - "concurrency": 4, - "gpu": "MI300X:192GB:1", - }, - }, - } + get_session_entry( + constraints=get_preset_constraints(input_tokens=10000, output_tokens=1500), + trials=get_trials_summary( + 1, + series=[100.0], + failed=[True], + best_failed=PresetTrial( + result=[ + get_trial_result( + tok_s=100.0, + tpot_ms=34.4, + ttft_ms=4300.0, + concurrency=4, + context_length=1048576, + failed=True, + ) + ] + ), + gpu="MI300X:192GB:1", + ), + ) ) assert "ttft=4.3s" in row["BENCHMARK"] @@ -528,19 +505,90 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): assert "*tps/user" in row["BENCHMARK"] +class TestSweepRow: + def test_shows_the_passing_sweep_of_the_best_trial_as_ranges(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + # The trial broke `max_ttft` at 192 only, so the row ranges over 48 and 96. + trial = PresetTrial( + result=[ + get_trial_result( + tok_s=100, tpot_ms=10.0, ttft_ms=400.0, concurrency=48, context_length=32768 + ), + get_trial_result( + tok_s=180, tpot_ms=20.0, ttft_ms=800.0, concurrency=96, context_length=32768 + ), + get_trial_result( + tok_s=200, + tpot_ms=40.0, + ttft_ms=6000.0, + concurrency=192, + context_length=32768, + failed=True, + ), + ] + ) + summary = _summarize_session_trials(_write_trials(tmp_path, [trial])) + + row = _session_row( + get_session_entry( + trials=summary, constraints=get_preset_constraints(concurrency=[48, 96, 192]) + ) + ) + + assert row["CONSTRAINTS"] == "[secondary]io=1K/1K c=48..96[/]" + assert row["BENCHMARK"] == "tps/user=50..100 ttft=400ms..800ms ctx=32K" + + def test_falls_back_to_the_whole_failed_sweep_when_nothing_passed(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + trial = PresetTrial( + result=[ + get_trial_result( + tok_s=100, + tpot_ms=10.0, + ttft_ms=400.0, + concurrency=48, + context_length=32768, + failed=True, + ), + get_trial_result( + tok_s=180, + tpot_ms=20.0, + ttft_ms=800.0, + concurrency=96, + context_length=32768, + failed=True, + ), + get_trial_result( + tok_s=200, + tpot_ms=40.0, + ttft_ms=6000.0, + concurrency=192, + context_length=32768, + failed=True, + ), + ] + ) + summary = _summarize_session_trials(_write_trials(tmp_path, [trial])) + + row = _session_row( + get_session_entry( + trials=summary, constraints=get_preset_constraints(concurrency=[48, 96, 192]) + ) + ) + + assert row["CONSTRAINTS"] == "[secondary]io=1K/1K c=48..192[/]" + assert row["BENCHMARK"] == "[secondary]*tps/user=25..100 ttft=400ms..6s ctx=32K[/]" + + class TestFailedTrialSpark: def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): # Its number is real, so it earns a bar rather than a `·`, and green goes # to the best trial that meets the constraints even on a lower number. - session = { - "id": "ab12cd34", - "status": "running", - "trials": { - "count": 3, - "series": [100.0, 900.0, 300.0], - "failed": [False, True, False], - }, - } + session = get_session_entry( + trials=get_trials_summary(3, series=[100.0, 900.0, 300.0], failed=[False, True, False]) + ) spark = output_module._format_trial_spark(session) @@ -552,15 +600,9 @@ def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): def test_the_best_failed_trial_is_gold_while_none_passes(self): # With nothing meeting the constraints, the best result so far is still # what the run has to show; the rest are context. - session = { - "id": "ab12cd34", - "status": "running", - "trials": { - "count": 3, - "series": [100.0, 900.0, 300.0], - "failed": [True, True, True], - }, - } + session = get_session_entry( + trials=get_trials_summary(3, series=[100.0, 900.0, 300.0], failed=[True, True, True]) + ) spark = output_module._format_trial_spark(session) diff --git a/src/tests/_internal/cli/services/presets/test_report_schema.py b/src/tests/_internal/cli/services/presets/test_report_schema.py index 9298e06d2a..4fc79f8734 100644 --- a/src/tests/_internal/cli/services/presets/test_report_schema.py +++ b/src/tests/_internal/cli/services/presets/test_report_schema.py @@ -7,6 +7,7 @@ ) from dstack._internal.cli.services.presets.verify import _AGENT_RESULT_ADAPTER from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.utils.common import get_or_error pytestmark = pytest.mark.windows @@ -18,8 +19,7 @@ "trial": None, "base": None, "model": None, - "context_length": None, - "benchmark": None, + "result": None, "failure_summary": None, } # Keywords the OpenAI strict structured-output mode rejects. @@ -105,32 +105,37 @@ def test_keeps_the_agent_facing_shape(self): "trial": 1, "base": "Qwen/Qwen2.5-0.5B-Instruct", "model": "Qwen/Qwen2.5-0.5B-Instruct", - "context_length": 2048, - "benchmark": { - "tool": "vllm.bench serve", - "tool_version": "0.27.0", - "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", - "workload": { - "api": "chat_completions", - "dataset": None, - "num_requests": 32, - "input_tokens": 128, - "output_tokens": 64, + "result": [ + { "concurrency": 1, - "shared_prefix_tokens": 0, - }, - "metrics": { - "successful_requests": 32, - "failed_requests": 0, - "duration_seconds": 40.5, - "total_input_tokens": 4096, - "total_output_tokens": 2048, - "output_tok_per_s": 50.6, - "per_user_tok_per_s": 50.6, - "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, - "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, - }, - }, + "context_length": 2048, + "benchmark": { + "tool": "vllm.bench serve", + "tool_version": "0.27.0", + "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", + "workload": { + "api": "chat_completions", + "dataset": None, + "num_requests": 32, + "input_tokens": 128, + "output_tokens": 64, + "concurrency": 1, + "shared_prefix_tokens": 0, + }, + "metrics": { + "successful_requests": 32, + "failed_requests": 0, + "duration_seconds": 40.5, + "total_input_tokens": 4096, + "total_output_tokens": 2048, + "output_tok_per_s": 50.6, + "per_user_tok_per_s": 50.6, + "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, + "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, + }, + }, + } + ], }, PresetAgentSuccess, ), @@ -146,5 +151,6 @@ def test_reports_codex_produced_against_the_schema_parse(self, data, expected): assert isinstance(report, expected) if isinstance(report, PresetAgentSuccess): assert isinstance(report.service_yaml, ServiceConfiguration) - assert report.benchmark.workload.dataset is None - assert report.benchmark.workload.shared_prefix_tokens == 0 + workload = get_or_error(report.result[0].benchmark).workload + assert workload.dataset is None + assert workload.shared_prefix_tokens == 0 diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index fb774a68ff..f3a1ee92cd 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -114,14 +114,14 @@ def test_loads_a_preset_that_requested_the_random_dataset(self, tmp_path: Path): path = tmp_path / "presets" / preset.id / "preset.yml" data = yaml.safe_load(path.read_text()) data["configuration"]["dataset"] = "random" - data["benchmark"]["workload"]["dataset"] = "random" - data["benchmark"]["workload"]["shared_prefix_tokens"] = 0 + data["result"][0]["benchmark"]["workload"]["dataset"] = "random" + data["result"][0]["benchmark"]["workload"]["shared_prefix_tokens"] = 0 path.write_text(yaml.safe_dump(data, sort_keys=False)) loaded = get_verified_preset(store, preset.id) assert loaded.configuration.dataset is None - assert loaded.benchmark.workload.dataset is None + assert get_or_error(loaded.result[0].benchmark).workload.dataset is None def test_loads_a_preset_that_predates_the_repo_field(self, tmp_path: Path): # Presets written before the rename store the served repo as `model`. @@ -226,7 +226,7 @@ def test_reports_one_arms_errors_for_an_unreadable_preset(self, tmp_path: Path): store.save(preset) path = tmp_path / "presets" / preset.id / "preset.yml" data = yaml.safe_load(path.read_text()) - del data["benchmark"] + del data["result"] path.write_text(yaml.safe_dump(data, sort_keys=False)) with pytest.raises(CLIError) as excinfo: @@ -236,7 +236,7 @@ def test_reports_one_arms_errors_for_an_unreadable_preset(self, tmp_path: Path): # Every reported error is located under the tag the file claims; the # untagged union used to add the other arm's errors on top. assert "1 validation error for" in message - assert "verified.benchmark" in message + assert "verified.result" in message assert "published." not in message def test_saving_same_id_overwrites_existing_preset(self, tmp_path: Path): @@ -244,7 +244,12 @@ def test_saving_same_id_overwrites_existing_preset(self, tmp_path: Path): preset = get_preset() store.save(preset) - updated = preset.model_copy(update={"base": "Qwen/Another-Model", "context_length": 16384}) + updated = preset.model_copy( + update={ + "base": "Qwen/Another-Model", + "result": [preset.result[0].model_copy(update={"context_length": 16384})], + } + ) store.save(updated) assert store.get(preset.id) == updated @@ -289,7 +294,7 @@ def test_skips_invalid_preset_on_list_but_keeps_it_deletable(self, tmp_path: Pat store.save(valid) path = store.save(get_preset()) data = yaml.safe_load(path.read_text()) - data["benchmark"].pop("metrics") + data["result"][0]["benchmark"].pop("metrics") path.write_text(yaml.safe_dump(data, sort_keys=False)) # One corrupt file must not take down every read; the warning goes to @@ -383,7 +388,7 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert preset.configuration.input_tokens == 10000 assert preset.configuration.output_tokens == 1500 assert preset.configuration.shared_prefix_tokens == 0 - assert preset.configuration.concurrency == 4 + assert preset.configuration.concurrency == [4] # `priority` is not an excluded field, so the regrouping keeps it. assert preset.service.priority == 0 assert [group.name for group in preset.verified_on] == [ @@ -391,7 +396,7 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): ] assert get_or_error(preset.verified_on[0].replicas[0].gpu).name == ["MI300X"] # The old format never recorded a dataset name for a synthetic workload. - assert preset.benchmark.workload.dataset is None + assert get_or_error(preset.result[0].benchmark).workload.dataset is None assert store.list() == [preset] def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index c7a4d9ea80..9498dc8ccd 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -21,7 +21,7 @@ from dstack._internal.core.models.configurations import PresetConfiguration from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.files import FilePathMapping -from dstack._internal.core.models.presets import PresetWorkload +from dstack._internal.core.models.presets import PresetResult, PresetWorkload from dstack._internal.core.models.profiles import ProfileParams from dstack._internal.core.models.runs import Run from dstack._internal.utils.common import get_or_error @@ -60,8 +60,9 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): verification_replica_groups=base.verified_on, base_model="Qwen/Qwen3.5-27B", repo="community/Qwen3.5-27B-GPTQ-Int4", - context_length=32768, - benchmark=get_preset_benchmark(), + result=[ + PresetResult(concurrency=1, context_length=32768, benchmark=get_preset_benchmark()) + ], configuration=configuration, best_trial=1, preset_id="8f3a12c4", @@ -76,12 +77,12 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): assert preset.configuration.spot_policy is None assert "secret" not in preset.model_dump_json() - def test_successful_report_requires_benchmark(self): + def test_successful_report_requires_result(self): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() - data.pop("benchmark") + data.pop("result") - with pytest.raises(ValidationError, match="benchmark"): + with pytest.raises(ValidationError, match="result"): validate_extra_ignore(AnyPresetAgentResult, data) def test_builds_portable_self_contained_preset(self, tmp_path): @@ -109,7 +110,7 @@ def test_builds_portable_self_contained_preset(self, tmp_path): assert preset.base == "Qwen/Qwen3.5-27B" assert preset.repo == "community/Qwen3.5-27B-GPTQ-Int4" - assert preset.context_length == 32768 + assert preset.result[0].context_length == 32768 assert preset.created_at == created_at assert preset.service.name is None assert preset.service.gateway is None @@ -202,8 +203,9 @@ def test_verifies_a_shared_prefix_benchmark_named_by_the_benchmark_tool(self, tm # The stored workload states the request: no dataset was asked for, so # none is recorded — the tool's own name for its generated data stays in # `command` — while the requested prefix survives into the record. - assert preset.benchmark.workload.dataset is None - assert preset.benchmark.workload.shared_prefix_tokens == 130048 + workload = get_or_error(preset.result[0].benchmark).workload + assert workload.dataset is None + assert workload.shared_prefix_tokens == 130048 def test_rejects_a_benchmark_without_the_requested_shared_prefix(self, tmp_path): run = get_running_service_run() @@ -226,7 +228,7 @@ def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): run = get_running_service_run() with pytest.raises( - CLIError, match="concurrency of 8 does not match the requested concurrency of 4" + CLIError, match=r"concurrency of 8 does not match the requested concurrency of \[4\]" ): build_verified_preset( run=run, @@ -239,6 +241,95 @@ def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) + def test_accepts_a_sweep_benchmarked_at_some_concurrencies_only(self, tmp_path): + # Of the requested 48, 96 and 192, the agent could benchmark 48 only. + run = get_running_service_run() + + preset = build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=_sweep_report(run, measured=(48,)), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert [r.concurrency for r in preset.result] == [48, 96, 192] + assert [r.benchmark is not None for r in preset.result] == [True, False, False] + + def test_rejects_a_sweep_with_no_completed_benchmark(self, tmp_path): + run = get_running_service_run() + + with pytest.raises(CLIError, match="no completed benchmark"): + build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=_sweep_report(run, measured=()), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_rejects_a_sweep_with_two_entries_at_the_same_concurrency(self, tmp_path): + run = get_running_service_run() + report = _sweep_report(run, measured=(48,)) + report = report.model_copy(update={"result": [report.result[0], *report.result]}) + + with pytest.raises(CLIError, match="more than one entry for the same concurrency"): + build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_rejects_an_unmeasured_entry_at_an_unrequested_concurrency(self, tmp_path): + run = get_running_service_run() + report = _sweep_report(run, measured=(48,)) + unrequested = report.result[2].model_copy(update={"concurrency": 500}) + report = report.model_copy(update={"result": [*report.result[:2], unrequested]}) + + with pytest.raises( + CLIError, + match=r"concurrency of 500 does not match the requested concurrency of \[48, 96, 192\]", + ): + build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_rejects_a_measured_entry_without_a_context_length(self, tmp_path): + run = get_running_service_run() + report = _sweep_report(run, measured=(48,)) + measured = report.result[0].model_copy(update={"context_length": None}) + report = report.model_copy(update={"result": [measured, *report.result[1:]]}) + + with pytest.raises(CLIError, match="benchmark at concurrency 48 but no context length"): + build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): run = get_running_service_run() report = _dataset_report(run, dataset="spec_bench") @@ -256,7 +347,7 @@ def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - assert preset.benchmark.workload.dataset == "spec_bench" + assert get_or_error(preset.result[0].benchmark).workload.dataset == "spec_bench" def test_stores_a_dataset_benchmark_without_a_stray_shared_prefix(self, tmp_path): # The mirror of the synthetic case: a dataset defines its own requests, @@ -282,8 +373,9 @@ def test_stores_a_dataset_benchmark_without_a_stray_shared_prefix(self, tmp_path created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - assert preset.benchmark.workload.dataset == "spec_bench" - assert preset.benchmark.workload.shared_prefix_tokens == 0 + workload = get_or_error(preset.result[0].benchmark).workload + assert workload.dataset == "spec_bench" + assert workload.shared_prefix_tokens == 0 # A named dataset is the one dataset name the report and the request share, so # the report either echoes it or does not answer the request. @@ -346,22 +438,23 @@ def test_redacts_known_secret_in_benchmark_command_instead_of_failing(self, tmp_ run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["result"][0]["benchmark"]["command"] = ( "python bench.py --header 'Authorization: Bearer sk-live-0123456789abcdef'" ) report = self._load(tmp_path, data, redacted_values=("sk-live-0123456789abcdef",)) assert isinstance(report, PresetAgentSuccess) - assert report.benchmark is not None - assert report.benchmark.command.endswith("Bearer [redacted]'") - assert "sk-live" not in report.benchmark.command + benchmark = report.result[0].benchmark + assert benchmark is not None + assert benchmark.command.endswith("Bearer [redacted]'") + assert "sk-live" not in benchmark.command def test_still_rejects_unknown_bearer_token(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["result"][0]["benchmark"]["command"] = ( "curl -H 'Authorization: Bearer sk-unknown-9876543210fedcba'" ) @@ -374,7 +467,7 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["result"][0]["benchmark"]["command"] = ( "./benchenv/bin/python bench_service.py --base $DSTACK_SERVER_URL/x" " (auth via DSTACK_TOKEN bearer header from env)" ) @@ -382,8 +475,9 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): report = self._load(tmp_path, data, redacted_values=()) assert isinstance(report, PresetAgentSuccess) - assert report.benchmark is not None - assert "bearer header" in report.benchmark.command + benchmark = report.result[0].benchmark + assert benchmark is not None + assert "bearer header" in benchmark.command def _shared_prefix_configuration() -> PresetConfiguration: @@ -422,7 +516,41 @@ def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: ) +def _sweep_configuration() -> PresetConfiguration: + return PresetConfiguration.model_validate( + { + "type": "preset", + "base": "Qwen/Qwen3.5-27B", + "min_context_length": 8192, + "max_ttft": 5000, + "trials": 2, + "concurrency": [48, 96, 192], + } + ) + + +def _sweep_report(run: Run, *, measured: tuple[int, ...]) -> PresetAgentSuccess: + """A report over the sweep 48, 96, 192 with a benchmark at `measured` only.""" + result = [] + for concurrency in (48, 96, 192): + benchmark = None + if concurrency in measured: + benchmark = get_preset_benchmark() + benchmark.workload = benchmark.workload.model_copy(update={"concurrency": concurrency}) + result.append( + PresetResult(concurrency=concurrency, context_length=32768, benchmark=benchmark) + ) + return get_successful_preset_report(run).model_copy(update={"result": result}) + + def _report_with_workload(run: Run, workload: dict[str, Any]) -> PresetAgentSuccess: benchmark = get_preset_benchmark() benchmark.workload = PresetWorkload.model_validate(workload) - return get_successful_preset_report(run).model_copy(update={"benchmark": benchmark}) + # The entry states the same concurrency as its benchmark, so the test + # exercises the request check and not the entry/benchmark consistency one. + entry = PresetResult( + concurrency=benchmark.workload.concurrency, + context_length=32768, + benchmark=benchmark, + ) + return get_successful_preset_report(run).model_copy(update={"result": [entry]})