diff --git a/mkdocs/docs/concepts/metrics.md b/mkdocs/docs/concepts/metrics.md index bf91901db0..1b6bfd0f66 100644 --- a/mkdocs/docs/concepts/metrics.md +++ b/mkdocs/docs/concepts/metrics.md @@ -20,6 +20,7 @@ This tab displays key CPU, memory, and GPU metrics collected during the last hou As an alternative to the UI, you can track essential metrics via the CLI. The `dstack metrics` command charts CPU, memory, and GPU utilization over the last hour. +For jobs on NVIDIA GPUs, it adds a `POWER` column that charts power draw against each GPU's enforced power limit.
@@ -126,6 +127,7 @@ telemetry, and more. | `dstack_job_gpu_usage_ratio` | *gauge* | Job GPU usage, percent (as 0.0-1.0) | `0.93` | | `dstack_job_gpu_memory_total_bytes` | *gauge* | Total GPU memory allocated for the job, bytes | `8589934592.0` | | `dstack_job_gpu_memory_usage_bytes` | *gauge* | GPU memory used by the job, bytes | `1048576.0` | + | `dstack_job_gpu_power_draw_watts` | *gauge* | Power draw of the job's GPU, watts (NVIDIA only) | `285.43` | | `DCGM_FI_DEV_GPU_UTIL` | *gauge* | GPU utilization (in %) | | | `DCGM_FI_DEV_MEM_COPY_UTIL` | *gauge* | Memory utilization (in %) | | | `DCGM_FI_DEV_ENC_UTIL` | *gauge* | Encoder utilization (in %) | | diff --git a/runner/internal/runner/metrics/metrics.go b/runner/internal/runner/metrics/metrics.go index 34f22c731a..688d91dce3 100644 --- a/runner/internal/runner/metrics/metrics.go +++ b/runner/internal/runner/metrics/metrics.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "math" "os" "os/exec" "path" @@ -159,13 +160,13 @@ func (s *MetricsCollector) GetGPUMetrics(ctx context.Context) ([]schemas.GPUMetr } func (s *MetricsCollector) GetNVIDIAGPUMetrics(ctx context.Context) ([]schemas.GPUMetrics, error) { - cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=memory.used,utilization.gpu", "--format=csv,noheader,nounits") + cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=memory.used,utilization.gpu,power.draw,enforced.power.limit", "--format=csv,noheader,nounits") var out bytes.Buffer cmd.Stdout = &out if err := cmd.Run(); err != nil { return []schemas.GPUMetrics{}, fmt.Errorf("failed to execute nvidia-smi: %w", err) } - return parseNVIDIASMILikeMetrics(out.String()) + return parseNVIDIASMILikeMetrics(out.String(), true) } func (s *MetricsCollector) GetAMDGPUMetrics(ctx context.Context) ([]schemas.GPUMetrics, error) { @@ -241,16 +242,22 @@ func (s *MetricsCollector) GetIntelAcceleratorMetrics(ctx context.Context) ([]sc if err := cmd.Run(); err != nil { return []schemas.GPUMetrics{}, fmt.Errorf("failed to execute hl-smi: %w", err) } - return parseNVIDIASMILikeMetrics(out.String()) + return parseNVIDIASMILikeMetrics(out.String(), false) } -func parseNVIDIASMILikeMetrics(output string) ([]schemas.GPUMetrics, error) { +// parseNVIDIASMILikeMetrics parses memory used and utilization, followed by power draw and +// limit if withPower is set (nvidia-smi is queried for them, hl-smi is not). +func parseNVIDIASMILikeMetrics(output string, withPower bool) ([]schemas.GPUMetrics, error) { metrics := []schemas.GPUMetrics{} + columns := 2 + if withPower { + columns = 4 + } lines := strings.Split(strings.TrimSpace(output), "\n") for _, line := range lines { parts := strings.Split(line, ", ") - if len(parts) != 2 { + if len(parts) != columns { continue } memUsed, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64) @@ -261,11 +268,27 @@ func parseNVIDIASMILikeMetrics(output string) ([]schemas.GPUMetrics, error) { if err != nil { return metrics, fmt.Errorf("failed to parse accelerator utilization: %w", err) } - metrics = append(metrics, schemas.GPUMetrics{ + gpuMetrics := schemas.GPUMetrics{ GPUMemoryUsage: memUsed * 1024 * 1024, // Convert MiB to bytes GPUUtil: utilization, - }) + } + if withPower { + gpuMetrics.GPUPowerDraw = parseWatts(parts[2]) + gpuMetrics.GPUPowerLimit = parseWatts(parts[3]) + } + metrics = append(metrics, gpuMetrics) } return metrics, nil } + +// parseWatts returns nil for readings the GPU does not provide, which nvidia-smi prints as +// [N/A], [Not Supported] or [Unknown Error], so that they do not drop the other metrics. +// NaN and infinity are dropped too, as they cannot be encoded in the JSON response. +func parseWatts(value string) *float64 { + watts, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + if err != nil || math.IsNaN(watts) || math.IsInf(watts, 0) { + return nil + } + return &watts +} diff --git a/runner/internal/runner/metrics/metrics_test.go b/runner/internal/runner/metrics/metrics_test.go index 844b02bd30..f1cf1bb737 100644 --- a/runner/internal/runner/metrics/metrics_test.go +++ b/runner/internal/runner/metrics/metrics_test.go @@ -1,6 +1,7 @@ package metrics import ( + "encoding/json" "testing" "github.com/dstackai/dstack/runner/internal/runner/schemas" @@ -45,3 +46,56 @@ func TestGetAMDGPUMetrics_ErrorGPUUtilNA(t *testing.T) { assert.ErrorContains(t, err, "GPU utilization is N/A") assert.Nil(t, metrics) } + +func TestParseNVIDIASMILikeMetrics(t *testing.T) { + watts := func(w float64) *float64 { return &w } + + cases := []struct { + name string + output string + withPower bool + expected []schemas.GPUMetrics + }{ + { + // NVIDIA driver 620.70.06, Tesla T4 under load + name: "nvidia-smi", + output: "525, 100, 68.23, 70.00\n", + withPower: true, + expected: []schemas.GPUMetrics{ + {GPUMemoryUsage: 550502400, GPUUtil: 100, GPUPowerDraw: watts(68.23), GPUPowerLimit: watts(70)}, + }, + }, + { + // the T4 at idle, then made-up lines for GPUs without power readings + name: "nvidia-smi, GPUs without power readings", + output: "525, 0, 26.88, 70.00\n1024, 5, [N/A], [N/A]\n2048, 7, [Not Supported], [Not Supported]\n" + + "512, 9, [Unknown Error], 70.00\n256, 1, NaN, Inf\n", + withPower: true, + expected: []schemas.GPUMetrics{ + {GPUMemoryUsage: 550502400, GPUUtil: 0, GPUPowerDraw: watts(26.88), GPUPowerLimit: watts(70)}, + {GPUMemoryUsage: 1073741824, GPUUtil: 5}, + {GPUMemoryUsage: 2147483648, GPUUtil: 7}, + {GPUMemoryUsage: 536870912, GPUUtil: 9, GPUPowerLimit: watts(70)}, + {GPUMemoryUsage: 268435456, GPUUtil: 1}, + }, + }, + { + name: "hl-smi", + output: "512, 3\n", + expected: []schemas.GPUMetrics{ + {GPUMemoryUsage: 536870912, GPUUtil: 3}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + metrics, err := parseNVIDIASMILikeMetrics(tc.output, tc.withPower) + assert.NoError(t, err) + assert.Equal(t, tc.expected, metrics) + // a value json.Marshal rejects would fail the whole metrics response + _, err = json.Marshal(metrics) + assert.NoError(t, err) + }) + } +} diff --git a/runner/internal/runner/schemas/schemas.go b/runner/internal/runner/schemas/schemas.go index c9102d732d..346c0b5981 100644 --- a/runner/internal/runner/schemas/schemas.go +++ b/runner/internal/runner/schemas/schemas.go @@ -148,8 +148,10 @@ type HealthcheckResponse struct { } type GPUMetrics struct { - GPUMemoryUsage uint64 `json:"gpu_memory_usage_bytes"` - GPUUtil uint64 `json:"gpu_util_percent"` + GPUMemoryUsage uint64 `json:"gpu_memory_usage_bytes"` + GPUUtil uint64 `json:"gpu_util_percent"` + GPUPowerDraw *float64 `json:"gpu_power_draw_watts,omitempty"` + GPUPowerLimit *float64 `json:"gpu_power_limit_watts,omitempty"` } type SystemMetrics struct { diff --git a/src/dstack/_internal/cli/utils/metrics.py b/src/dstack/_internal/cli/utils/metrics.py index f921d18f02..f98fba5141 100644 --- a/src/dstack/_internal/cli/utils/metrics.py +++ b/src/dstack/_internal/cli/utils/metrics.py @@ -28,17 +28,28 @@ NOW = "now" LIVE_THRESHOLD = timedelta(seconds=3 * WATCH_INTERVAL_SECONDS) -_FIXED_COLUMNS = 30 +_FIXED_COLUMNS = 32 """Everything but the sparklines and the job label: the `gpu=N` column, both numbers, and -the table's padding. Hand-measured against a `589GB/1480GB`-sized number; a wider one -overflows and Rich ellipsizes the labels rather than shrinking the sparklines.""" +the table's padding. Measured against `100%` and a `9999GB/9999GB`-sized number, which fits +hosts under 10 TB; a wider one overflows and Rich ellipsizes the labels rather than shrinking +the sparklines.""" + +_POWER_FIXED_COLUMNS = 14 +"""What the power column adds besides its sparkline: a `1000W/1000W`-sized number, the space +before it, and the column's padding.""" _SPARKLINE_COLUMNS = 2 -def _spark_width(console_width: int, label_width: int = 0) -> int: - budget = console_width - _FIXED_COLUMNS - label_width - return max(MIN_SPARK_WIDTH, min(MAX_SPARK_WIDTH, budget // _SPARKLINE_COLUMNS)) +def _spark_budget(console_width: int, label_width: int, show_power: bool) -> int: + fixed = _FIXED_COLUMNS + (_POWER_FIXED_COLUMNS if show_power else 0) + columns = _SPARKLINE_COLUMNS + (1 if show_power else 0) + return (console_width - fixed - label_width) // columns + + +def _spark_width(console_width: int, label_width: int = 0, show_power: bool = False) -> int: + budget = _spark_budget(console_width, label_width, show_power) + return max(MIN_SPARK_WIDTH, min(MAX_SPARK_WIDTH, budget)) def get_metrics_table( @@ -48,7 +59,13 @@ def get_metrics_table( ) -> RenderableType: labels = job_labels(jobs) label_width = max((len(label) for label in labels), default=0) - width = _spark_width(console_width or console.width, label_width) + console_width = console_width or console.width + # Older runners and non-NVIDIA GPUs report no power. The column is also left out if three + # charts of the minimum width don't fit, as Rich would cut the labels to make room. + show_power = any(_reports_power(m) for m in metrics) and ( + _spark_budget(console_width, label_width, show_power=True) >= MIN_SPARK_WIDTH + ) + width = _spark_width(console_width, label_width, show_power) window = _shared_window(metrics) table = Table(box=None) @@ -57,18 +74,21 @@ def get_metrics_table( table.add_column("", style="secondary", no_wrap=True) table.add_column("UTILIZATION", no_wrap=True) table.add_column("MEMORY", no_wrap=True) + if show_power: + table.add_column("POWER", no_wrap=True) + empty_row = [""] * len(table.columns) for index, (job, job_metrics) in enumerate(zip(jobs, metrics)): if index: - table.add_row("", "", "", "") - _add_job(table, job, job_metrics, width, labels[index], window) + table.add_row(*empty_row) + _add_job(table, job, job_metrics, width, labels[index], window, show_power) if window is not None: - table.add_row("", "", "", "") + table.add_row(*empty_row) # the axis spans the widest chart drawn: a job with fewer samples than cells draws # one cell per sample and cannot fill its share axis = _time_axis(max(_cells_drawn(m, window, width) for m in metrics), *window) - table.add_row("", "", axis, axis) + table.add_row("", "", *[axis] * (len(table.columns) - 2)) return table @@ -100,23 +120,39 @@ def _add_job( width: int, label: str, window: Optional[tuple[datetime, datetime]], + show_power: bool, ) -> None: resources = _get_resources(job) blanks = _blank_cells(metrics, window, width) cells = width - blanks - table.add_row( + host_row: List[RenderableType] = [ label, "cpu", _pad(_cpu_cell(metrics, resources, cells), blanks), _pad(_memory_cell(metrics, resources, cells), blanks), - ) + ] + if show_power: + host_row.append("") + table.add_row(*host_row) for index in range(_gpus_num(metrics, resources)): - table.add_row( + gpu_row: List[RenderableType] = [ "", f"gpu={index}", _pad(_gpu_util_cell(metrics, index, cells), blanks), _pad(_gpu_memory_cell(metrics, resources, index, cells), blanks), - ) + ] + if show_power: + power_blanks = max(blanks, _blank_cells(_power_series(metrics, index), window, width)) + gpu_row.append( + _pad(_gpu_power_cell(metrics, index, width - power_blanks), power_blanks) + ) + table.add_row(*gpu_row) + + +def _power_series(job_metrics: JobMetrics, index: int) -> JobMetrics: + """Power draw of GPU `index` only. It may start later than the other series.""" + name = f"gpu_power_draw_watts_gpu{index}" + return JobMetrics(metrics=[m for m in job_metrics.metrics if m.name == name]) def _shared_window(metrics: Sequence[JobMetrics]) -> Optional[tuple[datetime, datetime]]: @@ -197,6 +233,20 @@ def _gpu_util_cell(job_metrics: JobMetrics, index: int, width: int) -> Text: return Text.assemble(sparkline(values, width, GPU_RAMP), " ", f"{values[-1]:.0f}%") +def _gpu_power_cell(job_metrics: JobMetrics, index: int, width: int) -> Text: + values = _metric_values(job_metrics, f"gpu_power_draw_watts_gpu{index}") + if not values: + return no_data() + limit = _latest_value(job_metrics, f"gpu_power_limit_watts_gpu{index}") + if not limit: + # without the limit there is no fixed scale to chart against + return Text(f"{values[-1]:.0f}W") + percents = [v / limit * 100 for v in values] + return Text.assemble( + sparkline(percents, width, GPU_RAMP), " ", f"{values[-1]:.0f}W/{limit:.0f}W" + ) + + def _capacity_cell(values: List[float], total: Optional[float], width: int, ramp: Ramp) -> Text: """A memory chart drawn against capacity, labelled `used/total`.""" percents = [v / total * 100 for v in values] if total else values @@ -252,6 +302,10 @@ def _latest_value(job_metrics: JobMetrics, name: str) -> Optional[Any]: return values[-1] if values else None +def _reports_power(job_metrics: JobMetrics) -> bool: + return any(m.name.startswith("gpu_power_draw_watts_gpu") for m in job_metrics.metrics) + + def _gpus_num(job_metrics: JobMetrics, resources: Optional[Resources]) -> int: if resources is not None and resources.gpus: return len(resources.gpus) diff --git a/src/dstack/_internal/server/background/scheduled_tasks/metrics.py b/src/dstack/_internal/server/background/scheduled_tasks/metrics.py index 103e7795e8..a4e7e9b119 100644 --- a/src/dstack/_internal/server/background/scheduled_tasks/metrics.py +++ b/src/dstack/_internal/server/background/scheduled_tasks/metrics.py @@ -152,6 +152,8 @@ async def _collect_job_metrics(job_model: JobModel) -> Optional[JobMetricsPoint] gpus_memory_usage_bytes = [g.gpu_memory_usage_bytes for g in res.gpus] gpus_util_percent = [g.gpu_util_percent for g in res.gpus] + gpus_power_draw_watts = [g.gpu_power_draw_watts for g in res.gpus] + gpus_power_limit_watts = [g.gpu_power_limit_watts for g in res.gpus] return JobMetricsPoint( job_id=job_model.id, @@ -161,9 +163,18 @@ async def _collect_job_metrics(job_model: JobModel) -> Optional[JobMetricsPoint] memory_working_set_bytes=res.memory_working_set_bytes, gpus_memory_usage_bytes=json.dumps(gpus_memory_usage_bytes), gpus_util_percent=json.dumps(gpus_util_percent), + gpus_power_draw_watts=_dump_gpus_power(gpus_power_draw_watts), + gpus_power_limit_watts=_dump_gpus_power(gpus_power_limit_watts), ) +def _dump_gpus_power(values: List[Optional[float]]) -> Optional[str]: + # NULL if no GPU has a reading, as with older runners and non-NVIDIA GPUs + if all(v is None for v in values): + return None + return json.dumps(values) + + @runner_ssh_tunnel def _pull_runner_metrics( addresses: Mapping[int, client.LocalAddress], diff --git a/src/dstack/_internal/server/migrations/versions/2026/11_20_0019_eb9ea2fdd147_add_jobmetricspoint_gpu_power_columns.py b/src/dstack/_internal/server/migrations/versions/2026/11_20_0019_eb9ea2fdd147_add_jobmetricspoint_gpu_power_columns.py new file mode 100644 index 0000000000..45092e2fdf --- /dev/null +++ b/src/dstack/_internal/server/migrations/versions/2026/11_20_0019_eb9ea2fdd147_add_jobmetricspoint_gpu_power_columns.py @@ -0,0 +1,34 @@ +"""Add JobMetricsPoint GPU power columns + +Revision ID: eb9ea2fdd147 +Revises: 2d2692d5b3c2 +Create Date: 2026-11-20 00:19:27.385130+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "eb9ea2fdd147" +down_revision = "2d2692d5b3c2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("job_metrics_points", schema=None) as batch_op: + batch_op.add_column(sa.Column("gpus_power_draw_watts", sa.Text(), nullable=True)) + batch_op.add_column(sa.Column("gpus_power_limit_watts", sa.Text(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("job_metrics_points", schema=None) as batch_op: + batch_op.drop_column("gpus_power_limit_watts") + batch_op.drop_column("gpus_power_draw_watts") + + # ### end Alembic commands ### diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 27c5311ea1..85a1918a9b 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -1240,6 +1240,13 @@ class JobMetricsPoint(BaseModel): """`gpus_memory_usage_bytes` stores a JSON-encoded list of metric values with length `len(gpus)`.""" gpus_util_percent: Mapped[str] = mapped_column(Text) """`gpus_util_percent` stores a JSON-encoded list of metric values with length `len(gpus)`.""" + gpus_power_draw_watts: Mapped[Optional[str]] = mapped_column(Text) + """`gpus_power_draw_watts` stores a JSON-encoded list of metric values with length `len(gpus)`, + with `null` for GPUs without a reading. `None` if no GPU reported power, or if the point was + written by a server without this column. + """ + gpus_power_limit_watts: Mapped[Optional[str]] = mapped_column(Text) + """`gpus_power_limit_watts` has the same format as `gpus_power_draw_watts`.""" class JobPrometheusMetrics(BaseModel): diff --git a/src/dstack/_internal/server/routers/metrics.py b/src/dstack/_internal/server/routers/metrics.py index b44d4a86a3..feb2010c6b 100644 --- a/src/dstack/_internal/server/routers/metrics.py +++ b/src/dstack/_internal/server/routers/metrics.py @@ -58,6 +58,8 @@ async def get_job_metrics( * `gpu_memory_total_bytes` * `gpu_memory_usage_bytes_gpu{i}` * `gpu_util_percent_gpu{i}` + * `gpu_power_draw_watts_gpu{i}` (NVIDIA only; may have fewer samples than other metrics) + * `gpu_power_limit_watts_gpu{i}` (NVIDIA only; may have fewer samples than other metrics) """ _, project = user_project diff --git a/src/dstack/_internal/server/schemas/runner.py b/src/dstack/_internal/server/schemas/runner.py index 3a3aac4a0a..aa4ff51e05 100644 --- a/src/dstack/_internal/server/schemas/runner.py +++ b/src/dstack/_internal/server/schemas/runner.py @@ -182,6 +182,10 @@ class ComponentInstallRequest(CoreModel): class GPUMetrics(CoreModel): gpu_memory_usage_bytes: int gpu_util_percent: int + gpu_power_draw_watts: Optional[float] = None + """`gpu_power_draw_watts` is not set by old runners, non-NVIDIA GPUs, and GPUs without power readings.""" + gpu_power_limit_watts: Optional[float] = None + """`gpu_power_limit_watts` is the enforced limit. Set under the same conditions as `gpu_power_draw_watts`.""" class MetricsResponse(CoreModel): diff --git a/src/dstack/_internal/server/services/metrics.py b/src/dstack/_internal/server/services/metrics.py index d8f3c96830..7dba4a1d99 100644 --- a/src/dstack/_internal/server/services/metrics.py +++ b/src/dstack/_internal/server/services/metrics.py @@ -53,6 +53,13 @@ async def get_job_metrics( return _calculate_job_metrics(job_model, points) +def load_gpus_power(value: Optional[str], gpus_num: int) -> list[Optional[float]]: + """Decodes `JobMetricsPoint.gpus_power_draw_watts` or `gpus_power_limit_watts`.""" + if value is None: + return [None] * gpus_num + return json.loads(value) + + def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint]) -> JobMetrics: timestamps: list[datetime] = [] cpu_usage_points: list[int] = [] @@ -60,6 +67,8 @@ def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint memory_working_set_points: list[int] = [] gpus_memory_usage_points: defaultdict[int, list[int]] = defaultdict(list) gpus_util_points: defaultdict[int, list[int]] = defaultdict(list) + gpus_power_draw_points: defaultdict[int, list[tuple[datetime, float]]] = defaultdict(list) + gpus_power_limit_points: defaultdict[int, list[tuple[datetime, float]]] = defaultdict(list) cpus_detected_num: Optional[int] = None memory_total: Optional[int] = None @@ -81,7 +90,8 @@ def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint gpus_detected_num: Optional[int] = None gpus_detected_num_mismatch: bool = False for point, prev_point in zip(points, points[1:]): - timestamps.append(_unix_time_micro_to_datetime(point.timestamp_micro)) + timestamp = _unix_time_micro_to_datetime(point.timestamp_micro) + timestamps.append(timestamp) cpu_usage_points.append(_get_cpu_usage(point, prev_point)) memory_usage_points.append(point.memory_usage_bytes) memory_working_set_points.append(point.memory_working_set_bytes) @@ -92,9 +102,16 @@ def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint if len(gpus_memory_usage) != gpus_detected_num or len(gpus_util) != gpus_detected_num: gpus_detected_num_mismatch = True if not gpus_detected_num_mismatch: + gpus_power_draw = load_gpus_power(point.gpus_power_draw_watts, gpus_detected_num) + gpus_power_limit = load_gpus_power(point.gpus_power_limit_watts, gpus_detected_num) for i in range(gpus_detected_num): gpus_memory_usage_points[i].append(gpus_memory_usage[i]) gpus_util_points[i].append(gpus_util[i]) + power_draw, power_limit = gpus_power_draw[i], gpus_power_limit[i] + if power_draw is not None: + gpus_power_draw_points[i].append((timestamp, power_draw)) + if power_limit is not None: + gpus_power_limit_points[i].append((timestamp, power_limit)) metrics: list[Metric] = [ Metric( @@ -146,6 +163,14 @@ def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint values=gpu_util_points, ) ) + for index, gpu_power_draw_points in gpus_power_draw_points.items(): + metrics.append( + _make_sparse_metric(f"gpu_power_draw_watts_gpu{index}", gpu_power_draw_points) + ) + for index, gpu_power_limit_points in gpus_power_limit_points.items(): + metrics.append( + _make_sparse_metric(f"gpu_power_limit_watts_gpu{index}", gpu_power_limit_points) + ) return JobMetrics(metrics=metrics) @@ -157,6 +182,15 @@ def _make_constant_metric(name: str, timestamps: list[datetime], value: float) - ) +def _make_sparse_metric(name: str, points: list[tuple[datetime, float]]) -> Metric: + """A metric with its own timestamps, as it skips points that have no value.""" + return Metric( + name=name, + timestamps=[timestamp for timestamp, _ in points], + values=[value for _, value in points], + ) + + def _get_cpu_usage(last_point: JobMetricsPoint, prev_point: JobMetricsPoint) -> int: window = last_point.timestamp_micro - prev_point.timestamp_micro if window == 0: diff --git a/src/dstack/_internal/server/services/prometheus/custom_metrics.py b/src/dstack/_internal/server/services/prometheus/custom_metrics.py index 1f0015c299..ba7435ad83 100644 --- a/src/dstack/_internal/server/services/prometheus/custom_metrics.py +++ b/src/dstack/_internal/server/services/prometheus/custom_metrics.py @@ -26,6 +26,7 @@ ) from dstack._internal.server.services.instances import get_instance_offer from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data +from dstack._internal.server.services.metrics import load_gpus_power from dstack._internal.server.services.runs import get_run_spec from dstack._internal.utils.common import get_current_datetime @@ -188,10 +189,13 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]: metrics.add_sample(_JOB_MEMORY_WORKING_SET, labels, jmp.memory_working_set_bytes) if gpus: gpu_memory_total = gpus[0].memory_mib * 1024 * 1024 - for gpu_num, (gpu_util, gpu_memory_usage) in enumerate( + gpus_util = json.loads(jmp.gpus_util_percent) + gpus_power_draw = load_gpus_power(jmp.gpus_power_draw_watts, len(gpus_util)) + for gpu_num, (gpu_util, gpu_memory_usage, gpu_power_draw) in enumerate( zip( - json.loads(jmp.gpus_util_percent), + gpus_util, json.loads(jmp.gpus_memory_usage_bytes), + gpus_power_draw, ) ): gpu_labels = labels.copy() @@ -199,6 +203,8 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]: metrics.add_sample(_JOB_GPU_USAGE_RATIO, gpu_labels, gpu_util / 100) metrics.add_sample(_JOB_GPU_MEMORY_TOTAL, gpu_labels, gpu_memory_total) metrics.add_sample(_JOB_GPU_MEMORY_USAGE, gpu_labels, gpu_memory_usage) + if gpu_power_draw is not None: + metrics.add_sample(_JOB_GPU_POWER_DRAW, gpu_labels, gpu_power_draw) jpm = job_prometheus_metrics.get(job.id) if jpm is not None: for metric in text_string_to_metric_families(jpm.text): @@ -227,6 +233,7 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]: _JOB_GPU_USAGE_RATIO = "dstack_job_gpu_usage_ratio" _JOB_GPU_MEMORY_TOTAL = "dstack_job_gpu_memory_total_bytes" _JOB_GPU_MEMORY_USAGE = "dstack_job_gpu_memory_usage_bytes" +_JOB_GPU_POWER_DRAW = "dstack_job_gpu_power_draw_watts" class _Metrics(dict[str, Metric]): @@ -287,6 +294,7 @@ class _JobMetrics(_Metrics): (_JOB_GPU_USAGE_RATIO, _GAUGE, "Job GPU usage, percent (as 0.0-1.0)"), (_JOB_GPU_MEMORY_TOTAL, _GAUGE, "Total GPU memory allocated for the job, bytes"), (_JOB_GPU_MEMORY_USAGE, _GAUGE, "GPU memory used by the job, bytes"), + (_JOB_GPU_POWER_DRAW, _GAUGE, "Power draw of the job's GPU, watts (NVIDIA only)"), ] diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index 73b27b70a1..202904ffd6 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -1317,7 +1317,10 @@ async def create_job_metrics_point( memory_working_set_bytes: int = 1024, gpus_memory_usage_bytes: Optional[List[int]] = None, gpus_util_percent: Optional[List[int]] = None, + gpus_power_draw_watts: Optional[List[Optional[float]]] = None, + gpus_power_limit_watts: Optional[List[Optional[float]]] = None, ) -> JobMetricsPoint: + """Unset power lists are stored as NULL, as for a point without power readings.""" timestamp_micro = int(timestamp.timestamp() * 1_000_000) if gpus_memory_usage_bytes is None: gpus_memory_usage_bytes = [] @@ -1331,6 +1334,12 @@ async def create_job_metrics_point( memory_working_set_bytes=memory_working_set_bytes, gpus_memory_usage_bytes=json.dumps(gpus_memory_usage_bytes), gpus_util_percent=json.dumps(gpus_util_percent), + gpus_power_draw_watts=( + json.dumps(gpus_power_draw_watts) if gpus_power_draw_watts is not None else None + ), + gpus_power_limit_watts=( + json.dumps(gpus_power_limit_watts) if gpus_power_limit_watts is not None else None + ), ) session.add(jmp) await session.commit() diff --git a/src/tests/_internal/cli/utils/test_metrics.py b/src/tests/_internal/cli/utils/test_metrics.py index 099b86238f..ee643b84c8 100644 --- a/src/tests/_internal/cli/utils/test_metrics.py +++ b/src/tests/_internal/cli/utils/test_metrics.py @@ -1,6 +1,6 @@ import re from datetime import datetime, timedelta, timezone -from typing import List, Tuple +from typing import List, Optional, Tuple from unittest.mock import MagicMock import pytest @@ -41,6 +41,7 @@ def make_run( replica: int = 0, job_num: int = 0, group: str = "default", + memory_gb: int = 32, ) -> Tuple[MagicMock, JobMetrics]: """A job and its metrics. `state` decides whether the newest sample reads as `now`.""" newest = datetime.now(timezone.utc) @@ -62,7 +63,7 @@ def series(fn) -> List[float]: Metric( name="memory_working_set_bytes", timestamps=timestamps, - values=series(lambda t: memory(t) * 32 * GIB), + values=series(lambda t: memory(t) * memory_gb * GIB), ), ] for index in range(gpus): @@ -82,7 +83,7 @@ def series(fn) -> List[float]: job.job_spec.replica_group = group submission = MagicMock() resources = MagicMock() - resources.cpus, resources.memory_mib = cpus, 32 * 1024 + resources.cpus, resources.memory_mib = cpus, memory_gb * 1024 resources.gpus = [MagicMock(memory_mib=CAPACITY_GB * 1024) for _ in range(gpus)] submission.job_runtime_data.offer.instance.resources = resources job.job_submissions = [submission] @@ -113,9 +114,44 @@ def row(output: str, label: str) -> str: return next(line for line in lines(output) if line.strip().startswith(label)) -def bars(line: str) -> List[int]: - """Glyph heights of the first sparkline in `line`, left to right.""" - return [SPARKS.index(glyph) for glyph in re.findall(rf"[{SPARKS}]+", line)[0]] +def bars(line: str, chart: int = 0) -> List[int]: + """Glyph heights of a sparkline in `line`, the first by default, left to right.""" + return [SPARKS.index(glyph) for glyph in re.findall(rf"[{SPARKS}]+", line)[chart]] + + +def chart_start(line: str, chart: int = 0) -> int: + """The column where a sparkline in `line` begins.""" + return list(re.finditer(rf"[{SPARKS}]+", line))[chart].start() + + +def fits(output: str, width: int) -> bool: + """Nothing wider than the terminal and no label cut short.""" + return "…" not in output and max(len(line) for line in output.splitlines()) <= width + + +def with_power( + metrics: JobMetrics, draw: float, limit: Optional[float] = 700.0, gpus: int = 1 +) -> JobMetrics: + """`metrics` plus a steady power draw on each GPU, as an NVIDIA runner reports it.""" + timestamps = metrics.metrics[0].timestamps + power = [] + for index in range(gpus): + power.append( + Metric( + name=f"gpu_power_draw_watts_gpu{index}", + timestamps=timestamps, + values=[draw] * len(timestamps), + ) + ) + if limit is not None: + power.append( + Metric( + name=f"gpu_power_limit_watts_gpu{index}", + timestamps=timestamps, + values=[limit] * len(timestamps), + ) + ) + return JobMetrics(metrics=metrics.metrics + power) def colours(output: str, label: str) -> set: @@ -168,6 +204,16 @@ def test_no_data_is_not_zero(self): assert sum(1 for line in lines(missing) if "gpu=" in line) == 2 assert "1%" in render(*make_run("idle", gpus=2)) + def test_labels_fit_on_a_large_host(self): + """`100%` next to `1024GB/2048GB` is as wide as the numbers get on a host under 10 TB. + Below 57 columns, two charts of the minimum width don't fit anyway.""" + job, metrics = make_run("ramp", gpus=2, memory_gb=2048) + host = row(render(job, metrics), "job=0") + assert " 100% " in host and host.endswith(" 1024GB/2048GB") + # a step of 5 hits every remainder of the width split between charts + for width in range(57, 241, 5): + assert fits(render(job, metrics, width=width), width), width + class TestWindow: @pytest.mark.parametrize("state", ["running", "terminated"]) @@ -245,6 +291,63 @@ def test_a_late_job_starts_where_it_started(self): assert first["replica=1"] > first["replica=0"] # pushed right by the blank +class TestPower: + def test_no_column_without_power(self): + """Older runners and non-NVIDIA GPUs report no power: the table keeps its two columns + rather than gaining one that reads `no data` on every row.""" + assert "POWER" not in render(*make_run()) + + def test_height_is_a_fraction_of_the_limit(self): + """As memory against capacity: 350W on a 700W card is a half-height bar.""" + job, metrics = make_run() + gpu = row(render(job, with_power(metrics, 350.0)), "gpu=0") + assert gpu.endswith("350W/700W") + assert set(bars(gpu, chart=2)) == {(len(SPARKS) - 1) // 2} + + def test_without_the_limit_only_the_number_is_shown(self): + """With nothing to scale against, a chart would have to rescale to its own peak.""" + job, metrics = make_run() + gpu = row(render(job, with_power(metrics, 350.0, limit=None)), "gpu=0") + assert gpu.endswith("350W") + assert len(re.findall(rf"[{SPARKS}]+", gpu)) == 2 + + def test_a_job_without_power_reads_no_data(self): + nvidia, other = make_run(job_num=0), make_run(job_num=1) + output = render([nvidia[0], other[0]], [with_power(nvidia[1], 350.0), other[1]]) + gpus = [line for line in lines(output) if "gpu=0" in line] + assert gpus[0].endswith("350W/700W") + assert gpus[1].endswith("no data") + + def test_late_power_starts_where_it_started(self): + """Points collected before the server stored power have none, so the power chart + starts halfway through the window when only the newest half of the points has it.""" + job, metrics = make_run(samples=360) + metrics = with_power(metrics, 350.0) + late_power = JobMetrics( + metrics=[ + Metric(name=m.name, timestamps=m.timestamps[:180], values=m.values[:180]) + if m.name.startswith("gpu_power") + else m + for m in metrics.metrics + ] + ) + output = render(job, late_power) + header, gpu = lines(output)[0], row(output, "gpu=0") + width = len(bars(gpu, chart=0)) + assert chart_start(gpu, chart=0) == header.index("UTILIZATION") + assert abs(chart_start(gpu, chart=2) - header.index("POWER") - width / 2) <= 1 + + def test_the_column_fits(self): + """`1000W/1000W` on a 2 TB host. Where a third chart doesn't fit, the column is left out + so that Rich doesn't cut the labels.""" + job, metrics = make_run("ramp", gpus=2, memory_gb=2048) + metrics = with_power(metrics, 1000.0, limit=1000.0, gpus=2) + for width in range(57, 241, 5): + assert fits(render(job, metrics, width=width), width), width + assert row(render(job, metrics, width=120), "gpu=1").endswith("1000W/1000W") + assert "POWER" not in render(job, metrics, width=70) + + @pytest.mark.parametrize( "bytes_value,decimal_places,expected", [ diff --git a/src/tests/_internal/server/background/scheduled_tasks/test_metrics.py b/src/tests/_internal/server/background/scheduled_tasks/test_metrics.py index 59ac8a9c8c..65ecc7392e 100644 --- a/src/tests/_internal/server/background/scheduled_tasks/test_metrics.py +++ b/src/tests/_internal/server/background/scheduled_tasks/test_metrics.py @@ -1,4 +1,5 @@ from datetime import datetime, timezone +from typing import Optional from unittest.mock import patch import pytest @@ -35,7 +36,40 @@ class TestCollectMetrics: @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - async def test_collects_metrics(self, test_db, session: AsyncSession): + @pytest.mark.parametrize( + ["gpus", "power_draw", "power_limit"], + [ + pytest.param( + [ + GPUMetrics( + gpu_memory_usage_bytes=0, + gpu_util_percent=0, + gpu_power_draw_watts=72.5, + gpu_power_limit_watts=300.0, + ), + # a GPU that prints [N/A] for power + GPUMetrics(gpu_memory_usage_bytes=0, gpu_util_percent=0), + ], + "[72.5, null]", + "[300.0, null]", + id="nvidia", + ), + pytest.param( + [GPUMetrics(gpu_memory_usage_bytes=0, gpu_util_percent=0)], + None, + None, + id="no-power-readings", + ), + ], + ) + async def test_collects_metrics( + self, + test_db, + session: AsyncSession, + gpus: list[GPUMetrics], + power_draw: Optional[str], + power_limit: Optional[str], + ): user = await create_user(session=session, global_role=GlobalRole.USER) project = await create_project(session=session, owner=user) await add_project_member( @@ -76,12 +110,7 @@ async def test_collects_metrics(self, test_db, session: AsyncSession): cpu_usage_micro=2, memory_usage_bytes=3, memory_working_set_bytes=4, - gpus=[ - GPUMetrics( - gpu_memory_usage_bytes=0, - gpu_util_percent=0, - ) - ], + gpus=gpus, ) await collect_metrics() SSHTunnelMock.assert_called_once() @@ -89,6 +118,8 @@ async def test_collects_metrics(self, test_db, session: AsyncSession): res = await session.execute(select(JobMetricsPoint)) metrics_point = res.scalar_one() assert metrics_point.job_id == job.id + assert metrics_point.gpus_power_draw_watts == power_draw + assert metrics_point.gpus_power_limit_watts == power_limit @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) diff --git a/src/tests/_internal/server/routers/test_prometheus.py b/src/tests/_internal/server/routers/test_prometheus.py index 26da89a90e..9a1131c6b2 100644 --- a/src/tests/_internal/server/routers/test_prometheus.py +++ b/src/tests/_internal/server/routers/test_prometheus.py @@ -152,6 +152,7 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient) memory_usage_bytes=4_294_967_296, gpus_util_percent=[80, 90], gpus_memory_usage_bytes=[1_073_741_824, 2_147_483_648], + gpus_power_draw_watts=[52.5, None], ) # Older, ignored await create_job_metrics_point( @@ -304,6 +305,9 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient) # TYPE dstack_job_gpu_memory_usage_bytes gauge dstack_job_gpu_memory_usage_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4",dstack_gpu_num="0"}} 1073741824.0 dstack_job_gpu_memory_usage_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4",dstack_gpu_num="1"}} 2147483648.0 + # HELP dstack_job_gpu_power_draw_watts Power draw of the job's GPU, watts (NVIDIA only) + # TYPE dstack_job_gpu_power_draw_watts gauge + dstack_job_gpu_power_draw_watts{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4",dstack_gpu_num="0"}} 52.5 # HELP FIELD_1 Test field 1 # TYPE FIELD_1 gauge FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 350.0 @@ -323,6 +327,33 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient) ) assert response.text.strip() == expected + async def test_skips_gpu_power_if_point_has_none( + self, session: AsyncSession, client: AsyncClient + ): + """Points written before the power columns were added, or by an older server replica, + have no power, and the other GPU metrics are still exported.""" + user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER) + project = await _create_project(session, "project-1", user) + job = await _create_job( + session=session, + run_name="run-1", + project=project, + user=user, + status=JobStatus.RUNNING, + job_provisioning_data=get_job_provisioning_data(gpu_count=2), + ) + await create_job_metrics_point( + session=session, + job_model=job, + timestamp=FAKE_NOW - timedelta(seconds=30), + gpus_util_percent=[80, 90], + gpus_memory_usage_bytes=[1024, 2048], + ) + response = await client.get("/metrics") + assert response.status_code == 200 + assert response.text.count("dstack_job_gpu_usage_ratio{") == 2 + assert "dstack_job_gpu_power_draw_watts{" not in response.text + @patch("prometheus_client.generate_latest", lambda: BASE_CLIENT_METRICS) async def test_returns_empty_response_if_no_runs(self, client: AsyncClient): response = await client.get("/metrics") diff --git a/src/tests/_internal/server/services/test_metrics.py b/src/tests/_internal/server/services/test_metrics.py index 083855652a..a20fc6ad30 100644 --- a/src/tests/_internal/server/services/test_metrics.py +++ b/src/tests/_internal/server/services/test_metrics.py @@ -165,3 +165,47 @@ async def test_get_metrics( Metric(name="gpu_util_percent_gpu0", timestamps=ts, values=gpu0_util), Metric(name="gpu_util_percent_gpu1", timestamps=ts, values=gpu1_util), ] + + async def test_get_metrics_gpu_power(self, session: AsyncSession): + user = await create_user(session=session) + project = await create_project(session=session, owner=user) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run(session=session, project=project, repo=repo, user=user) + jpd = get_job_provisioning_data(gpu_count=2) + job = await create_job(session=session, run=run, job_provisioning_data=jpd) + # gpu1 never reports power, gpu0 misses one reading, and the two oldest points have no + # power columns (the oldest one is only used for the CPU usage of the next one) + power = [ + ([300.5, None], [400.0, None]), + ([None, None], [None, None]), + ([250.0, None], [400.0, None]), + (None, None), + (None, None), + ] + for dt, (draw, limit) in zip(self.ts, power): + await create_job_metrics_point( + session=session, + job_model=job, + timestamp=dt, + gpus_memory_usage_bytes=[1024, 1024], + gpus_util_percent=[50, 0], + gpus_power_draw_watts=draw, + gpus_power_limit_watts=limit, + ) + + metrics = await get_job_metrics(session, job) + + assert [m for m in metrics.metrics if m.name.startswith("gpu_power")] == [ + Metric( + name="gpu_power_draw_watts_gpu0", + timestamps=[self.ts[0], self.ts[2]], + values=[300.5, 250.0], + ), + Metric( + name="gpu_power_limit_watts_gpu0", + timestamps=[self.ts[0], self.ts[2]], + values=[400.0, 400.0], + ), + ] + util = next(m for m in metrics.metrics if m.name == "gpu_util_percent_gpu0") + assert len(util.timestamps) == 4