Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions mkdocs/docs/concepts/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ This tab displays key CPU, memory, and GPU metrics collected during the last hou

As an alternative to the UI, you can track essential metrics via the CLI.
The `dstack metrics` command charts CPU, memory, and GPU utilization over the last hour.
For jobs on NVIDIA GPUs, it adds a `POWER` column that charts power draw against each GPU's enforced power limit.

<div class="termy">

Expand Down Expand Up @@ -126,6 +127,7 @@ telemetry, and more.
| `dstack_job_gpu_usage_ratio` | *gauge* | Job GPU usage, percent (as 0.0-1.0) | `0.93` |
| `dstack_job_gpu_memory_total_bytes` | *gauge* | Total GPU memory allocated for the job, bytes | `8589934592.0` |
| `dstack_job_gpu_memory_usage_bytes` | *gauge* | GPU memory used by the job, bytes | `1048576.0` |
| `dstack_job_gpu_power_draw_watts` | *gauge* | Power draw of the job's GPU, watts (NVIDIA only) | `285.43` |
| `DCGM_FI_DEV_GPU_UTIL` | *gauge* | GPU utilization (in %) | |
| `DCGM_FI_DEV_MEM_COPY_UTIL` | *gauge* | Memory utilization (in %) | |
| `DCGM_FI_DEV_ENC_UTIL` | *gauge* | Encoder utilization (in %) | |
Expand Down
37 changes: 30 additions & 7 deletions runner/internal/runner/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
"math"
"os"
"os/exec"
"path"
Expand Down Expand Up @@ -159,13 +160,13 @@ func (s *MetricsCollector) GetGPUMetrics(ctx context.Context) ([]schemas.GPUMetr
}

func (s *MetricsCollector) GetNVIDIAGPUMetrics(ctx context.Context) ([]schemas.GPUMetrics, error) {
cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=memory.used,utilization.gpu", "--format=csv,noheader,nounits")
cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=memory.used,utilization.gpu,power.draw,enforced.power.limit", "--format=csv,noheader,nounits")
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
return []schemas.GPUMetrics{}, fmt.Errorf("failed to execute nvidia-smi: %w", err)
}
return parseNVIDIASMILikeMetrics(out.String())
return parseNVIDIASMILikeMetrics(out.String(), true)
}

func (s *MetricsCollector) GetAMDGPUMetrics(ctx context.Context) ([]schemas.GPUMetrics, error) {
Expand Down Expand Up @@ -241,16 +242,22 @@ func (s *MetricsCollector) GetIntelAcceleratorMetrics(ctx context.Context) ([]sc
if err := cmd.Run(); err != nil {
return []schemas.GPUMetrics{}, fmt.Errorf("failed to execute hl-smi: %w", err)
}
return parseNVIDIASMILikeMetrics(out.String())
return parseNVIDIASMILikeMetrics(out.String(), false)
}

func parseNVIDIASMILikeMetrics(output string) ([]schemas.GPUMetrics, error) {
// parseNVIDIASMILikeMetrics parses memory used and utilization, followed by power draw and
// limit if withPower is set (nvidia-smi is queried for them, hl-smi is not).
func parseNVIDIASMILikeMetrics(output string, withPower bool) ([]schemas.GPUMetrics, error) {
metrics := []schemas.GPUMetrics{}
columns := 2
if withPower {
columns = 4
}

lines := strings.Split(strings.TrimSpace(output), "\n")
for _, line := range lines {
parts := strings.Split(line, ", ")
if len(parts) != 2 {
if len(parts) != columns {
continue
}
memUsed, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
Expand All @@ -261,11 +268,27 @@ func parseNVIDIASMILikeMetrics(output string) ([]schemas.GPUMetrics, error) {
if err != nil {
return metrics, fmt.Errorf("failed to parse accelerator utilization: %w", err)
}
metrics = append(metrics, schemas.GPUMetrics{
gpuMetrics := schemas.GPUMetrics{
GPUMemoryUsage: memUsed * 1024 * 1024, // Convert MiB to bytes
GPUUtil: utilization,
})
}
if withPower {
gpuMetrics.GPUPowerDraw = parseWatts(parts[2])
gpuMetrics.GPUPowerLimit = parseWatts(parts[3])
}
metrics = append(metrics, gpuMetrics)
}

return metrics, nil
}

// parseWatts returns nil for readings the GPU does not provide, which nvidia-smi prints as
// [N/A], [Not Supported] or [Unknown Error], so that they do not drop the other metrics.
// NaN and infinity are dropped too, as they cannot be encoded in the JSON response.
func parseWatts(value string) *float64 {
watts, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
if err != nil || math.IsNaN(watts) || math.IsInf(watts, 0) {
return nil
}
return &watts
}
54 changes: 54 additions & 0 deletions runner/internal/runner/metrics/metrics_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package metrics

import (
"encoding/json"
"testing"

"github.com/dstackai/dstack/runner/internal/runner/schemas"
Expand Down Expand Up @@ -45,3 +46,56 @@ func TestGetAMDGPUMetrics_ErrorGPUUtilNA(t *testing.T) {
assert.ErrorContains(t, err, "GPU utilization is N/A")
assert.Nil(t, metrics)
}

func TestParseNVIDIASMILikeMetrics(t *testing.T) {
watts := func(w float64) *float64 { return &w }

cases := []struct {
name string
output string
withPower bool
expected []schemas.GPUMetrics
}{
{
// NVIDIA driver 620.70.06, Tesla T4 under load
name: "nvidia-smi",
output: "525, 100, 68.23, 70.00\n",
withPower: true,
expected: []schemas.GPUMetrics{
{GPUMemoryUsage: 550502400, GPUUtil: 100, GPUPowerDraw: watts(68.23), GPUPowerLimit: watts(70)},
},
},
{
// the T4 at idle, then made-up lines for GPUs without power readings
name: "nvidia-smi, GPUs without power readings",
output: "525, 0, 26.88, 70.00\n1024, 5, [N/A], [N/A]\n2048, 7, [Not Supported], [Not Supported]\n" +
"512, 9, [Unknown Error], 70.00\n256, 1, NaN, Inf\n",
withPower: true,
expected: []schemas.GPUMetrics{
{GPUMemoryUsage: 550502400, GPUUtil: 0, GPUPowerDraw: watts(26.88), GPUPowerLimit: watts(70)},
{GPUMemoryUsage: 1073741824, GPUUtil: 5},
{GPUMemoryUsage: 2147483648, GPUUtil: 7},
{GPUMemoryUsage: 536870912, GPUUtil: 9, GPUPowerLimit: watts(70)},
{GPUMemoryUsage: 268435456, GPUUtil: 1},
},
},
{
name: "hl-smi",
output: "512, 3\n",
expected: []schemas.GPUMetrics{
{GPUMemoryUsage: 536870912, GPUUtil: 3},
},
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
metrics, err := parseNVIDIASMILikeMetrics(tc.output, tc.withPower)
assert.NoError(t, err)
assert.Equal(t, tc.expected, metrics)
// a value json.Marshal rejects would fail the whole metrics response
_, err = json.Marshal(metrics)
assert.NoError(t, err)
})
}
}
6 changes: 4 additions & 2 deletions runner/internal/runner/schemas/schemas.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,10 @@ type HealthcheckResponse struct {
}

type GPUMetrics struct {
GPUMemoryUsage uint64 `json:"gpu_memory_usage_bytes"`
GPUUtil uint64 `json:"gpu_util_percent"`
GPUMemoryUsage uint64 `json:"gpu_memory_usage_bytes"`
GPUUtil uint64 `json:"gpu_util_percent"`
GPUPowerDraw *float64 `json:"gpu_power_draw_watts,omitempty"`
GPUPowerLimit *float64 `json:"gpu_power_limit_watts,omitempty"`
}

type SystemMetrics struct {
Expand Down
84 changes: 69 additions & 15 deletions src/dstack/_internal/cli/utils/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,28 @@
NOW = "now"

LIVE_THRESHOLD = timedelta(seconds=3 * WATCH_INTERVAL_SECONDS)
_FIXED_COLUMNS = 30
_FIXED_COLUMNS = 32
"""Everything but the sparklines and the job label: the `gpu=N` column, both numbers, and
the table's padding. Hand-measured against a `589GB/1480GB`-sized number; a wider one
overflows and Rich ellipsizes the labels rather than shrinking the sparklines."""
the table's padding. Measured against `100%` and a `9999GB/9999GB`-sized number, which fits
hosts under 10 TB; a wider one overflows and Rich ellipsizes the labels rather than shrinking
the sparklines."""

_POWER_FIXED_COLUMNS = 14
"""What the power column adds besides its sparkline: a `1000W/1000W`-sized number, the space
before it, and the column's padding."""

_SPARKLINE_COLUMNS = 2


def _spark_width(console_width: int, label_width: int = 0) -> int:
budget = console_width - _FIXED_COLUMNS - label_width
return max(MIN_SPARK_WIDTH, min(MAX_SPARK_WIDTH, budget // _SPARKLINE_COLUMNS))
def _spark_budget(console_width: int, label_width: int, show_power: bool) -> int:
fixed = _FIXED_COLUMNS + (_POWER_FIXED_COLUMNS if show_power else 0)
columns = _SPARKLINE_COLUMNS + (1 if show_power else 0)
return (console_width - fixed - label_width) // columns


def _spark_width(console_width: int, label_width: int = 0, show_power: bool = False) -> int:
budget = _spark_budget(console_width, label_width, show_power)
return max(MIN_SPARK_WIDTH, min(MAX_SPARK_WIDTH, budget))


def get_metrics_table(
Expand All @@ -48,7 +59,13 @@ def get_metrics_table(
) -> RenderableType:
labels = job_labels(jobs)
label_width = max((len(label) for label in labels), default=0)
width = _spark_width(console_width or console.width, label_width)
console_width = console_width or console.width
# Older runners and non-NVIDIA GPUs report no power. The column is also left out if three
# charts of the minimum width don't fit, as Rich would cut the labels to make room.
show_power = any(_reports_power(m) for m in metrics) and (
_spark_budget(console_width, label_width, show_power=True) >= MIN_SPARK_WIDTH
)
width = _spark_width(console_width, label_width, show_power)
window = _shared_window(metrics)

table = Table(box=None)
Expand All @@ -57,18 +74,21 @@ def get_metrics_table(
table.add_column("", style="secondary", no_wrap=True)
table.add_column("UTILIZATION", no_wrap=True)
table.add_column("MEMORY", no_wrap=True)
if show_power:
table.add_column("POWER", no_wrap=True)
empty_row = [""] * len(table.columns)

for index, (job, job_metrics) in enumerate(zip(jobs, metrics)):
if index:
table.add_row("", "", "", "")
_add_job(table, job, job_metrics, width, labels[index], window)
table.add_row(*empty_row)
_add_job(table, job, job_metrics, width, labels[index], window, show_power)

if window is not None:
table.add_row("", "", "", "")
table.add_row(*empty_row)
# the axis spans the widest chart drawn: a job with fewer samples than cells draws
# one cell per sample and cannot fill its share
axis = _time_axis(max(_cells_drawn(m, window, width) for m in metrics), *window)
table.add_row("", "", axis, axis)
table.add_row("", "", *[axis] * (len(table.columns) - 2))
return table


Expand Down Expand Up @@ -100,23 +120,39 @@ def _add_job(
width: int,
label: str,
window: Optional[tuple[datetime, datetime]],
show_power: bool,
) -> None:
resources = _get_resources(job)
blanks = _blank_cells(metrics, window, width)
cells = width - blanks
table.add_row(
host_row: List[RenderableType] = [
label,
"cpu",
_pad(_cpu_cell(metrics, resources, cells), blanks),
_pad(_memory_cell(metrics, resources, cells), blanks),
)
]
if show_power:
host_row.append("")
table.add_row(*host_row)
for index in range(_gpus_num(metrics, resources)):
table.add_row(
gpu_row: List[RenderableType] = [
"",
f"gpu={index}",
_pad(_gpu_util_cell(metrics, index, cells), blanks),
_pad(_gpu_memory_cell(metrics, resources, index, cells), blanks),
)
]
if show_power:
power_blanks = max(blanks, _blank_cells(_power_series(metrics, index), window, width))
gpu_row.append(
_pad(_gpu_power_cell(metrics, index, width - power_blanks), power_blanks)
)
table.add_row(*gpu_row)


def _power_series(job_metrics: JobMetrics, index: int) -> JobMetrics:
"""Power draw of GPU `index` only. It may start later than the other series."""
name = f"gpu_power_draw_watts_gpu{index}"
return JobMetrics(metrics=[m for m in job_metrics.metrics if m.name == name])


def _shared_window(metrics: Sequence[JobMetrics]) -> Optional[tuple[datetime, datetime]]:
Expand Down Expand Up @@ -197,6 +233,20 @@ def _gpu_util_cell(job_metrics: JobMetrics, index: int, width: int) -> Text:
return Text.assemble(sparkline(values, width, GPU_RAMP), " ", f"{values[-1]:.0f}%")


def _gpu_power_cell(job_metrics: JobMetrics, index: int, width: int) -> Text:
values = _metric_values(job_metrics, f"gpu_power_draw_watts_gpu{index}")
if not values:
return no_data()
limit = _latest_value(job_metrics, f"gpu_power_limit_watts_gpu{index}")
if not limit:
# without the limit there is no fixed scale to chart against
return Text(f"{values[-1]:.0f}W")
percents = [v / limit * 100 for v in values]
return Text.assemble(
sparkline(percents, width, GPU_RAMP), " ", f"{values[-1]:.0f}W/{limit:.0f}W"
)


def _capacity_cell(values: List[float], total: Optional[float], width: int, ramp: Ramp) -> Text:
"""A memory chart drawn against capacity, labelled `used/total`."""
percents = [v / total * 100 for v in values] if total else values
Expand Down Expand Up @@ -252,6 +302,10 @@ def _latest_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
return values[-1] if values else None


def _reports_power(job_metrics: JobMetrics) -> bool:
return any(m.name.startswith("gpu_power_draw_watts_gpu") for m in job_metrics.metrics)


def _gpus_num(job_metrics: JobMetrics, resources: Optional[Resources]) -> int:
if resources is not None and resources.gpus:
return len(resources.gpus)
Expand Down
11 changes: 11 additions & 0 deletions src/dstack/_internal/server/background/scheduled_tasks/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@ async def _collect_job_metrics(job_model: JobModel) -> Optional[JobMetricsPoint]

gpus_memory_usage_bytes = [g.gpu_memory_usage_bytes for g in res.gpus]
gpus_util_percent = [g.gpu_util_percent for g in res.gpus]
gpus_power_draw_watts = [g.gpu_power_draw_watts for g in res.gpus]
gpus_power_limit_watts = [g.gpu_power_limit_watts for g in res.gpus]

return JobMetricsPoint(
job_id=job_model.id,
Expand All @@ -161,9 +163,18 @@ async def _collect_job_metrics(job_model: JobModel) -> Optional[JobMetricsPoint]
memory_working_set_bytes=res.memory_working_set_bytes,
gpus_memory_usage_bytes=json.dumps(gpus_memory_usage_bytes),
gpus_util_percent=json.dumps(gpus_util_percent),
gpus_power_draw_watts=_dump_gpus_power(gpus_power_draw_watts),
gpus_power_limit_watts=_dump_gpus_power(gpus_power_limit_watts),
)


def _dump_gpus_power(values: List[Optional[float]]) -> Optional[str]:
# NULL if no GPU has a reading, as with older runners and non-NVIDIA GPUs
if all(v is None for v in values):
return None
return json.dumps(values)


@runner_ssh_tunnel
def _pull_runner_metrics(
addresses: Mapping[int, client.LocalAddress],
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
"""Add JobMetricsPoint GPU power columns

Revision ID: eb9ea2fdd147
Revises: 2d2692d5b3c2
Create Date: 2026-11-20 00:19:27.385130+00:00

"""

import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "eb9ea2fdd147"
down_revision = "2d2692d5b3c2"
branch_labels = None
depends_on = None


def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("job_metrics_points", schema=None) as batch_op:
batch_op.add_column(sa.Column("gpus_power_draw_watts", sa.Text(), nullable=True))
batch_op.add_column(sa.Column("gpus_power_limit_watts", sa.Text(), nullable=True))

# ### end Alembic commands ###


def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("job_metrics_points", schema=None) as batch_op:
batch_op.drop_column("gpus_power_limit_watts")
batch_op.drop_column("gpus_power_draw_watts")

# ### end Alembic commands ###
Loading