Repository navigation
fix(api): give each alert only its own history map in check-alerts #3668
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| --- | ||
| '@hyperdx/api': patch | ||
| --- | ||
|
|
||
| perf: check-alerts gives each alert only its own previous history, so a tick | ||
| no longer scans every alert's group history once per alert |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,11 +3,19 @@ import mongoose from 'mongoose'; | |
|
|
||
| import * as config from '@/config'; | ||
| import { ObjectId } from '@/models'; | ||
| import { AlertSource, AlertThresholdType, IAlert } from '@/models/alert'; | ||
| import { | ||
| AlertSource, | ||
| AlertState, | ||
| AlertThresholdType, | ||
| IAlert, | ||
| } from '@/models/alert'; | ||
| import { ISource } from '@/models/source'; | ||
| import { IWebhook } from '@/models/webhook'; | ||
| import * as checkAlerts from '@/tasks/checkAlerts'; | ||
| import CheckAlertTask from '@/tasks/checkAlerts'; | ||
| import CheckAlertTask, { | ||
| AggregatedAlertHistory, | ||
| partitionPreviousMapsByAlert, | ||
| } from '@/tasks/checkAlerts'; | ||
| import { | ||
| AlertDetails, | ||
| AlertProvider, | ||
|
|
@@ -24,6 +32,130 @@ jest.mock('@/tasks/checkAlerts/providers', () => { | |
| }; | ||
| }); | ||
|
|
||
| type TestHistory = Omit<AggregatedAlertHistory, '_id'>; | ||
|
|
||
| const buildHistory = (group?: string): TestHistory => ({ | ||
| createdAt: new Date('2026-01-01T00:00:00Z'), | ||
| state: AlertState.OK, | ||
| group, | ||
| }); | ||
|
|
||
| const buildPartitionTask = ( | ||
| alerts: { id: string; previousMap: Map<string, TestHistory> }[], | ||
| teamId = new mongoose.Types.ObjectId(), | ||
| ): AlertTask => | ||
| ({ | ||
| alerts: alerts.map(({ id, previousMap }) => ({ | ||
| alert: { | ||
| id, | ||
| team: { _id: teamId }, | ||
| source: AlertSource.SAVED_SEARCH, | ||
| threshold: 10, | ||
| thresholdType: AlertThresholdType.ABOVE, | ||
| interval: '1m', | ||
| channel: { type: 'webhook', webhookId: 'webhook-123' }, | ||
| }, | ||
| taskType: AlertTaskType.SAVED_SEARCH, | ||
| previousMap, | ||
| })), | ||
| conn: { | ||
| id: 'conn-123', | ||
| _id: new mongoose.Types.ObjectId(), | ||
| host: config.CLICKHOUSE_HOST, | ||
| username: config.CLICKHOUSE_USER, | ||
| password: config.CLICKHOUSE_PASSWORD, | ||
| name: '', | ||
| team: teamId, | ||
| }, | ||
| now: new Date(), | ||
| }) as unknown as AlertTask; | ||
|
|
||
| describe('partitionPreviousMapsByAlert', () => { | ||
| it('gives each alert only its own entries from a shared map, keys unchanged', () => { | ||
| const groupedA = buildHistory('service:api'); | ||
| const groupedB = buildHistory('service:web'); | ||
| const ungrouped = buildHistory(); | ||
| const otherAlert = buildHistory(); | ||
| const shared = new Map([ | ||
| ['grouped||service:api', groupedA], | ||
| ['grouped||service:web', groupedB], | ||
| ['ungrouped', ungrouped], | ||
| ['not-in-tick', otherAlert], | ||
| ]); | ||
|
|
||
| const [task] = partitionPreviousMapsByAlert([ | ||
| buildPartitionTask([ | ||
| { id: 'grouped', previousMap: shared }, | ||
| { id: 'ungrouped', previousMap: shared }, | ||
| { id: 'no-history', previousMap: shared }, | ||
| ]), | ||
| ]); | ||
|
|
||
| expect(task.alerts[0].previousMap).toEqual( | ||
| new Map([ | ||
| ['grouped||service:api', groupedA], | ||
| ['grouped||service:web', groupedB], | ||
| ]), | ||
| ); | ||
| expect(task.alerts[1].previousMap).toEqual( | ||
| new Map([['ungrouped', ungrouped]]), | ||
| ); | ||
| expect(task.alerts[2].previousMap.size).toBe(0); | ||
| }); | ||
|
|
||
| it('keeps a group name that contains the delimiter with its alert', () => { | ||
| const history = buildHistory('path:a||b'); | ||
| const shared = new Map([['alert-1||path:a||b', history]]); | ||
|
|
||
| const [task] = partitionPreviousMapsByAlert([ | ||
| buildPartitionTask([{ id: 'alert-1', previousMap: shared }]), | ||
| ]); | ||
|
|
||
| expect(task.alerts[0].previousMap).toEqual( | ||
| new Map([['alert-1||path:a||b', history]]), | ||
| ); | ||
| }); | ||
|
|
||
| it('splits maps shared across tasks and keeps already separate maps', () => { | ||
| const shared = new Map([ | ||
| ['alert-1', buildHistory()], | ||
| ['alert-2||g:1', buildHistory('g:1')], | ||
| ]); | ||
| const own = new Map([['alert-3', buildHistory()]]); | ||
|
|
||
| const tasks = partitionPreviousMapsByAlert([ | ||
| buildPartitionTask([{ id: 'alert-1', previousMap: shared }]), | ||
| buildPartitionTask([ | ||
| { id: 'alert-2', previousMap: shared }, | ||
| { id: 'alert-3', previousMap: own }, | ||
| ]), | ||
| ]); | ||
|
|
||
| expect([...tasks[0].alerts[0].previousMap.keys()]).toEqual(['alert-1']); | ||
| expect([...tasks[1].alerts[0].previousMap.keys()]).toEqual([ | ||
| 'alert-2||g:1', | ||
| ]); | ||
| expect([...tasks[1].alerts[1].previousMap.keys()]).toEqual(['alert-3']); | ||
| }); | ||
|
|
||
| it('does not mutate the input tasks or maps', () => { | ||
| const shared = new Map([ | ||
| ['alert-1', buildHistory()], | ||
| ['alert-2', buildHistory()], | ||
| ]); | ||
| const input = buildPartitionTask([ | ||
| { id: 'alert-1', previousMap: shared }, | ||
| { id: 'alert-2', previousMap: shared }, | ||
| ]); | ||
|
|
||
| partitionPreviousMapsByAlert([input]); | ||
|
|
||
| expect(shared.size).toBe(2); | ||
| expect(input.alerts[0].previousMap).toBe(shared); | ||
| expect(input.alerts[1].previousMap).toBe(shared); | ||
| }); | ||
| }); | ||
|
|
||
| describe('CheckAlertTask', () => { | ||
| describe('execute', () => { | ||
| let mockAlertProvider: jest.Mocked<AlertProvider>; | ||
|
|
@@ -331,6 +463,53 @@ describe('CheckAlertTask', () => { | |
| ); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔵 minor — No test covers the behaviour change on the task path: other alerts' ALERT/PENDING rows no longer affect a grouped alert The PR changes what Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond |
||
| }); | ||
|
|
||
| it('should pass each alert only its own previous history', async () => { | ||
| const args: CheckAlertsTaskArgs = { taskName: TaskName.CHECK_ALERTS }; | ||
| const task = new CheckAlertTask(args); | ||
|
|
||
| const groupA = buildHistory('g:a'); | ||
| const groupB = buildHistory('g:b'); | ||
| const ungrouped = buildHistory(); | ||
| const shared = new Map([ | ||
| ['alert-1||g:a', groupA], | ||
| ['alert-1||g:b', groupB], | ||
| ['alert-2', ungrouped], | ||
| ]); | ||
|
|
||
| mockAlertProvider.getAlertTasks.mockResolvedValue([ | ||
| buildPartitionTask([ | ||
| { id: 'alert-1', previousMap: shared }, | ||
| { id: 'alert-2', previousMap: shared }, | ||
| ]), | ||
| ]); | ||
| mockAlertProvider.getWebhooks.mockResolvedValue(new Map()); | ||
| mockAlertProvider.getClickHouseClient.mockResolvedValue( | ||
| new ClickhouseClient({}), | ||
| ); | ||
| const receivedMaps = new Map<string, Map<string, unknown>>(); | ||
| mockProcessAlert.mockImplementation( | ||
| async (_now: Date, details: AlertDetails) => { | ||
| receivedMaps.set(details.alert.id, details.previousMap); | ||
| }, | ||
| ); | ||
|
|
||
| await task.execute(); | ||
|
|
||
| expect(receivedMaps).toEqual( | ||
| new Map([ | ||
| [ | ||
| 'alert-1', | ||
| new Map([ | ||
| ['alert-1||g:a', groupA], | ||
| ['alert-1||g:b', groupB], | ||
| ]), | ||
| ], | ||
| ['alert-2', new Map([['alert-2', ungrouped]])], | ||
| ]), | ||
| ); | ||
| expect(shared.size).toBe(3); | ||
| }); | ||
|
|
||
| describe('when a connection fails', () => { | ||
| // The task queue runs batches fire-and-forget, and the worker's global | ||
| // unhandledRejection handler exits the process, so a single failing | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Could this test also assert that the shared map is iterated only once across tasks? I disabled the cache lookup and all 13 tests still passed, although each alert scanned the shared map again. An iterator-count assertion catches that regression without relying on timing.