Skip to content

Commit 9ebc306

Browse files
authored
Merge pull request #3061 from trycompai/main
[comp] Production Deploy
2 parents c57f71e + 8ec2cff commit 9ebc306

12 files changed

Lines changed: 884 additions & 9 deletions

File tree

‎apps/api/src/cloud-security/ai-remediation.prompt.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ NEVER omit AWSServiceName, leave it as null, or use a placeholder string.
290290
- NEVER use placeholder values like "{{variable}}", "<PLACEHOLDER>", or template syntax
291291
- ALWAYS use concrete values in fix step params
292292
- If a value depends on the account (like a log group name), put the discovery in readSteps and use a reasonable default or convention in fixSteps:
293-
- CloudTrail log group: discover the trail's CloudWatch Logs log group in a read step (e.g. from the trail's CloudWatchLogsLogGroupArn) and use that exact, real log group name in fixSteps — do not invent a name
293+
- CloudTrail log group: the finding evidence provides the real log group as "cloudWatchLogGroupName" — use that exact value for logGroupName in fixSteps. Only if it is absent, discover it in a read step (from the trail's CloudWatchLogsLogGroupArn) and use that exact, real name. Never invent a name and never use a placeholder like "CloudTrail/DefaultLogGroup"
294294
- SNS topic: use "CompAI-CIS-Alerts" (will be created if it doesn't exist)
295295
- KMS keys: use "alias/aws/service-name" for AWS-managed keys
296296
- The finding evidence contains REAL data from the AWS account scan — use those values

‎apps/api/src/cloud-security/ai-remediation.service.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ import {
2828
} from './azure-ai-remediation.prompt';
2929
import { normalizeFixPlan } from './plan-normalizer';
3030

31-
const MODEL = anthropic('claude-opus-4-6');
31+
const MODEL = anthropic('claude-opus-4-8');
3232
// Cheaper, faster model for the manual-steps fallback. The output is pure
3333
// natural language with no SDK-call shape to validate, so the strongest
3434
// model is overkill — we just need clear instructions.
35-
const FALLBACK_MODEL = anthropic('claude-sonnet-4-5');
35+
const FALLBACK_MODEL = anthropic('claude-sonnet-4-6');
3636
const REMEDIATION_ROLE_NAME = 'CompAI-Remediator';
3737

3838
export interface FindingContext {
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import { applyResolvedMetricFilterLogGroup } from './metric-filter-loggroup';
2+
import type { AwsCommandStep } from './ai-remediation.prompt';
3+
4+
const putMetricFilterStep = (params: Record<string, unknown>): AwsCommandStep => ({
5+
service: 'cloudwatch-logs',
6+
command: 'PutMetricFilterCommand',
7+
params,
8+
purpose: 'create metric filter',
9+
});
10+
11+
describe('applyResolvedMetricFilterLogGroup', () => {
12+
it('pins the trail log group from cloudWatchLogGroupName (missing-filter case)', () => {
13+
const steps = [putMetricFilterStep({ logGroupName: '', filterName: 'f' })];
14+
applyResolvedMetricFilterLogGroup(steps, {
15+
cloudWatchLogGroupName: 'my-ct-logs',
16+
});
17+
expect(steps[0].params.logGroupName).toBe('my-ct-logs');
18+
});
19+
20+
it('uses the existing filter log group (update case) when cloudWatchLogGroupName is absent', () => {
21+
const steps = [putMetricFilterStep({ logGroupName: 'wrong', filterName: 'f' })];
22+
applyResolvedMetricFilterLogGroup(steps, { logGroupName: 'existing-lg' });
23+
expect(steps[0].params.logGroupName).toBe('existing-lg');
24+
});
25+
26+
it('overwrites a wrong/placeholder value the AI produced', () => {
27+
const steps = [
28+
putMetricFilterStep({ logGroupName: 'CloudTrail/DefaultLogGroup' }),
29+
];
30+
applyResolvedMetricFilterLogGroup(steps, {
31+
cloudWatchLogGroupName: 'real-lg',
32+
});
33+
expect(steps[0].params.logGroupName).toBe('real-lg');
34+
});
35+
36+
it('only touches PutMetricFilter steps', () => {
37+
const other: AwsCommandStep = {
38+
service: 'sns',
39+
command: 'CreateTopicCommand',
40+
params: { Name: 'compai-cis-alerts' },
41+
purpose: 'create topic',
42+
};
43+
const steps = [other, putMetricFilterStep({ logGroupName: '' })];
44+
applyResolvedMetricFilterLogGroup(steps, {
45+
cloudWatchLogGroupName: 'my-ct-logs',
46+
});
47+
expect(steps[0].params).toEqual({ Name: 'compai-cis-alerts' }); // untouched
48+
expect(steps[1].params.logGroupName).toBe('my-ct-logs');
49+
});
50+
51+
it('is a no-op when no log group was resolved (e.g. DescribeTrails denied)', () => {
52+
const steps = [putMetricFilterStep({ logGroupName: '' })];
53+
applyResolvedMetricFilterLogGroup(steps, { keywords: ['Root'] });
54+
expect(steps[0].params.logGroupName).toBe('');
55+
});
56+
});
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import type { AwsCommandStep } from './ai-remediation.prompt';
2+
3+
/**
4+
* Force the real CloudTrail log group onto every PutMetricFilter fix step.
5+
*
6+
* PutMetricFilter cannot work without the exact CloudWatch Logs log group, and
7+
* the AI must never be the source of truth for it (it guessed and failed before,
8+
* surfacing "could not determine which CloudTrail log group..."). The scan
9+
* resolves the real log group deterministically and carries it in the finding
10+
* evidence — `cloudWatchLogGroupName` for a missing filter, `logGroupName` for an
11+
* existing-filter update — so we overwrite the step's logGroupName from evidence,
12+
* guaranteeing the executed value is correct regardless of what the model
13+
* produced. No-op for any non-PutMetricFilter step or when no log group was
14+
* resolved (e.g. DescribeTrails was denied at scan time).
15+
*/
16+
export function applyResolvedMetricFilterLogGroup(
17+
steps: AwsCommandStep[],
18+
evidence: Record<string, unknown>,
19+
): void {
20+
const fromName =
21+
typeof evidence.cloudWatchLogGroupName === 'string' &&
22+
evidence.cloudWatchLogGroupName.trim().length > 0
23+
? evidence.cloudWatchLogGroupName
24+
: null;
25+
const fromExisting =
26+
typeof evidence.logGroupName === 'string' &&
27+
evidence.logGroupName.trim().length > 0
28+
? evidence.logGroupName
29+
: null;
30+
const resolved = fromName ?? fromExisting;
31+
if (!resolved) return;
32+
33+
for (const step of steps) {
34+
if (step.command !== 'PutMetricFilterCommand') continue;
35+
if (!step.params || typeof step.params !== 'object') continue;
36+
(step.params as Record<string, unknown>).logGroupName = resolved;
37+
}
38+
}
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
const mockTrailSend = jest.fn();
2+
const mockLogsSend = jest.fn();
3+
const mockCwSend = jest.fn();
4+
5+
jest.mock('@aws-sdk/client-cloudtrail', () => ({
6+
CloudTrailClient: jest.fn(() => ({ send: mockTrailSend })),
7+
DescribeTrailsCommand: jest.fn((input: unknown) => ({
8+
_cmd: 'DescribeTrails',
9+
input,
10+
})),
11+
}));
12+
jest.mock('@aws-sdk/client-cloudwatch-logs', () => ({
13+
CloudWatchLogsClient: jest.fn(() => ({ send: mockLogsSend })),
14+
DescribeMetricFiltersCommand: jest.fn((input: unknown) => ({
15+
_cmd: 'DescribeMetricFilters',
16+
input,
17+
})),
18+
}));
19+
jest.mock('@aws-sdk/client-cloudwatch', () => ({
20+
CloudWatchClient: jest.fn(() => ({ send: mockCwSend })),
21+
DescribeAlarmsForMetricCommand: jest.fn((input: unknown) => ({
22+
_cmd: 'DescribeAlarmsForMetric',
23+
input,
24+
})),
25+
}));
26+
27+
import { CloudWatchAdapter, logGroupNameFromArn } from './cloudwatch.adapter';
28+
29+
const CREDS = {
30+
accessKeyId: 'AKIA',
31+
secretAccessKey: 'secret',
32+
sessionToken: 'token',
33+
};
34+
35+
const scan = () =>
36+
new CloudWatchAdapter().scan({ credentials: CREDS, region: 'us-east-1' });
37+
38+
beforeEach(() => {
39+
jest.clearAllMocks();
40+
mockCwSend.mockResolvedValue({ MetricAlarms: [] });
41+
});
42+
43+
describe('logGroupNameFromArn', () => {
44+
it('derives the bare name and strips the trailing :*', () => {
45+
expect(
46+
logGroupNameFromArn(
47+
'arn:aws:logs:us-east-1:123456789012:log-group:aws-cloudtrail-logs-xyz:*',
48+
),
49+
).toBe('aws-cloudtrail-logs-xyz');
50+
});
51+
52+
it('handles an ARN without a trailing :*', () => {
53+
expect(
54+
logGroupNameFromArn('arn:aws:logs:us-east-1:123:log-group:my-lg'),
55+
).toBe('my-lg');
56+
});
57+
58+
it('handles the GovCloud partition', () => {
59+
expect(
60+
logGroupNameFromArn(
61+
'arn:aws-us-gov:logs:us-gov-west-1:123:log-group:ct-logs:*',
62+
),
63+
).toBe('ct-logs');
64+
});
65+
66+
it('returns null for a missing or malformed ARN', () => {
67+
expect(logGroupNameFromArn(undefined)).toBeNull();
68+
expect(logGroupNameFromArn('not-an-arn')).toBeNull();
69+
});
70+
});
71+
72+
describe('CloudWatchAdapter — CloudTrail log group resolution', () => {
73+
it('injects the real log group name into the metric-filter-missing finding (the customer bug)', async () => {
74+
mockTrailSend.mockResolvedValue({
75+
trailList: [
76+
{
77+
Name: 'main',
78+
CloudWatchLogsLogGroupArn:
79+
'arn:aws:logs:us-east-1:123456789012:log-group:my-ct-logs:*',
80+
},
81+
],
82+
});
83+
mockLogsSend.mockResolvedValue({ metricFilters: [] }); // nothing configured
84+
85+
const findings = await scan();
86+
const missing = findings.find((f) =>
87+
f.title.includes('metric filter missing'),
88+
);
89+
90+
expect(missing).toBeDefined();
91+
expect(missing!.evidence?.cloudWatchLogGroupName).toBe('my-ct-logs');
92+
expect(missing!.remediation).toContain('logGroupName set to "my-ct-logs"');
93+
// The old generic phrasing that forced the AI to guess must be gone.
94+
expect(missing!.remediation).not.toContain(
95+
'logGroupName set to the CloudTrail log group',
96+
);
97+
});
98+
99+
it('uses the existing filter\'s own log group for the no-transformation update', async () => {
100+
mockTrailSend.mockResolvedValue({
101+
trailList: [
102+
{
103+
Name: 'main',
104+
CloudWatchLogsLogGroupArn:
105+
'arn:aws:logs:us-east-1:123:log-group:my-ct-logs:*',
106+
},
107+
],
108+
});
109+
// A filter matching CIS 4.3 (Root account usage) keywords, but with no
110+
// metric transformation → "no metric transformation" finding.
111+
mockLogsSend.mockResolvedValue({
112+
metricFilters: [
113+
{
114+
filterName: 'root-filter',
115+
filterPattern: '{ $.userIdentity.type = "Root" }',
116+
logGroupName: 'existing-lg',
117+
metricTransformations: [],
118+
},
119+
],
120+
});
121+
122+
const findings = await scan();
123+
const noTransform = findings.find((f) =>
124+
f.title.includes('no metric transformation'),
125+
);
126+
127+
expect(noTransform).toBeDefined();
128+
expect(noTransform!.evidence?.logGroupName).toBe('existing-lg');
129+
expect(noTransform!.remediation).toContain('logGroupName set to "existing-lg"');
130+
});
131+
132+
it('returns the prerequisite finding when no trail integrates with CloudWatch Logs', async () => {
133+
mockTrailSend.mockResolvedValue({
134+
trailList: [{ Name: 'main' }], // no CloudWatchLogsLogGroupArn
135+
});
136+
137+
const findings = await scan();
138+
expect(findings).toHaveLength(1);
139+
expect(findings[0].title).toMatch(/not integrated with CloudWatch Logs/i);
140+
});
141+
142+
it('falls back to a generic instruction when DescribeTrails is denied', async () => {
143+
// DescribeTrails throws (e.g. missing cloudtrail:DescribeTrails) -> the log
144+
// group stays unknown, so the finding keeps the generic text rather than a
145+
// wrong name. (Not the customer's case, but must not crash or fabricate.)
146+
mockTrailSend.mockRejectedValue(new Error('AccessDenied'));
147+
mockLogsSend.mockResolvedValue({ metricFilters: [] });
148+
149+
const findings = await scan();
150+
const missing = findings.find((f) =>
151+
f.title.includes('metric filter missing'),
152+
);
153+
expect(missing).toBeDefined();
154+
expect(missing!.evidence?.cloudWatchLogGroupName).toBeUndefined();
155+
expect(missing!.remediation).toContain("CloudWatch Logs log group");
156+
});
157+
});

0 commit comments

Comments
 (0)