Skip to content

Commit 163d9e3

Browse files
brokensound77rw-access
brokensound77
and
rw-access
authoredJul 21, 2021
Update cardinality field in schema for threshold rules (elastic#1349)
* Make cardinality array in schema for threshold rules * update master, 7.12, 7.13, and 7.14 schemas with cardinality fix * fix 7.12 downgrade to handle cardinality as an array * Add two new rules to detect agent spoofing Co-authored-by: Ross Wolf <31489089+rw-access@users.noreply.github.com>
·
v7.15.0ML-HostRiskScore-20210803-1
1 parent 95e6458 commit 163d9e3

13 files changed

+216
-72
lines changed
 

‎detection_rules/rule.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ class BaseRuleData(MarshmallowDataclassMixin):
183183
def save_schema(cls):
184184
"""Save the schema as a jsonschema."""
185185
fields: List[dataclasses.Field] = dataclasses.fields(cls)
186-
type_field = next(field for field in fields if field.name == "type")
186+
type_field = next(f for f in fields if f.name == "type")
187187
rule_type = typing.get_args(type_field.type)[0] if cls != BaseRuleData else "base"
188188
schema = cls.jsonschema()
189189
version_dir = SCHEMA_DIR / "master"
@@ -256,9 +256,9 @@ class ThresholdCardinality:
256256
field: str
257257
value: definitions.ThresholdValue
258258

259-
field: List[definitions.NonEmptyStr]
259+
field: definitions.CardinalityFields
260260
value: definitions.ThresholdValue
261-
cardinality: Optional[ThresholdCardinality]
261+
cardinality: Optional[List[ThresholdCardinality]]
262262

263263
type: Literal["threshold"]
264264
threshold: ThresholdMapping

‎detection_rules/rule_formatter.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -206,5 +206,5 @@ def _do_write(_data, _contents):
206206
_do_write(data, _contents)
207207

208208
finally:
209-
if needs_close:
209+
if needs_close and hasattr(outfile, "close"):
210210
outfile.close()

‎detection_rules/schemas/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ def downgrade_threshold_to_7_11(version: Version, api_contents: dict) -> dict:
123123
if len(threshold_field) > 1:
124124
raise ValueError('Cannot downgrade a threshold rule that has multiple threshold fields defined')
125125

126-
if threshold.get('cardinality', {}).get('field') or threshold.get('cardinality', {}).get('value'):
126+
if threshold.get('cardinality'):
127127
raise ValueError('Cannot downgrade a threshold rule that has a defined cardinality')
128128

129129
api_contents = api_contents.copy()

‎detection_rules/schemas/definitions.py

+5-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
"""Custom shared definitions for schemas."""
77

8-
from typing import Literal, Final
8+
from typing import List, Literal, Final
99

1010
from marshmallow import validate
1111
from marshmallow_dataclass import NewType
@@ -44,19 +44,21 @@
4444
}
4545

4646

47+
NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1))
48+
4749
BranchVer = NewType('BranchVer', str, validate=validate.Regexp(BRANCH_PATTERN))
50+
CardinalityFields = NewType('CardinalityFields', List[NonEmptyStr], validate=validate.Length(min=0, max=3))
4851
CodeString = NewType("CodeString", str)
4952
ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN))
5053
Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
5154
FilterLanguages = Literal["kuery", "lucene"]
5255
Interval = NewType('Interval', str, validate=validate.Regexp(INTERVAL_PATTERN))
53-
PositiveInteger = NewType('PositiveInteger', int, validate=validate.Range(min=1))
5456
Markdown = NewType("MarkdownField", CodeString)
5557
Maturity = Literal['development', 'experimental', 'beta', 'production', 'deprecated']
5658
MaxSignals = NewType("MaxSignals", int, validate=validate.Range(min=1))
57-
NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1))
5859
Operator = Literal['equals']
5960
OSType = Literal['windows', 'linux', 'macos']
61+
PositiveInteger = NewType('PositiveInteger', int, validate=validate.Range(min=1))
6062
RiskScore = NewType("MaxSignals", int, validate=validate.Range(min=1, max=100))
6163
RuleType = Literal['query', 'saved_query', 'machine_learning', 'eql', 'threshold', 'threat_match']
6264
SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN))

‎etc/api_schemas/7.12/7.12.threshold.json

+26-15
Original file line numberDiff line numberDiff line change
@@ -919,35 +919,46 @@
919919
"additionalProperties": false,
920920
"properties": {
921921
"cardinality": {
922-
"additionalProperties": false,
923-
"properties": {
924-
"field": {
925-
"type": "string"
922+
"items": {
923+
"additionalProperties": false,
924+
"properties": {
925+
"field": {
926+
"type": "string"
927+
},
928+
"value": {
929+
"description": "ThresholdValue",
930+
"format": "integer",
931+
"minimum": 1,
932+
"type": "number"
933+
}
926934
},
927-
"value": {
928-
"minimum": 1,
929-
"type": "integer"
930-
}
935+
"required": [
936+
"field",
937+
"value"
938+
],
939+
"type": "object"
931940
},
932-
"required": [
933-
"field",
934-
"value"
935-
],
936-
"type": "object"
941+
"type": "array"
937942
},
938943
"field": {
944+
"description": "CardinalityFields",
939945
"items": {
940-
"default": "",
946+
"description": "NonEmptyStr",
947+
"minLength": 1,
941948
"type": "string"
942949
},
950+
"maxItems": 3,
943951
"type": "array"
944952
},
945953
"value": {
954+
"description": "ThresholdValue",
955+
"format": "integer",
946956
"minimum": 1,
947-
"type": "integer"
957+
"type": "number"
948958
}
949959
},
950960
"required": [
961+
"field",
951962
"value"
952963
],
953964
"type": "object"

‎etc/api_schemas/7.13/7.13.threshold.json

+32-15
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,9 @@
112112
"type": "string"
113113
},
114114
"operator": {
115+
"enum": [
116+
"equals"
117+
],
115118
"type": "string"
116119
},
117120
"value": {
@@ -151,6 +154,9 @@
151154
"type": "string"
152155
},
153156
"operator": {
157+
"enum": [
158+
"equals"
159+
],
154160
"type": "string"
155161
},
156162
"severity": {
@@ -178,6 +184,9 @@
178184
"additionalProperties": false,
179185
"properties": {
180186
"framework": {
187+
"enum": [
188+
"MITRE ATT&CK"
189+
],
181190
"type": "string"
182191
},
183192
"tactic": {
@@ -265,30 +274,35 @@
265274
"additionalProperties": false,
266275
"properties": {
267276
"cardinality": {
268-
"additionalProperties": false,
269-
"properties": {
270-
"field": {
271-
"type": "string"
277+
"items": {
278+
"additionalProperties": false,
279+
"properties": {
280+
"field": {
281+
"type": "string"
282+
},
283+
"value": {
284+
"description": "ThresholdValue",
285+
"format": "integer",
286+
"minimum": 1,
287+
"type": "number"
288+
}
272289
},
273-
"value": {
274-
"description": "ThresholdValue",
275-
"format": "integer",
276-
"minimum": 1,
277-
"type": "number"
278-
}
290+
"required": [
291+
"field",
292+
"value"
293+
],
294+
"type": "object"
279295
},
280-
"required": [
281-
"field",
282-
"value"
283-
],
284-
"type": "object"
296+
"type": "array"
285297
},
286298
"field": {
299+
"description": "CardinalityFields",
287300
"items": {
288301
"description": "NonEmptyStr",
289302
"minLength": 1,
290303
"type": "string"
291304
},
305+
"maxItems": 3,
292306
"type": "array"
293307
},
294308
"value": {
@@ -336,6 +350,9 @@
336350
"type": "string"
337351
},
338352
"type": {
353+
"enum": [
354+
"threshold"
355+
],
339356
"type": "string"
340357
}
341358
},

‎etc/api_schemas/7.14/7.14.threshold.json

+20-15
Original file line numberDiff line numberDiff line change
@@ -274,30 +274,35 @@
274274
"additionalProperties": false,
275275
"properties": {
276276
"cardinality": {
277-
"additionalProperties": false,
278-
"properties": {
279-
"field": {
280-
"type": "string"
277+
"items": {
278+
"additionalProperties": false,
279+
"properties": {
280+
"field": {
281+
"type": "string"
282+
},
283+
"value": {
284+
"description": "ThresholdValue",
285+
"format": "integer",
286+
"minimum": 1,
287+
"type": "number"
288+
}
281289
},
282-
"value": {
283-
"description": "ThresholdValue",
284-
"format": "integer",
285-
"minimum": 1,
286-
"type": "number"
287-
}
290+
"required": [
291+
"field",
292+
"value"
293+
],
294+
"type": "object"
288295
},
289-
"required": [
290-
"field",
291-
"value"
292-
],
293-
"type": "object"
296+
"type": "array"
294297
},
295298
"field": {
299+
"description": "CardinalityFields",
296300
"items": {
297301
"description": "NonEmptyStr",
298302
"minLength": 1,
299303
"type": "string"
300304
},
305+
"maxItems": 3,
301306
"type": "array"
302307
},
303308
"value": {

‎etc/api_schemas/master/master.threshold.json

+20-15
Original file line numberDiff line numberDiff line change
@@ -274,30 +274,35 @@
274274
"additionalProperties": false,
275275
"properties": {
276276
"cardinality": {
277-
"additionalProperties": false,
278-
"properties": {
279-
"field": {
280-
"type": "string"
277+
"items": {
278+
"additionalProperties": false,
279+
"properties": {
280+
"field": {
281+
"type": "string"
282+
},
283+
"value": {
284+
"description": "ThresholdValue",
285+
"format": "integer",
286+
"minimum": 1,
287+
"type": "number"
288+
}
281289
},
282-
"value": {
283-
"description": "ThresholdValue",
284-
"format": "integer",
285-
"minimum": 1,
286-
"type": "number"
287-
}
290+
"required": [
291+
"field",
292+
"value"
293+
],
294+
"type": "object"
288295
},
289-
"required": [
290-
"field",
291-
"value"
292-
],
293-
"type": "object"
296+
"type": "array"
294297
},
295298
"field": {
299+
"description": "CardinalityFields",
296300
"items": {
297301
"description": "NonEmptyStr",
298302
"minLength": 1,
299303
"type": "string"
300304
},
305+
"maxItems": 3,
301306
"type": "array"
302307
},
303308
"value": {

‎etc/stack-schema-map.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@
99

1010
"7.14.0":
1111
beats: "master" # TODO: 7.14.x
12-
ecs: "1.10.0"
12+
ecs: "master" # TODO: master came out after 7.13.0 release

‎requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ PyYAML~=5.3
88
eql==0.9.9
99
elasticsearch~=7.9
1010
XlsxWriter~=1.3.6
11-
marshmallow~=3.10.0
11+
marshmallow~=3.12.2
1212
marshmallow-dataclass[union]~=8.4
1313

1414
# test deps

‎rules/cross-platform/defense_evasion_agent_spoofing_mismatched_id.toml

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
[metadata]
2+
creation_date = "2021/07/14"
3+
maturity = "production"
4+
updated_date = "2021/07/14"
5+
min_stack_version = "7.14.0"
6+
7+
[rule]
8+
author = ["Elastic"]
9+
description = """Detects events which have a mismatch on the expected event agent ID. The status "agent_id_mismatch"
10+
occurs when the expected agent ID associated with the API key does not match the actual agent ID in an event. This could
11+
indicate attempts to spoof events in order to masquerade actual activity to evade detection.
12+
"""
13+
false_positives = [
14+
"""
15+
This is meant to run only on datasources using agents v7.14+ since versions prior to that will be missing the
16+
necessary field, resulting in false positives.
17+
""",
18+
]
19+
from = "now-9m"
20+
index = ["logs-*", "metrics-*", "traces-*"]
21+
language = "kuery"
22+
license = "Elastic License v2"
23+
name = "Agent Spoofing - Mismatched Agent ID"
24+
risk_score = 73
25+
rule_id = "3115bd2c-0baa-4df0-80ea-45e474b5ef93"
26+
severity = "high"
27+
tags = ["Elastic", "Threat Detection", "Defense Evasion"]
28+
timestamp_override = "event.ingested"
29+
type = "query"
30+
31+
query = '''
32+
event.agent_id_status:agent_id_mismatch
33+
'''
34+
35+
36+
[[rule.threat]]
37+
framework = "MITRE ATT&CK"
38+
[[rule.threat.technique]]
39+
id = "T1036"
40+
name = "Masquerading"
41+
reference = "https://attack.mitre.org/techniques/T1036/"
42+
43+
44+
[rule.threat.tactic]
45+
id = "TA0005"
46+
name = "Defense Evasion"
47+
reference = "https://attack.mitre.org/tactics/TA0005/"
48+

‎rules/cross-platform/defense_evasion_agent_spoofing_multiple_hosts.toml

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
[metadata]
2+
creation_date = "2021/07/14"
3+
maturity = "production"
4+
updated_date = "2021/07/14"
5+
min_stack_version = "7.14.0"
6+
7+
[rule]
8+
author = ["Elastic"]
9+
description = """Detects when multiple hosts are using the same agent ID. This could occur in the event of an agent
10+
being taken over and used to inject illegitimate documents into an instance as an attempt to spoof events in order to
11+
masquerade actual activity to evade detection.
12+
"""
13+
false_positives = [
14+
"""
15+
This is meant to run only on datasources using agents v7.14+ since versions prior to that will be missing the
16+
necessary field, resulting in false positives.
17+
""",
18+
]
19+
from = "now-9m"
20+
index = ["logs-*", "metrics-*", "traces-*"]
21+
language = "kuery"
22+
license = "Elastic License v2"
23+
name = "Agent Spoofing - Multiple Hosts Using Same Agent"
24+
risk_score = 73
25+
rule_id = "493834ca-f861-414c-8602-150d5505b777"
26+
severity = "high"
27+
tags = ["Elastic", "Threat Detection", "Defense Evasion"]
28+
timestamp_override = "event.ingested"
29+
type = "threshold"
30+
31+
query = '''
32+
event.agent_id_status:*
33+
'''
34+
35+
36+
[[rule.threat]]
37+
framework = "MITRE ATT&CK"
38+
[[rule.threat.technique]]
39+
id = "T1036"
40+
name = "Masquerading"
41+
reference = "https://attack.mitre.org/techniques/T1036/"
42+
43+
44+
[rule.threat.tactic]
45+
id = "TA0005"
46+
name = "Defense Evasion"
47+
reference = "https://attack.mitre.org/tactics/TA0005/"
48+
49+
50+
[rule.threshold]
51+
field = ["agent.id"]
52+
value = 2
53+
54+
[[rule.threshold.cardinality]]
55+
field = "host.id"
56+
value = 2

‎tests/test_schemas.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,10 @@ def setUpClass(cls):
8686
cls.v712_threshold_rule = dict(copy.deepcopy(cls.v79_threshold_contents), threshold={
8787
'field': ['destination.bytes', 'process.args'],
8888
'value': 75,
89-
'cardinality': {
89+
'cardinality': [{
9090
'field': 'user.name',
9191
'value': 2
92-
}
92+
}]
9393
})
9494

9595
def test_query_downgrade(self):

0 commit comments

Comments
 (0)
Please sign in to comment.