diff --git a/detection_rules/rule.py b/detection_rules/rule.py
index d26624e9b04..99df1d95480 100644
--- a/detection_rules/rule.py
+++ b/detection_rules/rule.py
@@ -183,7 +183,7 @@ class BaseRuleData(MarshmallowDataclassMixin):
def save_schema(cls):
"""Save the schema as a jsonschema."""
fields: List[dataclasses.Field] = dataclasses.fields(cls)
- type_field = next(field for field in fields if field.name == "type")
+ type_field = next(f for f in fields if f.name == "type")
rule_type = typing.get_args(type_field.type)[0] if cls != BaseRuleData else "base"
schema = cls.jsonschema()
version_dir = SCHEMA_DIR / "master"
@@ -256,9 +256,9 @@ class ThresholdCardinality:
field: str
value: definitions.ThresholdValue
- field: List[definitions.NonEmptyStr]
+ field: definitions.CardinalityFields
value: definitions.ThresholdValue
- cardinality: Optional[ThresholdCardinality]
+ cardinality: Optional[List[ThresholdCardinality]]
type: Literal["threshold"]
threshold: ThresholdMapping
diff --git a/detection_rules/rule_formatter.py b/detection_rules/rule_formatter.py
index ff43dab9791..bb4cb485d21 100644
--- a/detection_rules/rule_formatter.py
+++ b/detection_rules/rule_formatter.py
@@ -206,5 +206,5 @@ def _do_write(_data, _contents):
_do_write(data, _contents)
finally:
- if needs_close:
+ if needs_close and hasattr(outfile, "close"):
outfile.close()
diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py
index f20eb1b30ca..2787dcf199f 100644
--- a/detection_rules/schemas/__init__.py
+++ b/detection_rules/schemas/__init__.py
@@ -123,7 +123,7 @@ def downgrade_threshold_to_7_11(version: Version, api_contents: dict) -> dict:
if len(threshold_field) > 1:
raise ValueError('Cannot downgrade a threshold rule that has multiple threshold fields defined')
- if threshold.get('cardinality', {}).get('field') or threshold.get('cardinality', {}).get('value'):
+ if threshold.get('cardinality'):
raise ValueError('Cannot downgrade a threshold rule that has a defined cardinality')
api_contents = api_contents.copy()
diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
index 9d4bcb3c2d9..1fb005e2915 100644
--- a/detection_rules/schemas/definitions.py
+++ b/detection_rules/schemas/definitions.py
@@ -5,7 +5,7 @@
"""Custom shared definitions for schemas."""
-from typing import Literal, Final
+from typing import List, Literal, Final
from marshmallow import validate
from marshmallow_dataclass import NewType
@@ -44,19 +44,21 @@
}
+NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1))
+
BranchVer = NewType('BranchVer', str, validate=validate.Regexp(BRANCH_PATTERN))
+CardinalityFields = NewType('CardinalityFields', List[NonEmptyStr], validate=validate.Length(min=0, max=3))
CodeString = NewType("CodeString", str)
ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN))
Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
FilterLanguages = Literal["kuery", "lucene"]
Interval = NewType('Interval', str, validate=validate.Regexp(INTERVAL_PATTERN))
-PositiveInteger = NewType('PositiveInteger', int, validate=validate.Range(min=1))
Markdown = NewType("MarkdownField", CodeString)
Maturity = Literal['development', 'experimental', 'beta', 'production', 'deprecated']
MaxSignals = NewType("MaxSignals", int, validate=validate.Range(min=1))
-NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1))
Operator = Literal['equals']
OSType = Literal['windows', 'linux', 'macos']
+PositiveInteger = NewType('PositiveInteger', int, validate=validate.Range(min=1))
RiskScore = NewType("MaxSignals", int, validate=validate.Range(min=1, max=100))
RuleType = Literal['query', 'saved_query', 'machine_learning', 'eql', 'threshold', 'threat_match']
SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN))
diff --git a/etc/api_schemas/7.12/7.12.threshold.json b/etc/api_schemas/7.12/7.12.threshold.json
index d426ff86832..f5c606b1643 100644
--- a/etc/api_schemas/7.12/7.12.threshold.json
+++ b/etc/api_schemas/7.12/7.12.threshold.json
@@ -919,35 +919,46 @@
"additionalProperties": false,
"properties": {
"cardinality": {
- "additionalProperties": false,
- "properties": {
- "field": {
- "type": "string"
+ "items": {
+ "additionalProperties": false,
+ "properties": {
+ "field": {
+ "type": "string"
+ },
+ "value": {
+ "description": "ThresholdValue",
+ "format": "integer",
+ "minimum": 1,
+ "type": "number"
+ }
},
- "value": {
- "minimum": 1,
- "type": "integer"
- }
+ "required": [
+ "field",
+ "value"
+ ],
+ "type": "object"
},
- "required": [
- "field",
- "value"
- ],
- "type": "object"
+ "type": "array"
},
"field": {
+ "description": "CardinalityFields",
"items": {
- "default": "",
+ "description": "NonEmptyStr",
+ "minLength": 1,
"type": "string"
},
+ "maxItems": 3,
"type": "array"
},
"value": {
+ "description": "ThresholdValue",
+ "format": "integer",
"minimum": 1,
- "type": "integer"
+ "type": "number"
}
},
"required": [
+ "field",
"value"
],
"type": "object"
diff --git a/etc/api_schemas/7.13/7.13.threshold.json b/etc/api_schemas/7.13/7.13.threshold.json
index 724d46ccee9..aeb982b1912 100644
--- a/etc/api_schemas/7.13/7.13.threshold.json
+++ b/etc/api_schemas/7.13/7.13.threshold.json
@@ -112,6 +112,9 @@
"type": "string"
},
"operator": {
+ "enum": [
+ "equals"
+ ],
"type": "string"
},
"value": {
@@ -151,6 +154,9 @@
"type": "string"
},
"operator": {
+ "enum": [
+ "equals"
+ ],
"type": "string"
},
"severity": {
@@ -178,6 +184,9 @@
"additionalProperties": false,
"properties": {
"framework": {
+ "enum": [
+ "MITRE ATT&CK"
+ ],
"type": "string"
},
"tactic": {
@@ -265,30 +274,35 @@
"additionalProperties": false,
"properties": {
"cardinality": {
- "additionalProperties": false,
- "properties": {
- "field": {
- "type": "string"
+ "items": {
+ "additionalProperties": false,
+ "properties": {
+ "field": {
+ "type": "string"
+ },
+ "value": {
+ "description": "ThresholdValue",
+ "format": "integer",
+ "minimum": 1,
+ "type": "number"
+ }
},
- "value": {
- "description": "ThresholdValue",
- "format": "integer",
- "minimum": 1,
- "type": "number"
- }
+ "required": [
+ "field",
+ "value"
+ ],
+ "type": "object"
},
- "required": [
- "field",
- "value"
- ],
- "type": "object"
+ "type": "array"
},
"field": {
+ "description": "CardinalityFields",
"items": {
"description": "NonEmptyStr",
"minLength": 1,
"type": "string"
},
+ "maxItems": 3,
"type": "array"
},
"value": {
@@ -336,6 +350,9 @@
"type": "string"
},
"type": {
+ "enum": [
+ "threshold"
+ ],
"type": "string"
}
},
diff --git a/etc/api_schemas/7.14/7.14.threshold.json b/etc/api_schemas/7.14/7.14.threshold.json
index e0715120611..aeb982b1912 100644
--- a/etc/api_schemas/7.14/7.14.threshold.json
+++ b/etc/api_schemas/7.14/7.14.threshold.json
@@ -274,30 +274,35 @@
"additionalProperties": false,
"properties": {
"cardinality": {
- "additionalProperties": false,
- "properties": {
- "field": {
- "type": "string"
+ "items": {
+ "additionalProperties": false,
+ "properties": {
+ "field": {
+ "type": "string"
+ },
+ "value": {
+ "description": "ThresholdValue",
+ "format": "integer",
+ "minimum": 1,
+ "type": "number"
+ }
},
- "value": {
- "description": "ThresholdValue",
- "format": "integer",
- "minimum": 1,
- "type": "number"
- }
+ "required": [
+ "field",
+ "value"
+ ],
+ "type": "object"
},
- "required": [
- "field",
- "value"
- ],
- "type": "object"
+ "type": "array"
},
"field": {
+ "description": "CardinalityFields",
"items": {
"description": "NonEmptyStr",
"minLength": 1,
"type": "string"
},
+ "maxItems": 3,
"type": "array"
},
"value": {
diff --git a/etc/api_schemas/master/master.threshold.json b/etc/api_schemas/master/master.threshold.json
index e0715120611..aeb982b1912 100644
--- a/etc/api_schemas/master/master.threshold.json
+++ b/etc/api_schemas/master/master.threshold.json
@@ -274,30 +274,35 @@
"additionalProperties": false,
"properties": {
"cardinality": {
- "additionalProperties": false,
- "properties": {
- "field": {
- "type": "string"
+ "items": {
+ "additionalProperties": false,
+ "properties": {
+ "field": {
+ "type": "string"
+ },
+ "value": {
+ "description": "ThresholdValue",
+ "format": "integer",
+ "minimum": 1,
+ "type": "number"
+ }
},
- "value": {
- "description": "ThresholdValue",
- "format": "integer",
- "minimum": 1,
- "type": "number"
- }
+ "required": [
+ "field",
+ "value"
+ ],
+ "type": "object"
},
- "required": [
- "field",
- "value"
- ],
- "type": "object"
+ "type": "array"
},
"field": {
+ "description": "CardinalityFields",
"items": {
"description": "NonEmptyStr",
"minLength": 1,
"type": "string"
},
+ "maxItems": 3,
"type": "array"
},
"value": {
diff --git a/etc/stack-schema-map.yaml b/etc/stack-schema-map.yaml
index e992786fa37..871ca3764a8 100644
--- a/etc/stack-schema-map.yaml
+++ b/etc/stack-schema-map.yaml
@@ -9,4 +9,4 @@
"7.14.0":
beats: "master" # TODO: 7.14.x
- ecs: "1.10.0"
+ ecs: "master" # TODO: master came out after 7.13.0 release
diff --git a/requirements.txt b/requirements.txt
index 9292b86002c..3c3e4e79952 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,7 +8,7 @@ PyYAML~=5.3
eql==0.9.9
elasticsearch~=7.9
XlsxWriter~=1.3.6
-marshmallow~=3.10.0
+marshmallow~=3.12.2
marshmallow-dataclass[union]~=8.4
# test deps
diff --git a/rules/cross-platform/defense_evasion_agent_spoofing_mismatched_id.toml b/rules/cross-platform/defense_evasion_agent_spoofing_mismatched_id.toml
new file mode 100644
index 00000000000..484819cbf38
--- /dev/null
+++ b/rules/cross-platform/defense_evasion_agent_spoofing_mismatched_id.toml
@@ -0,0 +1,48 @@
+[metadata]
+creation_date = "2021/07/14"
+maturity = "production"
+updated_date = "2021/07/14"
+min_stack_version = "7.14.0"
+
+[rule]
+author = ["Elastic"]
+description = """Detects events which have a mismatch on the expected event agent ID. The status "agent_id_mismatch"
+occurs when the expected agent ID associated with the API key does not match the actual agent ID in an event. This could
+indicate attempts to spoof events in order to masquerade actual activity to evade detection.
+"""
+false_positives = [
+ """
+ This is meant to run only on datasources using agents v7.14+ since versions prior to that will be missing the
+ necessary field, resulting in false positives.
+ """,
+]
+from = "now-9m"
+index = ["logs-*", "metrics-*", "traces-*"]
+language = "kuery"
+license = "Elastic License v2"
+name = "Agent Spoofing - Mismatched Agent ID"
+risk_score = 73
+rule_id = "3115bd2c-0baa-4df0-80ea-45e474b5ef93"
+severity = "high"
+tags = ["Elastic", "Threat Detection", "Defense Evasion"]
+timestamp_override = "event.ingested"
+type = "query"
+
+query = '''
+event.agent_id_status:agent_id_mismatch
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1036"
+name = "Masquerading"
+reference = "https://attack.mitre.org/techniques/T1036/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"
+
diff --git a/rules/cross-platform/defense_evasion_agent_spoofing_multiple_hosts.toml b/rules/cross-platform/defense_evasion_agent_spoofing_multiple_hosts.toml
new file mode 100644
index 00000000000..5caecf3a49b
--- /dev/null
+++ b/rules/cross-platform/defense_evasion_agent_spoofing_multiple_hosts.toml
@@ -0,0 +1,56 @@
+[metadata]
+creation_date = "2021/07/14"
+maturity = "production"
+updated_date = "2021/07/14"
+min_stack_version = "7.14.0"
+
+[rule]
+author = ["Elastic"]
+description = """Detects when multiple hosts are using the same agent ID. This could occur in the event of an agent
+being taken over and used to inject illegitimate documents into an instance as an attempt to spoof events in order to
+masquerade actual activity to evade detection.
+"""
+false_positives = [
+ """
+ This is meant to run only on datasources using agents v7.14+ since versions prior to that will be missing the
+ necessary field, resulting in false positives.
+ """,
+]
+from = "now-9m"
+index = ["logs-*", "metrics-*", "traces-*"]
+language = "kuery"
+license = "Elastic License v2"
+name = "Agent Spoofing - Multiple Hosts Using Same Agent"
+risk_score = 73
+rule_id = "493834ca-f861-414c-8602-150d5505b777"
+severity = "high"
+tags = ["Elastic", "Threat Detection", "Defense Evasion"]
+timestamp_override = "event.ingested"
+type = "threshold"
+
+query = '''
+event.agent_id_status:*
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1036"
+name = "Masquerading"
+reference = "https://attack.mitre.org/techniques/T1036/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"
+
+
+[rule.threshold]
+field = ["agent.id"]
+value = 2
+
+[[rule.threshold.cardinality]]
+field = "host.id"
+value = 2
diff --git a/tests/test_schemas.py b/tests/test_schemas.py
index e78c8326535..68ed2a3d1d0 100644
--- a/tests/test_schemas.py
+++ b/tests/test_schemas.py
@@ -86,10 +86,10 @@ def setUpClass(cls):
cls.v712_threshold_rule = dict(copy.deepcopy(cls.v79_threshold_contents), threshold={
'field': ['destination.bytes', 'process.args'],
'value': 75,
- 'cardinality': {
+ 'cardinality': [{
'field': 'user.name',
'value': 2
- }
+ }]
})
def test_query_downgrade(self):