Skip to content

Commit 04685ea

Browse files
georgschoellygeorgschoelly
committed
joe: (1) allow users to disable PE object import (2) set 'to_ids' to False
1 parent 840ada4 commit 04685ea

File tree

3 files changed

+91
-46
lines changed

3 files changed

+91
-46
lines changed

‎misp_modules/lib/joe_parser.py

Lines changed: 49 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,15 @@
5151

5252

5353
class JoeParser():
54-
def __init__(self):
54+
def __init__(self, config):
5555
self.misp_event = MISPEvent()
5656
self.references = defaultdict(list)
5757
self.attributes = defaultdict(lambda: defaultdict(set))
5858
self.process_references = {}
5959

60+
self.import_pe = config["import_pe"]
61+
self.create_mitre_attack = config["mitre_attack"]
62+
6063
def parse_data(self, data):
6164
self.data = data
6265
if self.analysis_type() == "file":
@@ -72,7 +75,9 @@ def parse_data(self, data):
7275

7376
if self.attributes:
7477
self.handle_attributes()
75-
self.parse_mitre_attack()
78+
79+
if self.create_mitre_attack:
80+
self.parse_mitre_attack()
7681

7782
def build_references(self):
7883
for misp_object in self.misp_event.objects:
@@ -97,12 +102,12 @@ def parse_dropped_files(self):
97102
file_object = MISPObject('file')
98103
for key, mapping in dropped_file_mapping.items():
99104
attribute_type, object_relation = mapping
100-
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key]})
105+
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key], 'to_ids': False})
101106
if droppedfile['@malicious'] == 'true':
102-
file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious'})
107+
file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious', 'to_ids': False})
103108
for h in droppedfile['value']:
104109
hash_type = dropped_hash_mapping[h['@algo']]
105-
file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']})
110+
file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$'], 'to_ids': False})
106111
self.misp_event.add_object(**file_object)
107112
self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({
108113
'referenced_uuid': file_object.uuid,
@@ -132,9 +137,12 @@ def parse_network_behavior(self):
132137
for object_relation, attribute in attributes.items():
133138
network_connection_object.add_attribute(object_relation, **attribute)
134139
network_connection_object.add_attribute('first-packet-seen',
135-
**{'type': 'datetime', 'value': min(tuple(min(timestamp) for timestamp in data.values()))})
140+
**{'type': 'datetime',
141+
'value': min(tuple(min(timestamp) for timestamp in data.values())),
142+
'to_ids': False})
136143
for protocol in data.keys():
137-
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
144+
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]),
145+
**{'type': 'text', 'value': protocol, 'to_ids': False})
138146
self.misp_event.add_object(**network_connection_object)
139147
self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid,
140148
relationship_type='initiates'))
@@ -143,8 +151,8 @@ def parse_network_behavior(self):
143151
network_connection_object = MISPObject('network-connection')
144152
for object_relation, attribute in attributes.items():
145153
network_connection_object.add_attribute(object_relation, **attribute)
146-
network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)})
147-
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
154+
network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps), 'to_ids': False})
155+
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol, 'to_ids': False})
148156
self.misp_event.add_object(**network_connection_object)
149157
self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid,
150158
relationship_type='initiates'))
@@ -154,7 +162,8 @@ def parse_screenshot(self):
154162
if screenshotdata:
155163
screenshotdata = screenshotdata['interesting']['$']
156164
attribute = {'type': 'attachment', 'value': 'screenshot.jpg',
157-
'data': screenshotdata, 'disable_correlation': True}
165+
'data': screenshotdata, 'disable_correlation': True,
166+
'to_ids': False}
158167
self.misp_event.add_attribute(**attribute)
159168

160169
def parse_system_behavior(self):
@@ -166,9 +175,9 @@ def parse_system_behavior(self):
166175
general = process['general']
167176
process_object = MISPObject('process')
168177
for feature, relation in process_object_fields.items():
169-
process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
178+
process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature], 'to_ids': False})
170179
start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
171-
process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
180+
process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time, 'to_ids': False})
172181
self.misp_event.add_object(**process_object)
173182
for field, to_call in process_activities.items():
174183
if process.get(field):
@@ -203,7 +212,7 @@ def parse_url_analysis(self):
203212
url_object = MISPObject("url")
204213
self.analysisinfo_uuid = url_object.uuid
205214

206-
url_object.add_attribute("url", generalinfo["target"]["url"])
215+
url_object.add_attribute("url", generalinfo["target"]["url"], to_ids=False)
207216
self.misp_event.add_object(**url_object)
208217

209218
def parse_fileinfo(self):
@@ -213,10 +222,10 @@ def parse_fileinfo(self):
213222
self.analysisinfo_uuid = file_object.uuid
214223

215224
for field in file_object_fields:
216-
file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]})
225+
file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field], 'to_ids': False})
217226
for field, mapping in file_object_mapping.items():
218227
attribute_type, object_relation = mapping
219-
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]})
228+
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field], 'to_ids': False})
220229
arch = self.data['generalinfo']['arch']
221230
if arch in arch_type_mapping:
222231
to_call = arch_type_mapping[arch]
@@ -234,9 +243,9 @@ def parse_apk(self, fileinfo, file_object):
234243
attribute_type = 'text'
235244
for comment, permissions in permission_lists.items():
236245
permission_object = MISPObject('android-permission')
237-
permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment))
246+
permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment, to_ids=False))
238247
for permission in permissions:
239-
permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission))
248+
permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission, to_ids=False))
240249
self.misp_event.add_object(**permission_object)
241250
self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid,
242251
relationship_type='grants'))
@@ -255,24 +264,24 @@ def parse_elf(self, fileinfo, file_object):
255264
if elf.get('type'):
256265
# Haven't seen anything but EXEC yet in the files I tested
257266
attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type']
258-
elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value))
267+
elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value, to_ids=False))
259268
for feature, relation in elf_object_mapping.items():
260269
if elf.get(feature):
261-
elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature]))
270+
elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature], to_ids=False))
262271
sections_number = len(fileinfo['sections']['section'])
263-
elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
272+
elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False})
264273
self.misp_event.add_object(**elf_object)
265274
for section in fileinfo['sections']['section']:
266275
section_object = MISPObject('elf-section')
267276
for feature in ('name', 'type'):
268277
if section.get(feature):
269-
section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature]))
278+
section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature], to_ids=False))
270279
if section.get('size'):
271-
section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16)))
280+
section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16), to_ids=False))
272281
for flag in section['flagsdesc']:
273282
try:
274283
attribute_value = elf_section_flags_mapping[flag]
275-
section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value))
284+
section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value, to_ids=False))
276285
except KeyError:
277286
print(f'Unknown elf section flag: {flag}')
278287
continue
@@ -281,6 +290,8 @@ def parse_elf(self, fileinfo, file_object):
281290
relationship_type=relationship))
282291

283292
def parse_pe(self, fileinfo, file_object):
293+
if not self.import_pe:
294+
return
284295
try:
285296
peinfo = fileinfo['pe']
286297
except KeyError:
@@ -292,27 +303,27 @@ def parse_pe(self, fileinfo, file_object):
292303
self.misp_event.add_object(**file_object)
293304
for field, mapping in pe_object_fields.items():
294305
attribute_type, object_relation = mapping
295-
pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]})
296-
pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)})
306+
pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field], 'to_ids': False})
307+
pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16), 'to_ids': False})
297308
program_name = fileinfo['filename']
298309
if peinfo['versions']:
299310
for feature in peinfo['versions']['version']:
300311
name = feature['name']
301312
if name == 'InternalName':
302313
program_name = feature['value']
303314
if name in pe_object_mapping:
304-
pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
315+
pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value'], 'to_ids': False})
305316
sections_number = len(peinfo['sections']['section'])
306-
pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
317+
pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False})
307318
signatureinfo = peinfo['signature']
308319
if signatureinfo['signed']:
309320
signerinfo_object = MISPObject('authenticode-signerinfo')
310321
pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
311322
self.misp_event.add_object(**pe_object)
312-
signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
323+
signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name, 'to_ids': False})
313324
for feature, mapping in signerinfo_object_mapping.items():
314325
attribute_type, object_relation = mapping
315-
signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
326+
signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature], 'to_ids': False})
316327
self.misp_event.add_object(**signerinfo_object)
317328
else:
318329
self.misp_event.add_object(**pe_object)
@@ -327,7 +338,7 @@ def parse_pe_section(self, section):
327338
for feature, mapping in pe_section_object_mapping.items():
328339
if section.get(feature):
329340
attribute_type, object_relation = mapping
330-
section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]})
341+
section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature], 'to_ids': False})
331342
return section_object
332343

333344
def parse_network_interactions(self):
@@ -339,21 +350,21 @@ def parse_network_interactions(self):
339350
for key, mapping in domain_object_mapping.items():
340351
attribute_type, object_relation = mapping
341352
domain_object.add_attribute(object_relation,
342-
**{'type': attribute_type, 'value': domain[key]})
353+
**{'type': attribute_type, 'value': domain[key], 'to_ids': False})
343354
self.misp_event.add_object(**domain_object)
344355
reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts')
345356
self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
346357
else:
347358
attribute = MISPAttribute()
348-
attribute.from_dict(**{'type': 'domain', 'value': domain['@name']})
359+
attribute.from_dict(**{'type': 'domain', 'value': domain['@name'], 'to_ids': False})
349360
self.misp_event.add_attribute(**attribute)
350361
reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts')
351362
self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
352363
ipinfo = self.data['ipinfo']
353364
if ipinfo:
354365
for ip in ipinfo['ip']:
355366
attribute = MISPAttribute()
356-
attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']})
367+
attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip'], 'to_ids': False})
357368
self.misp_event.add_attribute(**attribute)
358369
reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts')
359370
self.add_process_reference(ip['@targetid'], ip['@currentpath'], reference)
@@ -363,7 +374,7 @@ def parse_network_interactions(self):
363374
target_id = int(url['@targetid'])
364375
current_path = url['@currentpath']
365376
attribute = MISPAttribute()
366-
attribute_dict = {'type': 'url', 'value': url['@name']}
377+
attribute_dict = {'type': 'url', 'value': url['@name'], 'to_ids': False}
367378
if target_id != -1 and current_path != 'unknown':
368379
self.references[self.process_references[(target_id, current_path)]].append({
369380
'referenced_uuid': attribute.uuid,
@@ -384,8 +395,8 @@ def parse_registryactivities(self, process_uuid, registryactivities):
384395
registry_key = MISPObject('registry-key')
385396
for field, mapping in regkey_object_mapping.items():
386397
attribute_type, object_relation = mapping
387-
registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field]})
388-
registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper())})
398+
registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field], 'to_ids': False})
399+
registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper()), 'to_ids': False})
389400
self.misp_event.add_object(**registry_key)
390401
self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid,
391402
relationship_type=relationship))
@@ -398,7 +409,7 @@ def add_process_reference(self, target, currentpath, reference):
398409

399410
def create_attribute(self, attribute_type, attribute_value):
400411
attribute = MISPAttribute()
401-
attribute.from_dict(**{'type': attribute_type, 'value': attribute_value})
412+
attribute.from_dict(**{'type': attribute_type, 'value': attribute_value, 'to_ids': False})
402413
self.misp_event.add_attribute(**attribute)
403414
return attribute.uuid
404415

@@ -419,5 +430,5 @@ def prefetch_attributes_data(connection):
419430
attributes = {}
420431
for field, value in zip(network_behavior_fields, connection):
421432
attribute_type, object_relation = network_connection_object_mapping[field]
422-
attributes[object_relation] = {'type': attribute_type, 'value': value}
433+
attributes[object_relation] = {'type': attribute_type, 'value': value, 'to_ids': False}
423434
return attributes

0 commit comments

Comments
 (0)