From b821df118bec4bb66c6fdbdfab9bd11f3f529901 Mon Sep 17 00:00:00 2001 From: Jerome Leonard Date: Tue, 29 Aug 2017 12:01:10 +0200 Subject: [PATCH 1/4] DigitalShadows gets incidents and intel-incidents and creates TheHive alerts --- DigitalShadows/api.py | 79 ++++++++++++-- ds2markdown.py | 47 +++++---- ds2th.py | 238 ++++++++++++++++++++++++++---------------- requirements.txt | 1 + test.py | 173 ++++++++++++++++++++++++++++++ 5 files changed, 418 insertions(+), 120 deletions(-) create mode 100755 test.py diff --git a/DigitalShadows/api.py b/DigitalShadows/api.py index f3c5bbf..4b6fa41 100644 --- a/DigitalShadows/api.py +++ b/DigitalShadows/api.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, - print_function, unicode_literals) -import requests +import requests +import json class DigitalShadowsApi(): """ @@ -21,8 +20,8 @@ def __init__(self, config): self.proxies = config['proxies'] self.verify = config['verify'] self.headers = { - 'Content-Type': 'application/vnd.polaris-v22+json', - 'Accept': 'application/vnd.polaris-v22+json' + 'Content-Type': 'application/vnd.polaris-v28+json', + 'Accept': 'application/vnd.polaris-v28+json' } self.session = requests.Session() self.auth = requests.auth.HTTPBasicAuth(username=self.key, @@ -33,7 +32,7 @@ def getIncidents(self, id, fulltext='false'): headers = self.headers try: return self.session.get(req, headers=headers, auth=self.auth, - proxies=self.proxies, verify=False) + proxies=self.proxies, verify=self.verify) except requests.exceptions.RequestException as e: sys.exit("Error: {}".format(e)) @@ -45,3 +44,69 @@ def getIntelIncidents(self, id, fulltext='false'): proxies=self.proxies, verify=self.verify) except requests.exceptions.RequestException as e: sys.exit("Error: {}".format(e)) + + def find_incident(self, since, property='occurred', direction='DESCENDING', detailed='true', fulltext='false'): + req = self.url + '/api/incidents/find' + headers = self.headers + payload = {'since': since , 'sort.property': property, 'sort.direction':direction, 'detailed': detailed, 'fulltext':fulltext} + try: + return self.session.get(req, headers=headers, auth=self.auth, proxies=self.proxies, params=payload, verify=self.verify) + except requests.exceptions.RequestException as e: + sys.exit("Error: {}".format(e)) + + def find_intel_incident(self, since, property='verified', direction='ASCENDING'): + req = self.url + '/api/intel-incidents/find' + headers = self.headers + + payload = json.dumps({ + "filter": { + "severities": [], + "tags": [], + "tagOperator": "AND", + "dateRange": since, + "dateRangeField": "occurred", + "types": [], + "withFeedback": True, + "withoutFeedback": True + }, + "sort": { + "property": property, + "direction": direction + }, + "pagination": { + "size": 50, + "offset": 0 + } + }) + + + try: + return self.session.post(req, headers=headers, auth=self.auth, proxies=self.proxies, data=payload, verify=self.verify) + except requests.exceptions.RequestException as e: + sys.exit("Error: {}".format(e)) + + def get_intel_incident_iocs(self, id): + req = "{}/api/intel-incidents/{}/iocs".format(self.url, id) + headers = self.headers + payload = { + "filter": {}, + "sort": { + "property": "value", + "direction": "ASCENDING" + } + } + try: + return self.session.post(req, headers=headers, auth=self.auth, proxies=self.proxies, + data=json.dumps(payload), verify=self.verify) + except requests.exceptions.RequestException as e: + sys.exit("Error: {}".format(e)) + + + def get_intel_incident_thumbnail(self, id): + req = "{}/api/thumbnails/{}".format(self.url, id) + headers = self.headers + try: + return self.session.get(req, headers=headers, auth=self.auth, proxies=self.proxies, + verify=self.verify) + except requests.exceptions.RequestException as e: + sys.exit("Error: {}".format(e)) diff --git a/ds2markdown.py b/ds2markdown.py index cde5be9..ce5e245 100644 --- a/ds2markdown.py +++ b/ds2markdown.py @@ -1,16 +1,15 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import print_function -from __future__ import unicode_literals + import json class ds2markdown(): - def __init__(self, content): + def __init__(self, content, thumbnail): - self.source ="" - self.taskLog = "{0} {1} {2} {3} {4}".format( + self.source = "" + self.thdescription = "{0} {1} {2} {3} {4} {5} {6}".format( "**Scope:**: {0}\n\n**Type:** {1}\n\n**Occurred:** {2}\n\n**Verified:** {3}\n\n**Modified:** {4}\n\n**Publiched:** {5}\n\n**Identifier:** {6}\n\n**Tags:** {7}\n\n".format( content.get('scope',"None"), content.get('type',"None"), @@ -20,29 +19,31 @@ def __init__(self, content): content.get('published',"None"), str(content.get('id',"None")), self.tags(content) - ),"----\n\n#### Description #### \n\n{}\n\n".format(content.get('description')), + ),"----\n\n#### Summary #### \n\n{}\n\n".format(content.get('summary')), + "----\n\n#### Description #### \n\n{}\n\n".format(content.get('description')), "{}\n\n".format(self.impactDescription(content)), "{}\n\n".format(self.mitigation(content)), - "{}\n\n".format(self.entitySummary(content)) + "{}\n\n".format(self.entitySummary(content, thumbnail)), + "{}\n\n".format(self.lci(content)) ) - def entitySummary(self, content): + def entitySummary(self, content, thumbnail): source = "" if 'entitySummary' in content: c = content.get('entitySummary',"None") - source += self.Summary(c) + source += self.information(c, thumbnail) if 'summaryText' in c: summaryText = c.get('summaryText',"None") - source += "#### Source data #### \n\n" + \ + source += "\n\n----\n\n#### Source data #### \n\n" + \ "```\n{}\n```\n\n".format(summaryText) if 'IpAddressEntitySummary' in content: c = content.get('IpAddressEntity',"None") - source = self.Summary(c) + source = self.information(c, thumbnail) if 'IpAddressDetails' in c: details = c.get('IpAddressDetails',"None") @@ -81,7 +82,7 @@ def entitySummary(self, content): if 'MessageEntitySummary' in content: c = content['MessageEntitySummary'] - source += self.Summary(c) + source += self.information(c, thumbnail) if 'conversationFragment' in c: conv = c.get('conversationFragment') @@ -103,21 +104,23 @@ def entitySummary(self, content): return source - def Summary(self, content): + def information(self, content, thumbnail): source = "" source += "----\n\n" + \ "#### Source Information #### \n\n" + \ - "**Source:** {0}\n\n**Domain:** {1}\n\n**Date:** {2}\n\n**Type:** {3}\n\n".format( + "**Source:** {0}\n\n**Domain:** {1}\n\n**Date:** {2}\n\n**Type:** {3}\n\n**Thumbnail**: ![thumbnail][thumb]\n\n[thumb]: {4}\n\n".format( content.get('source',"None"), content.get('domain',"None"), content.get('sourceDate',"None"), - content.get('type',"None") + content.get('type',"None"), + thumbnail.get('thumbnail', "None") + ) if 'dataBreach' in content: - dataBreach = content.get('entitySummary').get('dataBreach') + dataBreach = content.get('dataBreach') source += "**Databreach target** \n\n" + \ - "**Title: {0}\n\n**Target domain:** {1}\n\n**Published:** {2}\n\n**Occured:** {3}\n\n**Modified:** {4}\n\n**Id:** {5}\n\n".format( + "**Title:** {0}\n\n**Target domain:** {1}\n\n**Published:** {2}\n\n**Occured:** {3}\n\n**Modified:** {4}\n\n**Id:** {5}\n\n".format( dataBreach.get('title',"None"), dataBreach.get('domainName',"None"), dataBreach.get('published',"None"), @@ -136,7 +139,7 @@ def Summary(self, content): def impactDescription(self, content): impact = "" if "impactDescription" in content: - impact = "\n\n#### Impact Description #### \n\n{}" .format( + impact = "----\n\n#### Impact Description #### \n\n{}" .format( content.get('impactDescription', "None") ) @@ -145,17 +148,17 @@ def impactDescription(self, content): def mitigation(self, content): mitigation = "" if "mitigation" in content: - mitigation = "\n\n#### Mitigation #### \n\n{}".format(content.get('mitigation', "None")) + mitigation = "----\n\n#### Mitigation #### \n\n{}".format(content.get('mitigation', "None")) return mitigation def lci(self, content): - if content["linkedContentIncidents"] not in []: - linkedContentIncidents = "" + linkedContentIncidents = "----\n\n#### Linked incidents #### \n\n" + if content.get("linkedContentIncidents"): for lci in content["linkedContentIncidents"]: linkedContentIncidents += "- {} \n\n".format(lci) else: - linkedContentIncidents = "None" + linkedContentIncidents += "None" return linkedContentIncidents diff --git a/ds2th.py b/ds2th.py index e03b47d..272f22d 100755 --- a/ds2th.py +++ b/ds2th.py @@ -1,24 +1,44 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import print_function -from __future__ import unicode_literals import sys import getopt -import json import getpass -import re +import datetime +from io import BytesIO +import base64 from DigitalShadows.api import DigitalShadowsApi from thehive4py.api import TheHiveApi -from thehive4py.models import Case,CaseTask,CaseTaskLog +from thehive4py.models import Alert, AlertArtifact from config import DigitalShadows, TheHive from ds2markdown import ds2markdown -def thSeverity(sev): +def add_tags(tags, content): + + """ + add tag to tags + + :param tags is list + :param content is list + """ + t = tags + for newtag in content: + t.append("DS:{}".format(newtag)) + return t + +def th_alert_tags(incident): + tags = [] + add_tags(tags, ["id={}".format(incident.get('id')), "type={}".format(incident.get('type'))]) + for t in incident.get('tags'): + add_tags(tags, ["{}={}".format(t.get('type'),t.get('name'))]) + + return tags + +def th_severity(sev): """ convert DigitalShadows severity in TH severity @@ -35,95 +55,161 @@ def thSeverity(sev): } return severities[sev] +def th_dataType(type): + """ + convert DigitalShadows IOC type to TH dataType + :param type: str + :return: str + """ + types = { + 'IP':'ip', + 'HOST': 'domain', + 'URL':'url', + 'SHA256':'hash', + 'SHA1':'hash', + 'MD5':'hash', + 'FILENAME':'filename', + 'FILEPATH':'filename', + 'EMAIL': 'mail' + } + if type in types: + return types[type] + else: + return "other" -def convertDs2ThCase(content): - +def add_alert_artefact(artefacts, dataType, data, tags, tlp): """ - convert Digital Shadows incident in a TheHive Case - :content dict object + :param artefacts: array + :param dataType: string + :param data: string + :param tags: array + :param tlp: int + :return: array """ - tasks = [] - tags = ['src:DigitalShadows'] - for tag in content['tags']: - tags.append('DS:'+tag['type']+'='+tag['name']) + return artefacts.append(AlertArtifact(tags=tags, + dataType=dataType, + data=data, + message="From DigitalShadows", + tlp=tlp) + ) - if ('summary' in content) and (len(content['summary']) > 1): - description = content.get('summary') - else: - description = content.get('description', {"-"}) - case = Case( - title="[DigitalShadows] #{} ".format(content['id']) + content['title'], - tlp=2, - severity=thSeverity(content['severity']), - flag=False, - tags=tags, - description = description) - return case +def build_observables(observables): + artefacts = [] + if observables.get('total', 0) > 0: + + for ioc in observables.get('content'): + a = AlertArtifact( + data=ioc.get('value'), + dataType=th_dataType(ioc.get('type')), + message="Observable from DigitalShadows. Source: {}".format(ioc.get('source')), + tlp=2, + tags=["src:DigitalShadows"] + ) + artefacts.append(a) + return artefacts -def caseAddTask(thapi, caseId, content): + +def build_alert(incident, observables, thumbnail): """ - Add task in existing case with its log - Return the task "Imported from DigitalShadows" in the TheHive + Convert DigitalShadows alert into a TheHive Alert - : caseId Id of the case created by the import program - : content DigitalShadows response.content (JSON) + :param incident: dict + :param observables: dict + :param thumbnail: str + :return: Alert object """ - task = CaseTask( - title = "Incident imported from DigitalShadows", - description = "Incident from DigitalShadows" - ) - m = ds2markdown(content).taskLog - log = CaseTaskLog(message = m) - thresponse = thapi.create_case_task(caseId, task) - r = thresponse.json() - thresponse = thapi.create_task_log(r['id'], log) + return Alert(title="{}".format(incident.get('title')), + tlp=2, + severity=th_severity(incident.get('severity')), + description=ds2markdown(incident, thumbnail).thdescription, + type=incident.get('type'), + tags=th_alert_tags(incident), + caseTemplate=TheHive['template'], + source="DigitalShadows", + sourceRef=str(incident.get('id')), + artifacts=build_observables(observables) + ) + +def get_incidents(dsapi, thapi, since): + s = (datetime.datetime.now() - datetime.timedelta(minutes=int(since))).isoformat() + 'Z' + response = DigitalShadowsApi.find_incident(dsapi, s).json() + + + for i in response.get('content'): + alert = build_alert(i, {}, {"thumbnail":""}) + thapi.create_alert(alert) -def import2th(thapi, response): +def get_intel_incidents(dsapi, thapi, since): """ - Convert DigitalShadows response and import it in TheHive - Call convertDs2ThCase - Call CaseAddTask - Return the case fully created in TheHive - :response dict Response from DigitalShadows + :param dsapi: request to DigitalShadows + :param thapi: reauest to TheHive + :param since: int, number of minutes, period of time + :return: """ + s = "{}/{}".format((datetime.datetime.now() - datetime.timedelta(minutes=int(since))).isoformat(), + datetime.datetime.now().isoformat()) + response = DigitalShadowsApi.find_intel_incident(dsapi, s).json() + + for i in response.get('content'): + iocs = DigitalShadowsApi.get_intel_incident_iocs(dsapi, i.get('id')).json() + + if i.get('entitySummary') and i.get('entitySummary').get('screenshotThumbnailId'): + # i.get('entitySummary') + # i.get('entitySummary').get('screenshotThumbnailId') + thumbnail = get_thumbnails(dsapi, i.get('entitySummary').get('screenshotThumbnailId')) + else: + thumbnail = {'thumbnail':''} + alert = build_alert(i, iocs, thumbnail) - case = convertDs2ThCase(response) - thresponse = thapi.create_case(case) - r = thresponse.json() - caseAddTask(thapi, r['id'], response) + thapi.create_alert(alert) +def get_thumbnails(dsapi, thumbnail_id): + """ + Get Intel Incident screenshot thumbnail + :param dsapi: + :param thumbnail_id: + :return: dict {base64:} + """ + response = DigitalShadowsApi.get_intel_incident_thumbnail(dsapi,thumbnail_id) + if response.status_code == 200: + with BytesIO(response.content) as bytes: + encoded = base64.b64encode(bytes.read()) + b64_thumbnail = encoded.decode() + + return {"thumbnail":"data:{};base64,{}".format(response.headers['Content-Type'], b64_thumbnail)} + else: + return {thumbnail: ""} def run(argv): """ - Download Digital SHadows incident and create a new Case in TheHive + Download DigitalShadows incident and create a new Case in TheHive :argv incident number """ # get options - incidentId = '' try: - opts, args = getopt.getopt(argv, 'hi:',["incident="]) + opts, args = getopt.getopt(argv, 'ht:',["time="]) except getopt.GetoptError: - print(__file__ + " -i ") + print(__file__ + " -t