Download cron/duo_logpull.py from ineso22/Mozdef: direct link, hf CLI and curl.
- Browser
- Download file 9.37 kB
-
https://huggingface.co/ineso22/Mozdef/resolve/main/cron/duo_logpull.py
- Command line
-
hf download hf://ineso22/Mozdef/cron/duo_logpull.py
-
curl -L -o duo_logpull.py https://huggingface.co/ineso22/Mozdef/resolve/main/cron/duo_logpull.py
9.37 kB
| #!/usr/bin/env python | |
| # This Source Code Form is subject to the terms of the Mozilla Public | |
| # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| # file, You can obtain one at https://mozilla.org/MPL/2.0/. | |
| # | |
| import sys | |
| import json | |
| import duo_client | |
| import mozdef_client as mozdef | |
| import pickle | |
| from configlib import getConfig, OptionParser | |
| from mozdef_util.utilities.toUTC import toUTC | |
| def normalize(details): | |
| # Normalizes fields to conform to http://mozdef.readthedocs.io/en/latest/usage.html#mandatory-fields | |
| # This is mainly used for common field names to put inside the details structure | |
| # There might be faster ways to do this | |
| normalized = {} | |
| for f in details: | |
| if f in ("ip", "ip_address", "client_ip"): | |
| normalized["sourceipaddress"] = details[f] | |
| continue | |
| if f in ("eventtype", "event_type"): | |
| normalized["eventtype"] = details[f] | |
| continue | |
| if f in ("host", "hostname"): | |
| normalized["hostname"] = details[f] | |
| continue | |
| if f == "result": | |
| if details[f].lower() == "success": | |
| normalized["success"] = True | |
| else: | |
| normalized["success"] = False | |
| normalized[f] = details[f] | |
| if "user" in normalized and type(normalized["user"]) is dict: | |
| if "name" in normalized["user"]: | |
| normalized["username"] = normalized["user"]["name"] | |
| if "key" in normalized["user"]: | |
| normalized["userkey"] = normalized["user"]["key"] | |
| del (normalized["user"]) | |
| return normalized | |
| def process_events(mozmsg, duo_events, etype, state): | |
| """ | |
| Data format of duo_events in api_version == 2 (str): | |
| duo_events.metadata = {u'total_objects': 49198, u'next_offset': [u'1547244648000', u'4da7180c-b1e5-47b4-9f4d-ee10dc3b5ac8']} | |
| duo_events.authlogs = [{...}, {...}, ...] | |
| authlogs entry = {u'access_device': {u'ip': u'a.b.c.d', u'location': {u'city': None, u'state': u'Anhui', u'country': | |
| u'China'}}, u'event_type': u'authentication', u'timestamp': 1547244800, u'factor': u'not_available', u'reason': | |
| u'deny_unenrolled_user', u'txid': u'68b33dd3-d341-46c6-a985-0640592fb7b0', u'application': {u'name': u'Integration | |
| Name Here', u'key': u'SOME KEY HERE'}, u'host': u'api-blah.duosecurity.com', u'result': u'denied', u'eventtype': u'authentication', u'auth_device': {u'ip': None, u'location': {u'city': None, u'state': None, u'country': None}, u'name': None}, u'user': {u'name': u'root', u'key': None}} | |
| """ | |
| # There are some key fields that we use as MozDef fields, those are set to "noconsume" | |
| # After processing these fields, we just pour everything into the "details" fields of Mozdef, except for the | |
| # noconsume fields. | |
| if etype == "administration": | |
| noconsume = ["timestamp", "host", "action"] | |
| elif etype == "telephony": | |
| noconsume = ["timestamp", "host", "context"] | |
| elif etype == "authentication": | |
| noconsume = ["timestamp", "host", "event_type"] | |
| else: | |
| return | |
| # Care for API v2 | |
| if isinstance(duo_events, dict) and "authlogs" in duo_events: | |
| offset = duo_events["metadata"]["next_offset"] | |
| if offset is not None: | |
| state["{}_offset".format(etype)] = offset | |
| duo_events = duo_events["authlogs"] | |
| api_version = 2 | |
| else: | |
| api_version = 1 | |
| for e in duo_events: | |
| details = {} | |
| # Timestamp format: http://mozdef.readthedocs.io/en/latest/usage.html#mandatory-fields | |
| # Duo logs come as a UTC timestamp | |
| if 'timestamp' in e: | |
| mozmsg.timestamp = toUTC(e['timestamp']).isoformat() | |
| # mozdef_client sets hostname to the host that the client runs on, | |
| # so we'll set the clientname in this cron to be the host we pulled from. | |
| if 'host' in e and not None: | |
| mozmsg.log["hostname"] = e["host"] | |
| if 'hostname' in e and not None: | |
| mozmsg.log["hostname"] = e["hostname"] | |
| for i in e: | |
| if i in noconsume: | |
| continue | |
| # Duo client doesn't translate inner dicts to dicts for some reason - its just a string, so we have to process and parse it | |
| if e[i] is not None and type(e[i]) == str and e[i].startswith("{"): | |
| j = json.loads(e[i]) | |
| for x in j: | |
| details[x] = j[x] | |
| continue | |
| details[i] = e[i] | |
| mozmsg.set_category(etype) | |
| localdetails = normalize(details) | |
| if "access_device" in localdetails: | |
| if "ip" in localdetails["access_device"]: | |
| localdetails["sourceipaddress"] = localdetails["access_device"]["ip"] | |
| if "hostname" in localdetails["access_device"]: | |
| if localdetails["access_device"]["hostname"] is None: | |
| del localdetails["access_device"]["hostname"] | |
| mozmsg.details = localdetails | |
| del(localdetails) | |
| mozmsg.hostname = options.URL | |
| if etype == "administration": | |
| if 'error' in e: | |
| mozmsg.summary = ( | |
| e["action"] + " because of " + e["error"] + " by " + e["username"] | |
| ) | |
| else: | |
| mozmsg.summary = ( | |
| e["action"] + " by " + e["username"] | |
| ) | |
| elif etype == "telephony": | |
| mozmsg.summary = e["context"] | |
| elif etype == "authentication": | |
| if api_version == 1: | |
| mozmsg.summary = ( | |
| e["eventtype"] + " " + e["result"] + " for " + e["username"] | |
| ) | |
| else: | |
| if 'reason' in e and e['reason'] is not None: | |
| mozmsg.summary = ( | |
| e["eventtype"] + " " + e["result"] + " for " + e["user"]["name"] + " due to " + e["reason"] | |
| ) | |
| else: | |
| mozmsg.summary = ( | |
| e["eventtype"] + " " + e["result"] + " for " + e["user"]["name"] | |
| ) | |
| mozmsg.send() | |
| # last event timestamp record is stored and returned so that we can save our last position in the log. | |
| try: | |
| state[etype] = e["timestamp"] | |
| except UnboundLocalError: | |
| # duo_events was empty, no new event | |
| pass | |
| return state | |
| def main(): | |
| try: | |
| state = pickle.load(open(options.statepath, "rb")) | |
| except IOError: | |
| # Oh, you're new. | |
| # Note API v2 expect full, correct and within range timestamps in millisec so we start recently | |
| # API v1 uses normal timestamps in seconds instead | |
| state = { | |
| "administration": 0, | |
| "administration_offset": None, | |
| "authentication": 1547000000000, | |
| "authentication_offset": None, | |
| "telephony": 0, | |
| "telephony_offset": None, | |
| } | |
| # Convert v1 (sec) timestamp to v2 (ms)... | |
| if state["authentication"] < 1547000000000: | |
| state["authentication"] = int(str(state["authentication"]) + "000") | |
| duo = duo_client.Admin(ikey=options.IKEY, skey=options.SKEY, host=options.URL) | |
| mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL) | |
| mozmsg.tags = ["duosecurity"] | |
| if options.update_tags != "": | |
| mozmsg.tags.append(options.update_tags) | |
| mozmsg.set_category("authentication") | |
| mozmsg.source = "DuoSecurityAPI" | |
| if options.DEBUG: | |
| mozmsg.debug = options.DEBUG | |
| mozmsg.set_send_to_syslog(True, only_syslog=True) | |
| # This will process events for all 3 log types and send them to MozDef. the state stores the last position in the | |
| # log when this script was last called. | |
| # NOTE: If administration and telephone logs support a "v2" API in the future it will most likely need to have the | |
| # same code with `next_offset` as authentication uses. | |
| state = process_events( | |
| mozmsg, | |
| duo.get_administrator_log(mintime=state["administration"] + 1), | |
| "administration", | |
| state, | |
| ) | |
| state = process_events( | |
| mozmsg, | |
| duo.get_authentication_log( | |
| api_version=2, | |
| limit="1000", | |
| sort="ts:asc", | |
| mintime=state["authentication"] + 1, | |
| next_offset=state["authentication_offset"], | |
| ), | |
| "authentication", | |
| state, | |
| ) | |
| state = process_events( | |
| mozmsg, | |
| duo.get_telephony_log(mintime=state["telephony"] + 1), | |
| "telephony", | |
| state, | |
| ) | |
| pickle.dump(state, open(options.statepath, "wb")) | |
| def initConfig(): | |
| options.IKEY = getConfig("IKEY", "", options.configfile) | |
| options.SKEY = getConfig("SKEY", "", options.configfile) | |
| options.URL = getConfig("URL", "", options.configfile) | |
| options.MOZDEF_URL = getConfig("MOZDEF_URL", "", options.configfile) | |
| options.DEBUG = getConfig("DEBUG", True, options.configfile) | |
| options.statepath = getConfig("statepath", "", options.configfile) | |
| options.update_tags = getConfig("addtag", "", options.configfile) | |
| if __name__ == "__main__": | |
| parser = OptionParser() | |
| defaultconfigfile = sys.argv[0].replace(".py", ".conf") | |
| parser.add_option( | |
| "-c", | |
| dest="configfile", | |
| default=defaultconfigfile, | |
| help="configuration file to use", | |
| ) | |
| (options, args) = parser.parse_args() | |
| initConfig() | |
| main() | |