Downloadable data
+Reputation score
- List of all IPs in NERD database with their reputation scores:
{% if file_sizes['ip_rep.csv'] is number %}ip_rep.csv ({{file_sizes['ip_rep.csv']|filesizeformat}}){% else %}ERROR File not found{% endif %}
@@ -18,6 +19,35 @@
Downloadable data
Threat categorization
+-
+
-
+ List of all IPs in NERD with their categories:
+
-
+
- + Line format: + {% if file_sizes['ip_category.csv'] is number %}ip_category.csv ({{file_sizes['ip_category.csv']|filesizeformat}}){% else %}ERROR File not found{% endif %} + +
- + Table format: + {% if file_sizes['ip_category_table.csv'] is number %}ip_category_table.csv ({{file_sizes['ip_category_table.csv']|filesizeformat}}){% else %}ERROR File not found{% endif %} + +
+ -
+ Collection of blacklists for each category (IPs with confidence greater than 0.5):
+
-
+ {% for bl_file in bl_files %}
+
- + {% if file_sizes[bl_file] is number %}{{bl_file}} ({{file_sizes[bl_file]|filesizeformat}}){% else %}ERROR File not found{% endif %} + + {% endfor %} +
+
We recommend downloading it few minutes after a whole hour (i.e. between xx:01 and xx:05). +
All files are updated once per hour.
+ {% endblock %} diff --git a/NERDweb/templates/ip.html b/NERDweb/templates/ip.html index a2bcb602..833b2ae8 100644 --- a/NERDweb/templates/ip.html +++ b/NERDweb/templates/ip.html @@ -155,6 +155,27 @@IP address
{% set dbl = ipinfo.pop('dbl') %} {% endif %} +{# Threat category summary #} +Threat categories
+
+
+
+
+| TL | Role | Category | Details |
|---|---|---|---|
| {{ (tag.conf*100)|round|int }} | +{{ tag.role|safe }} | +{{ tag.cat|safe }} | +{% for subcat in tag.subcats %}{{ subcat|safe }} {% endfor %} |
+
| No threat category tags assigned | |||
+ {# Warden events #} {% if ipinfo.events %}
IP address
{% endif %} {# MISP events #} +{# TODO: filter "showable" events in backend! #} {% if ipinfo.misp_events %} {% set misp_events = namespace(showable=0) -%} {% for misp_event in ipinfo.misp_events -%} @@ -187,10 +209,10 @@IP address
{% set misp_events.showable = misp_events.showable + 1 -%} {% endif -%} {% endfor -%} - {% if misp_events.showable %} + {% if misp_events.showable > 0 %}[{{ misp_event.event_id }}] {{ misp_event.pop('date', 'no date') }} | {{ misp_event.pop('info', 'no info') }}
diff --git a/NERDweb/templates/ips.html b/NERDweb/templates/ips.html
index f96de341..76d08474 100644
--- a/NERDweb/templates/ips.html
+++ b/NERDweb/templates/ips.html
@@ -154,7 +154,62 @@
| Device type | -->Rep.(?) | {% if ac('fmp') %}FMP | {% endif %} +Threat category | Other properties | Time added | Last activity | @@ -311,6 +367,23 @@
+
+ {% if ip._threat_category_data_for_tags %}
+ {% for tag in ip._threat_category_data_for_tags %}
+
+
|
{% if ip.bl -%}
{% set bl_cnt = ip.bl|selectattr("v")|list|length -%}
@@ -337,7 +410,8 @@ Search IP addresses by ...{% if ip.open_ntp %}Open NTP{% endif %} {% if ip.open_snmp %}Open SNMP{% endif %} #} - + + {# {% if ip.tags %} {% for tag_id,tag_param in ip.tags.items() %} {% if tag_id in config_tags and "name" in config_tags[tag_id] %} @@ -373,7 +447,7 @@Search IP addresses by ...{% endif %} {% endfor %} {% endif %} - + #} {% if ip.shodan %} {% if ip.shodan.ports %} }}){kind=icon} {{ip.shodan.ports|join_max(5)}}{% endif %}
{% if ip.shodan.tags %}{{ip.shodan.tags|join(', ')}}{% endif %}
diff --git a/common/config.py b/common/config.py
index aff7d800..e79f007f 100644
--- a/common/config.py
+++ b/common/config.py
@@ -21,15 +21,20 @@ def hierarchical_get(self, key, default=NoDefault):
instead.
"""
d = self
+ full_key = key
try:
while '.' in key:
first_key, key = key.split('.', 1)
d = d[first_key]
- return d[key]
+ result = d[key]
+ if isinstance(result, dict):
+ return HierarchicalDict(result)
+ else:
+ return result
except (KeyError, TypeError):
pass # not found - continue below
if default is NoDefault:
- raise MissingConfigError("Mandatory configuration element is missing: " + key)
+ raise MissingConfigError("Mandatory configuration element is missing: " + full_key)
else:
return default
diff --git a/common/threat_categorization.py b/common/threat_categorization.py
new file mode 100644
index 00000000..f6a4778e
--- /dev/null
+++ b/common/threat_categorization.py
@@ -0,0 +1,195 @@
+import yaml
+import ast
+import re
+from datetime import datetime
+
+from .utils import parse_rfc_time
+
+
+class ClassifiableEvent:
+ def __getattr__(self, name):
+ """
+ Override __getattr__ so that no error is raised when a trigger tries to use non-existing attribute
+ :param name: Name of the attribute
+ :return: Value of the attribute (or None if it does not exist)
+ """
+ return self.__dict__[name] if name in self.__dict__ else None
+
+ def __str__(self):
+ """
+ Override __str__ for easier logging of assigned categories
+ :param name: Name of the attribute
+ :return: String representation of the object's attribute dictionary
+ """
+ return str(self.__dict__)
+
+ def __init__(self, module_name=None, *args):
+ """
+ Initialize the event (fill metadata from source module)
+ :param module_name: Name of the attribute
+ :param *args: Module specific attributes
+ :return:
+ """
+ init_fn = getattr(self, f"init_{module_name}")
+ init_fn(*args)
+
+ def init_warden_receiver(self, event, source):
+ """
+ Fill in metadata from a warden event
+ :param event: Source event
+ :return:
+ """
+ detect_time = parse_rfc_time(event["DetectTime"])
+ self.date = detect_time.strftime("%Y-%m-%d")
+ self.categories = event.get('Category', [])
+ self.description = event.get("Description", "")
+ self.note = event.get("Note", "")
+ self.ip_info = source.get('Note', "")
+ self.source_types = source.get('Type', [])
+ target_ports = []
+ protocols = []
+ for source in event.get('Source', []):
+ protocols += source.get('Proto', [])
+ for target in event.get('Target', []):
+ target_ports += target.get('Port', [])
+ protocols += target.get('Proto', [])
+ self.target_ports = [str(port) for port in set(target_ports)]
+ self.protocols = list(set(protocols) - {'tcp', 'udp'}) # don't include L4 protocols (TCP, UDP), keep just the application layer ones
+
+ def init_otx_receiver(self, pulse):
+ """
+ Fill in metadata from an OTX pulse
+ :param pulse: Source pulse
+ :return:
+ """
+ self.date = datetime.strftime(pulse.get('pulse_modified', datetime.now()), "%Y-%m-%d")
+ self.indicator_role = str(pulse.get('indicator_role', None))
+ self.ip_info = str(pulse.get('indicator_title', None))
+ self.description = str(pulse.get('pulse_name', None))
+ self.protocols = []
+ self.target_ports = []
+
+ def init_misp_receiver(self, event, attrib, ip_role):
+ """
+ Fill in metadata from a MISP event
+ :param event: Source event
+ :param attrib: Attribute with the source IP
+ :param ip_role: Role of the IP address (src/dst/both)
+ :return:
+ """
+ self.date = datetime.strftime(attrib.get('date', datetime.now()), "%Y-%m-%d")
+ self.tags = [tag["name"] for tag in event.get('tag_list', [])]
+ self.description = event.get('info', "")
+ self.ip_info = attrib.get('comment', "")
+ self.ip_role = ip_role
+ self.protocols = []
+ self.target_ports = []
+ try:
+ if attrib['type'] == "ip-dst|port":
+ split_attrib = attrib['value'].split('|')
+ if len(split_attrib) == 1:
+ split_attrib = attrib['value'].split(':')
+ if len(split_attrib) > 1:
+ self.target_ports = [int(split_attrib[1])]
+ except ValueError:
+ pass
+
+ def init_blacklists(self, blacklist_id, ip_info, download_time):
+ """
+ Fill in metadata from a blacklist record
+ :param blacklist_id: ID of the blacklist
+ :param ip_info: Additional info about the IP
+ :param download_time: Time when the blacklist was downloaded
+ :return:
+ """
+ self.date = download_time.strftime("%Y-%m-%d")
+ self.description = blacklist_id
+ self.ip_info = str(ip_info)
+ self.protocols = []
+ self.target_ports = []
+
+
+def classify_ip(ip_addr, module_name, logger, config, *args):
+ """
+ Assign a threat category based on the information provided in the incoming event
+
+ :return: List of assigned categories
+ """
+ try:
+ output = []
+ event = ClassifiableEvent(module_name, *args)
+ for category_id, category_params in config["categories"].items():
+ category_triggers = category_params.get("triggers", {}).get("general", "False").split("\n") + \
+ category_params.get("triggers", {}).get(module_name, "False").split("\n")
+ for trigger in category_triggers:
+ result, subcategories = eval_trigger(trigger, event, category_params, config, logger)
+ if result is True:
+ output.append({
+ "date": event.date,
+ "id": category_id,
+ "role": category_params["role"],
+ "subcategories": subcategories
+ })
+ break
+ except Exception as e:
+ logger.error(f"Error in threat category classification for IP {ip_addr}: {e}")
+ if not output:
+ output.append({"date": event.date, "id": "unknown", "role": "src", "subcategories": {}})
+ # with open(f"/var/log/nerd/threat_categorization_unknown.log", "a+") as logfile:
+ # logfile.write(f"[{datetime.now()}] MODULE: {module_name} IP: {ip_addr} EVENT-INFO: {event}\n")
+ logger.debug(f"Threat category classification for {ip_addr}: {output}; Event info: {event}")
+ return output
+
+
+def eval_trigger(trigger, event, category_params, config, logger):
+ """
+ Evaluate a category trigger
+ :param trigger: Trigger to be evaluated
+ :param event: Source event (instance of ClassifiableEvent) from which the trigger reads data
+ :param category_params: Category parameters (e.g. list of subcategories)
+ :param logger: Source module logger
+ :return: Result of the evaluation (True/False), dictionary with subcategory assignments
+ """
+ result = False
+ required_subcategories = category_params.get("subcategories", [])
+ subcategories = {s: [] for s in required_subcategories}
+
+ try:
+ split_trigger = trigger.split("->")
+ if eval(split_trigger[0]) is True:
+ result = True
+ if len(split_trigger) > 1:
+ subcategories.update(ast.literal_eval(split_trigger[1].lstrip()))
+ except Exception as e:
+ logger.error(f"Error when evaluating category trigger ({trigger}): {e}")
+ logger.error(f"Event info: {event}")
+
+ if result is True:
+ if "port" in required_subcategories:
+ subcategories["port"] += event.target_ports
+ subcategories["port"] = list(set(subcategories["port"]))
+ if "protocol" in required_subcategories:
+ subcategories["protocol"] += event.protocols
+ subcategories["protocol"] = list(set(subcategories["protocol"]))
+ if "malware_family" in required_subcategories:
+ text = f"{event.description};{event.ip_info}"
+ for family_id, family_data in config["malware_families"].items():
+ if match_str(family_data["common_name"], text):
+ subcategories["malware_family"].append(family_id.lower())
+ subcategories["malware_family"] = list(set(subcategories["malware_family"]))
+
+ for key in list(subcategories):
+ if not subcategories[key]:
+ subcategories.pop(key)
+ return result, subcategories
+
+
+def match_str(str_a, str_b):
+ """
+ Approximate (sub)string matching
+
+ Ignores character casing, whitespace and some special characters
+ """
+ simplified_a = str_a.strip().replace("_", "").replace(".", "").replace("-", "").lower()
+ simplified_b = str_b.strip().replace("_", "").replace(".", "").replace("-", "").lower()
+ return simplified_a in simplified_b
diff --git a/etc/nerd.yml b/etc/nerd.yml
index 42f2bd36..4e8214ab 100644
--- a/etc/nerd.yml
+++ b/etc/nerd.yml
@@ -73,6 +73,9 @@ dnsbl: dns_blacklists.yml
# Configuration file for EventCountLogger
event_logging_config: event_logging.yml
+# Threat categorization configuration file
+threat_categorization_config: threat_categorization.yml
+
# EventDB type (where to store/read events), may be one of:
# 'psql' - (default) Local PostgreSQL database (needs config in 'eventdb_psql' in nerdd.yml)
# 'mentat' - External Mentat instance (no storage by NERD, load via Mentat API) (needs config in 'eventdb_mentat')
diff --git a/etc/threat_categorization.yml b/etc/threat_categorization.yml
new file mode 100644
index 00000000..e6155df8
--- /dev/null
+++ b/etc/threat_categorization.yml
@@ -0,0 +1,289 @@
+# Path to YAML file with Malpedia's list of malware families
+# Used for malware subcategory classification
+malpedia_family_list_path: "/data/malpedia/malware_families.yml"
+
+# Colors for roles (src/dst) used in tags in web UI
+role_colors:
+ src: "#ffaa66" # light orange
+ dst: "#ff595b" # light red
+
+# Threat categorization
+# Structure:
+# dict{category ID -> category parameters}
+# Category parameters:
+# - role: IP role (src/dst) that will be assigned along with the main category
+# - label: Displayed name of the category
+# - description: General description of the category
+# - color: Color of the category tag in web UI (any CSS-compatible format)
+# - subcategories: List of required subcategories (port, protocol, malware_family)
+# Supported subcategories:
+# - port
+# - protocol
+# - malware_family
+#
+# - triggers: List of category triggers (single string separated by newlines)
+# Divided to sections for each source module so that they do not have to evaluate unnecessary triggers
+# IP is assigned a category if a related trigger is evaluated as True
+#
+# Supported syntax:
+# - triggers can have two parts separated by '->', both use standard Python syntax
+# - first part is mandatory and should resolve to either True or False
+# - second part is optional and contains a dictionary definition, which is used to specify subcategories
+#
+# Evaluation:
+# - triggers are evaluated by source modules when a new event is received (except for blacklists module
+# which instead uses it as a blacklist ID for blacklist -> category mapping)
+# - IP is assigned a category if any of the related statements resolve to True
+# - if required by the category configuration, the IP is also assigned subcategories based on the second
+# part of the statement (can be empty)
+# - within each statement it is possible to access an 'event' object (instance of ClassifiableEvent),
+# which represents the event that is currently being classified.
+# - event properties:
+# - date: Date of the event (YY-MM-DD)
+# - description: Event description (string)
+# - ip_info: Additional info about the IP (string, e.g. attribute comment from MISP)
+# - categories/tags/indicator_role: List of event categories/tags/indicators that can be used for classification
+# - protocols: List of protocols used by the IP
+# - target_ports: List of target ports used by the IP
+
+threat_categories:
+ unknown:
+ role: src
+ description: The IP was reported as a source of malicious/unexpected/rogue packets, but without any further specification.
+ label: Unknown
+ color: "#cccccc"
+
+ scan:
+ role: src
+ description: The IP address performs network scanning, i.e. it tries to connect to various targets to search for open ports/services.
+ label: Scanning
+ color: "#aaffff"
+ subcategories:
+ - port
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)scanning|scanner|probing', event.ip_info + event.description))
+ warden_receiver: |-
+ 'Recon.Scanning' in event.categories
+ otx_receiver: |-
+ bool(re.findall(r'(?i)scanning|scanner|probing', event.indicator_role))
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)scanning|scanner|probing', tag)) for tag in event.tags])
+ blacklists: |-
+ event.ip_info == 'crowdsecurity/iptables-scan-multi_ports'
+ event.ip_info == 'crowdsecurity/http-crawl-non_statics'
+ event.ip_info == 'crowdsecurity/http-path-traversal-probing'
+ event.ip_info == 'crowdsecurity/http-admin-interface-probing'
+ event.ip_info == 'crowdsecurity/http-probing'
+ event.ip_info == 'crowdsecurity/http-sensitive-files'
+
+ login:
+ role: src
+ description: The IP tries to access password-protected services without authorization (e.g. dictionary or bruteforce attacks, login attempts on honeypots).
+ label: Login attempts
+ color: "#55ddaa"
+ subcategories:
+ - protocol
+ - port
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)ssh.*(brute[\s_-]?force|login|intrusion|honeypot)', event.ip_info + event.description)) -> {'protocol': ['ssh']}
+ bool(re.findall(r'(?i)rdp.*(brute[\s_-]?force|login|intrusion|honeypot)', event.ip_info + event.description)) -> {'protocol': ['rdp']}
+ bool(re.findall(r'(?i)telnet.*(brute[\s_-]?force|login|intrusion|honeypot)', event.ip_info + event.description)) -> {'protocol': ['telnet']}
+ bool(re.findall(r'(?i)vnc.*(brute[\s_-]?force|login|intrusion|honeypot)', event.ip_info + event.description)) -> {'protocol': ['vnc']}
+ bool(re.findall(r'(?i)redis.*(brute[\s_-]?force|login|intrusion|honeypot)', event.ip_info + event.description)) -> {'protocol': ['redis']}
+ bool(re.findall(r'(?i)postgresql.*(brute[\s_-]?force|login|intrusion|honeypot)', event.ip_info + event.description)) -> {'protocol': ['postgresql']}
+ warden_receiver: |-
+ 'Attempt.Login' in event.categories
+ 'Intrusion.UserCompromise' in event.categories
+ 'Intrusion.AdminCompromise' in event.categories
+ otx_receiver: |-
+ bool(re.findall(r'(?i)brute[\s_-]?force', event.indicator_role))
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)login.*attempt', tag)) for tag in event.tags])
+ any([bool(re.findall(r'(?i)brute[\s_-]?force', tag)) for tag in event.tags])
+ blacklists: |-
+ bool(re.findall(r'(?i)brute[\s_-]?force.*ssh', event.ip_info)) -> {'protocol': ['ssh']}
+ bool(re.findall(r'(?i)brute[\s_-]?force.*ftp', event.ip_info)) -> {'protocol': ['ftp']}
+ bool(re.findall(r'(?i)brute[\s_-]?force.*sip', event.ip_info)) -> {'protocol': ['sip']}
+ event.description == 'blocklist_de-ssh' -> {'protocol': ['ssh']}
+ event.description == 'charles_the_haleys_ssh_dico_ips' -> {'protocol': ['ssh']}
+ event.description == 'charles_the_haleys_smtp_dico_ips' -> {'protocol': ['smtp']}
+ event.description == 'dataplane_org_sshclient' -> {'protocol': ['ssh']}
+ event.description == 'dataplane_org_sshpwauth' -> {'protocol': ['ssh']}
+ event.description == 'dataplane_org_telnet_login' -> {'protocol': ['telnet']}
+ event.description == 'bruteforceblocker'
+ event.description == 'blocklist_de-bruteforcelogin'
+ event.ip_info == 'crowdsecurity/http-generic-bf' -> {'protocol': ['http']}
+ event.ip_info.startswith('crowdsecurity/ssh-') -> {'protocol': ['ssh']}
+
+ ddos:
+ role: src
+ description: The IP has been observed as a source of volumetric (D)DoS attacks.
+ label: DDoS
+ color: "#c44a48"
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)http.*flood', event.ip_info + event.description)) -> {'protocol': ['http']}
+ bool(re.findall(r'(?i)dns.*flood', event.ip_info + event.description)) -> {'protocol': ['dns']}
+ bool(re.findall(r'(?i)udp.*flood', event.ip_info + event.description)) -> {'protocol': ['udp']}
+ bool(re.findall(r'(?i)(ping|icmp).*flood', event.ip_info + event.description)) -> {'protocol': ['icmp']}
+ bool(re.findall(r'(?i)syn.*flood', event.ip_info + event.description)) -> {'protocol': ['tcp']}
+ warden_receiver: |-
+ 'Availability.DoS' in event.categories
+ 'Availability.DDoS' in event.categories
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)d?dos', tag)) for tag in event.tags])
+ any([bool(re.findall(r'(?i)denial[\s_-]of[\s_-]service', tag)) for tag in event.tags])
+ blacklists: |-
+ bool(re.findall(r'(?i)d?dos', event.ip_info))
+
+ ddos-amplifier:
+ role: dst
+ description: The IP runs a service which can be (and often is) misused as an amplifier for DDoS attacks, e.g. open DNS resolvers, NTP servers, memcached, etc.
+ label: DDoS amplifier
+ color: "#ff9769"
+ subcategories:
+ - protocol
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)(Open|Abusable)[\s_-]DNS', event.ip_info + event.description)) -> {'protocol': ['dns']}
+ bool(re.findall(r'(?i)(Open|Abusable)[\s_-]Memcached', event.ip_info + event.description)) -> {'protocol': ['memcached']}
+ bool(re.findall(r'(?i)(Open|Abusable)[\s_-]NTP', event.ip_info + event.description)) -> {'protocol': ['ntp']}
+ warden_receiver: |-
+ ('Availability.DoS' in event.categories or 'Availability.DDoS' in event.categories) and bool(re.findall(r'(?i)dns.*amplification', event.description + event.note)) -> {'protocol': ['dns']}
+ 'Vulnerable.Config' in event.categories and 'dns' in event.protocols -> {'protocol': ['dns']}
+ 'Vulnerable.Config' in event.categories and 'ntp' in event.protocols -> {'protocol': ['ntp']}
+ 'Vulnerable.Config' in event.categories and 'memcached' in event.protocols -> {'protocol': ['memcached']}
+ 'Backscatter' in event.ip_info and 'dns' in event.protocols -> {'protocol': ['dns']}
+ 'Backscatter' in event.ip_info and 'ntp' in event.protocols -> {'protocol': ['ntp']}
+ 'Backscatter' in event.ip_info and 'memcached' in event.protocols -> {'protocol': ['memcached']}
+
+ spam:
+ role: src
+ description: The IP is sending spam.
+ label: Spam
+ color: "#2255bb"
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)send.*spam', event.ip_info + event.description))
+ warden_receiver: |-
+ 'Abusive.Spam' in event.categories
+ any([type == 'Spam' for type in event.source_types])
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)spam', tag)) for tag in event.tags])
+ blacklists: |-
+ event.description == 'sblam_ips'
+ event.description == 'psbl'
+ event.description == 'spamhaus_edrop'
+ bool(re.findall(r'(?i)send.*spam', event.ip_info))
+ event.ip_info == 'crowdsecurity/postfix-spam'
+
+ malware_distribution:
+ role: dst
+ description: The IP is used to distribute malware (e.g. hosts an HTTP URL from which a malware is being downloaded).
+ label: Malware distribution
+ color: "#dd77ff"
+ subcategories:
+ - malware_family
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)malware.*host', event.ip_info + event.description))
+ bool(re.findall(r'(?i)malware.*download', event.ip_info + event.description))
+ warden_receiver: |-
+ any([type == 'Malware' for type in event.source_types])
+ otx_receiver: |-
+ bool(re.findall(r'(?i)malware.*host', event.indicator_role))
+ bool(re.findall(r'(?i)malware.*download', event.indicator_role))
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)(malware|trojan|ransomware)', tag)) for tag in event.tags]) and event.ip_role == "dst"
+ blacklists: |-
+ event.description == 'urlhaus_ips'
+
+ cc:
+ role: dst
+ description: The IP is used as a Command&Control server for a botnet/malware.
+ label: Command and control
+ color: "#ff77dd"
+ subcategories:
+ - malware_family
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)command.*control', event.ip_info + event.description))
+ bool(re.findall(r'(?i)botnet[\s_-]cc', event.ip_info + event.description))
+ bool(re.findall(r'(?i)c2[\s_-]server', event.ip_info + event.description))
+ bool(re.findall(r'(?i)c&?c[\s_-]server', event.ip_info + event.description))
+ warden_receiver: |-
+ any([type == 'CC' for type in event.source_types])
+ otx_receiver: |-
+ bool(re.findall(r'(?i)command.*control', event.indicator_role))
+ bool(re.findall(r'(?i)c2', event.indicator_role))
+ bool(re.findall(r'(?i)c&?c', event.indicator_role))
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)command.*control', tag)) for tag in event.tags])
+ any([bool(re.findall(r'(?i)c2', tag)) for tag in event.tags])
+ any([bool(re.findall(r'(?i)c&c', tag)) for tag in event.tags])
+ blacklists: |-
+ event.description == 'feodo' -> {'malware_family': ['win.feodo']}
+ event.description == 'bambenek_c2'
+
+ botnet_drone:
+ role: src
+ description: The IP is acting as a bot/drone of a botnet.
+ label: Botnet drone
+ color: "#d090d0"
+ subcategories:
+ - malware_family
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)botnet.*drone', event.ip_info + event.description))
+ bool(re.findall(r'(?i)botnet.*member', event.ip_info + event.description))
+ warden_receiver: |-
+ 'Intrusion.Botnet' in event.categories
+ any([type == 'Botnet' for type in event.source_types])
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)botnet', tag)) for tag in event.tags])
+ blacklists: |-
+ event.description == 'mirai_tracker_ips' -> {'malware_family': ['elf.mirai']}
+
+ phishing_site:
+ role: dst
+ description: The IP is hosting a phishing website.
+ label: Phishing site
+ color: "#dddd00"
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)phishing.*site', event.ip_info + event.description))
+ warden_receiver: |-
+ any([type == 'Phishing' for type in event.source_types])
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)phishing.*site', tag)) for tag in event.tags])
+ any([bool(re.findall(r'(?i)phishing', tag)) for tag in event.tags]) and event.ip_role == "dst"
+ blacklists: |-
+ event.description == 'openphish'
+
+ exploit:
+ role: src
+ description: The IP is attempting to exploit known vulnerabilities.
+ label: Exploit
+ color: "#22ee88"
+ subcategories:
+ - protocol
+ triggers:
+ general: |-
+ bool(re.findall(r'(?i)attempt.*exploit', event.ip_info + event.description))
+ warden_receiver: |-
+ 'Attempt.Exploit' in event.categories
+ otx_receiver: |-
+ 'Apache honeypot logs' in event.description -> {'protocol': ['http']}
+ bool(re.findall(r'(?i)exploit', event.indicator_role))
+ misp_receiver: |-
+ any([bool(re.findall(r'(?i)exploit', tag)) for tag in event.tags])
+ 'CERT-XLM:intrusion-attempts="new-attack-signature"' in event.tags
+ 'circl:incident-classification="XSS"' in event.tags
+ 'circl:incident-classification="sql-injection"' in event.tags
+ blacklists: |-
+ bool(re.findall(r'(?i)CVE[-_]20', event.ip_info))
+ event.ip_info == 'http-sqli-probing' -> {'protocol': ['http']}
+ event.ip_info == 'http-xss-probing' -> {'protocol': ['http']}
+ event.ip_info == 'http-backdoors-attempts' -> {'protocol': ['http']}
diff --git a/install/cron/nerd b/install/cron/nerd
index c1f93109..42c4657e 100644
--- a/install/cron/nerd
+++ b/install/cron/nerd
@@ -11,6 +11,9 @@
00 * * * * nerd /nerd/scripts/generate_blocklist.sh 0.5 | sort -n > /data/web_data/bad_ips.txt.tmp && mv /data/web_data/bad_ips.txt{.tmp,}
00 * * * * nerd /nerd/scripts/generate_blocklist.sh 0.2 | sort -n > /data/web_data/bad_ips_med_conf.txt.tmp && mv /data/web_data/bad_ips_med_conf.txt{.tmp,}
+# Generate lists of IPs and their categories every hour
+00 * * * * nerd python3 /nerd/scripts/generate_ip_category_files.py --threshold 0.5 --output /data/web_data
+
# Remove old IDEA messages from PostgreSQL every day at 03:00
# (enable if local PSQL is used to store alerts from Warden)
#0 03 * * * nerd /nerd/scripts/nerd_clean_eventdb.sh > /dev/null
@@ -24,6 +27,8 @@
40 01,09,17 * * * nerd rsync -azq rsync-mirrors.uceprotect.net::RBLDNSD-ALL/dnsbl-1.uceprotect.net /data/blacklists/uceprotect-level1
# rsync PSBL blacklist 3 times a day
41 01,09,17 * * * nerd rsync -zq psbl-mirror.surriel.com::psbl/psbl.txt /data/blacklists/psbl.txt
+# rsync Crowdsec blacklist 3 times a day
+42 01,09,17 * * * nerd rsync -zq logs.liberouter.org::crowdsec/crowdsec_blacklist.csv /data/blacklists/crowdsec.csv
# Export Crowdsec community blacklist to CSV
# (enable if you're using Crowdsec)
diff --git a/scripts/download_malpedia_families.py b/scripts/download_malpedia_families.py
new file mode 100644
index 00000000..d00301c6
--- /dev/null
+++ b/scripts/download_malpedia_families.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+# Download Malpedia's list of malware families to /data/malpedia/malware_families.yml
+
+import requests
+import yaml
+import json
+import os
+
+url = "https://malpedia.caad.fkie.fraunhofer.de/api/get/families"
+response = requests.get(url)
+data = json.loads(response.content)
+output = {}
+output_dir = "/data/malpedia/"
+
+for family_id, family_data in data.items():
+ name = family_data.get("common_name", "")
+ if name == "":
+ try:
+ name = family_id.split('.')[1]
+ except Exception:
+ name = family_id
+ output[family_id] = {
+ "common_name": name,
+ "description": family_data.get("description", ""),
+ "url": f"https://malpedia.caad.fkie.fraunhofer.de/details/{family_id}"
+ }
+
+if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+with open(f"{output_dir}/malware_families.yml", "w+") as outfile:
+ yaml.dump(output, outfile, default_flow_style=False)
diff --git a/scripts/generate_ip_category_files.py b/scripts/generate_ip_category_files.py
new file mode 100644
index 00000000..c64ea5f4
--- /dev/null
+++ b/scripts/generate_ip_category_files.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""
+Generate list of all IPs in NERD's database with their categories
+Parameters - path to config directory
+ - path to output directory
+ - blacklist confidence threshold
+"""
+import os
+import sys
+import subprocess
+import argparse
+
+# Add to path the "one directory above the current file location" to find modules from "common"
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')))
+
+from common.config import read_config
+
+
+# parse arguments
+parser = argparse.ArgumentParser(
+ prog="generate_ip_category_files.py",
+ description="Generate list of all IPs in NERD's database with their categories"
+)
+parser.add_argument("-c", '--config', dest='cfg_file', default="/etc/nerd/threat_categorization.yml",
+ help="Path to configuration file (default: /etc/nerd/threat_categorization.yml)")
+parser.add_argument("-o", '--output', dest='out_dir', default="/data/web_data/",
+ help="Path to output directory (default: /data/web_data/)")
+parser.add_argument("-t", '--threshold', dest='conf_thr', default="0.5",
+ help="Blacklist confidence threshold (default: 0.5)")
+parser.add_argument("-v", '--verbose', dest="verbose", action="store_true", help="Verbose mode")
+args = parser.parse_args()
+
+# read categorization config
+config = read_config(args.cfg_file)
+categories = [cat for cat in config.get('threat_categories')]
+categories.remove("unknown")
+
+# bash script used to execute the DB query
+script = """
+ echo \"# generated at $(date -u '+%Y-%m-%d %H:%M UTC')\" > {out_file}.tmp &&
+ echo \"# {header}\" >> {out_file}.tmp &&
+ mongosh nerd --quiet --eval '{query}' | grep -v \"^$\" | sort -n >> {out_file}.tmp &&
+ mv {out_file}{{.tmp,}}
+"""
+
+def fstr(template):
+ return eval(f"f'''{template}'''")
+
+########################################################################################################################
+
+# full list - line format (ip,category,confidence)
+if args.verbose:
+ print("Generating full IP list (line format)")
+
+out_file = f"{args.out_dir}/ip_category.csv"
+header = "ip,category,confidence"
+query = '''
+function int2ip (ipInt) {
+ return ( (ipInt>>>24) + "." + (ipInt>>16 & 255) + "." + (ipInt>>8 & 255) + "." + (ipInt & 255) );
+}
+db.ip.aggregate([
+ { $unwind: "$_threat_category_summary" },
+ { $set: { category: "$_threat_category_summary.c", confidence: { $toString: "$_threat_category_summary.conf" } } },
+ { $project: { _id: 1, category: 1, confidence: 1 } }
+]).forEach(function(rec) { print(int2ip(rec._id) + "," + rec.category + "," + rec.confidence); })
+'''
+subprocess.run(fstr(script), shell=True)
+
+########################################################################################################################
+
+# full list - table format (ip,conf_scan,conf_bruteforce,...)
+if args.verbose:
+ print("Generating full IP list (table format)")
+
+out_file = f"{args.out_dir}/ip_category_table.csv"
+header = f"# ip,conf_{',conf_'.join(categories)}"
+query = '''
+function int2ip (ipInt) {
+ return ( (ipInt>>>24) + "." + (ipInt>>16 & 255) + "." + (ipInt>>8 & 255) + "." + (ipInt & 255) );
+}
+db.ip.aggregate([
+ { $project: { _id: 1, _threat_category_summary: 1 } },
+ { $set: { categories: { $arrayToObject: { $map: { input: "$_threat_category_summary", as: "cat", in: { k: "$$cat.c", v: { $toString: "$$cat.conf" } } } } } } },
+ { $replaceWith: { $mergeObjects: ["$$ROOT", "$categories"] } },
+ { $project: {
+ "_id": 1,
+'''
+for category in categories:
+ query += f'"{category}": {{ $ifNull: ["${category}", "0"] }},\n'
+query += '''
+}}]).forEach(function(rec) {
+ var categories = Object.keys(rec).filter(key => key !== "_id");
+ print(
+ int2ip(rec._id) + "," +
+ categories.map(key => rec[key]).join(",")
+ );
+})
+'''
+subprocess.run(fstr(script), shell=True)
+
+########################################################################################################################
+
+# blacklists
+if args.verbose:
+ print("Generating blacklists")
+
+for category in categories:
+ out_file = f"{args.out_dir}/bl_{category}.txt"
+ header = ""
+ query = f'''
+ function int2ip (ipInt) {{
+ return ( (ipInt>>>24) + "." + (ipInt>>16 & 255) + "." + (ipInt>>8 & 255) + "." + (ipInt & 255) );
+ }}
+ db.ip.find({{"_threat_category_summary": {{$elemMatch: {{"c": "{category}", "conf": {{$gt: {args.conf_thr}}}}}}}, "tags.whitelist": {{$exists: false}}}}, {{_id: 1}}).sort({{"_threat_category_summary.conf": -1}}).forEach( function(rec) {{ print(int2ip(rec._id)); }} );
+ '''
+ subprocess.run(fstr(script), shell=True)
+
+if args.verbose:
+ print("Done")
diff --git a/scripts/misp_updater.py b/scripts/misp_updater.py
index fb0eec68..26a62812 100755
--- a/scripts/misp_updater.py
+++ b/scripts/misp_updater.py
@@ -20,6 +20,7 @@
from common.config import read_config
import NERDd.core.mongodb as mongodb
from common.task_queue import TaskQueueWriter
+from common.threat_categorization import *
DEFAULT_MONGO_HOST = 'localhost'
DEFAULT_MONGO_PORT = 27017
@@ -64,6 +65,15 @@
logger.info("Loading config file {}".format(common_cfg_file))
config.update(read_config(common_cfg_file))
+# Read categorization config
+categorization_cfg_file = os.path.join(config_base_path, 'threat_categorization.yml')
+logger.info("Loading config file {}".format(categorization_cfg_file))
+config.update(read_config(categorization_cfg_file))
+categorization_config = {
+ "categories": config.get('threat_categorization'),
+ "malware_families": read_config(config.get('malpedia_family_list_path'))
+}
+
inactive_ip_lifetime = config.get('record_life_length.misp', 180)
db = mongodb.MongoEntityDatabase(config)
@@ -330,6 +340,7 @@ def process_ip(ip_addr, ip_info):
:return: None
"""
logger.debug("Processing IP: {}".format(ip_addr))
+ update_requests = []
# check ip record in DB
try:
@@ -358,6 +369,9 @@ def process_ip(ip_addr, ip_info):
dedup_event['role'] = "src and dst at the same time"
event_ids_roles[index][1] = "src and dst at the same time"
+ # aggregated threat category classification records
+ threat_category = {}
+
# create all misp events and save the youngest datetime of the event for keep alive token
events = []
youngest_date = datetime(year=2000, month=1, day=1, hour=0, minute=0, second=0)
@@ -368,18 +382,56 @@ def process_ip(ip_addr, ip_info):
if youngest_date < new_event['date']:
youngest_date = new_event['date']
+ attrib = {'type': '', 'value': '', 'comment': ''} # TODO get attrib info from misp
+ ip_role = event_info.get('role', "")
+ for category in classify_ip(ip_addr, "misp_receiver", logger, categorization_config, new_event, attrib, ip_role):
+ role = category['role']
+ id = category['id']
+ date = category['date']
+ subcategories = category['subcategories']
+ if role not in threat_category:
+ threat_category[role] = {}
+ if id not in threat_category[role]:
+ threat_category[role][id] = {}
+ if date not in threat_category[role][id]:
+ threat_category[role][id][date] = {}
+ if 'n_reports' not in threat_category[role][id][date]:
+ threat_category[role][id][date]['n_reports'] = 0
+ if 'subcategories' not in threat_category[role][id][date]:
+ threat_category[role][id][date]['subcategories'] = {}
+ for subcategory in subcategories:
+ old = threat_category[role][id][date]['subcategories'].get(subcategory, [])
+ new = subcategories[subcategory]
+ threat_category[role][id][date]['subcategories'][subcategory] = list(set(old + new))
+ threat_category[role][id][date]['n_reports'] += 1
+
+ # threat category updates
+ for role in threat_category:
+ for id in threat_category[role]:
+ for date in threat_category[role][id]:
+ n_reports = threat_category[role][id][date]['n_reports']
+ subcategory_updates = []
+ for subcategory, values in threat_category[role][id][date]['subcategories'].items():
+ subcategory_updates.append(('extend_set', subcategory, values))
+ update_requests += [(
+ 'array_upsert',
+ '_threat_category',
+ {'d': date, 'c': id},
+ [('add', 'src.misp', n_reports), *subcategory_updates]
+ )]
+
if events:
live_till = youngest_date + timedelta(days=inactive_ip_lifetime)
if db_entity is not None:
# compare 'misp_events' attrib from NERD with events list, if not same --> insert, else do not insert
if db_entity.get('misp_events', {}) != events:
# construct new update request and send it
- update_requests = [('set', 'misp_events', events), ('set', '_ttl.misp', live_till),
+ update_requests += [('set', 'misp_events', events), ('set', '_ttl.misp', live_till),
('setmax', 'last_activity', youngest_date)]
tq.put_task('ip', ip_addr, update_requests, "misp_updater")
else:
# ip address not even in NERD --> insert it
- update_requests = [('set', 'misp_events', events), ('set', '_ttl.misp', live_till), ('setmax',
+ update_requests += [('set', 'misp_events', events), ('set', '_ttl.misp', live_till), ('setmax',
'last_activity', youngest_date)]
tq.put_task('ip', ip_addr, update_requests, "misp_updater")
|
|---|