diff --git a/security/etpro_telemetry/Makefile b/security/etpro_telemetry/Makefile new file mode 100644 index 000000000..8ccca174a --- /dev/null +++ b/security/etpro_telemetry/Makefile @@ -0,0 +1,8 @@ +PLUGIN_NAME= etpro_telemetry +PLUGIN_VERSION= 1.0 +#PLUGIN_REVISION= 1 +PLUGIN_COMMENT= ETPRO telemetry edition +PLUGIN_MAINTAINER= ad@opnsense.org +PLUGIN_WWW= https://docs.opnsense.org/manual/etpro_telemetry.html + +.include "../../Mk/plugins.mk" diff --git a/security/etpro_telemetry/pkg-descr b/security/etpro_telemetry/pkg-descr new file mode 100644 index 000000000..43a8d3e25 --- /dev/null +++ b/security/etpro_telemetry/pkg-descr @@ -0,0 +1,20 @@ +Todays cybersecurity engineers need timely and accurate data about eminent threats and how they spread around the globe. +With this data cybersecurity researchers and analysts can improve the detection of malicious network traffic. +The times when we could rely on just firewall rules for our protection are long gone. +Additional layers of security are desperately needed to guard against these attacks. + +With growing risks the need to fortify our security is growing for both big enterprises as for SMEs alike, but often +out of reach for the latter. +An important extra security addition is an Intrusion Detection and Prevention System (IDS/IPS). + +The IDS/IPS available in OPNsense is based on Suricata. +This open source IDS/IPS engine has proven its value in OPNsense, especially in combination with the free Proofpoint ETOpen ruleset. + +The need for valuable threat detection data and the increasing importance of additional network security +has brought Proofpoint and OPNsense together. +Our joined efforts resulted in the ETPro Telemetry edition. + +The ETPro Telemetry edition embraces our vision that sharing knowledge leads to better products. + +When you allow your OPNsense system to share anonymized information about detected threats - the alerts - +you are able to use the ETPro ruleset free of charge. diff --git a/security/etpro_telemetry/src/etc/cron.d/etpro-telemetry.cron b/security/etpro_telemetry/src/etc/cron.d/etpro-telemetry.cron new file mode 100644 index 000000000..bd4d7b1e7 --- /dev/null +++ b/security/etpro_telemetry/src/etc/cron.d/etpro-telemetry.cron @@ -0,0 +1,10 @@ +# DO NOT EDIT THIS FILE -- OPNsense auto-generated file +# +# User-defined crontab files can be loaded via /etc/cron.d +# or /usr/local/etc/cron.d and follow the same format as +# /etc/crontab, see the crontab(5) manual page. +SHELL=/bin/sh +PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin +#minute hour mday month wday who command +0,30 * * * * root /usr/local/opnsense/scripts/etpro_telemetry/send_heartbeat.py +* * * * * root /usr/local/opnsense/scripts/etpro_telemetry/send_telemetry.py diff --git a/security/etpro_telemetry/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/ProofpointEtController.php b/security/etpro_telemetry/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/ProofpointEtController.php new file mode 100644 index 000000000..e817ae4f1 --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/ProofpointEtController.php @@ -0,0 +1,53 @@ +sessionClose(); + $backend = new Backend(); + $response = $backend->configdRun('proofpoint et status'); + $activity = json_decode($response, true); + + return $activity; + } +} diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/dump_data.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/dump_data.py new file mode 100755 index 000000000..884b52bf4 --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/dump_data.py @@ -0,0 +1,60 @@ +#!/usr/local/bin/python2.7 + +""" + Copyright (c) 2018-2019 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" + +import argparse +import urllib3 +import datetime +import ujson +import telemetry.log +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +parser = argparse.ArgumentParser() +parser.add_argument('-l', '--log', help='log directory containing eve.json files', default="/var/log/suricata/") +parser.add_argument('-t', '--time', help='max seconds to read from now()', type=int, default=3600) +parser.add_argument('-p', '--parsed', help='show data as shipped using send_telemetry', + default=False, action="store_true") +parser.add_argument('-L', '--limit', help='limit number of rows', type=int, default=-1) +args = parser.parse_args() + +last_update = datetime.datetime.now() - datetime.timedelta(seconds=float(args.time)) + +event_collector = telemetry.EventCollector() +row_count = 0 +for record in telemetry.log.reader(args.log, last_update=last_update): + if args.parsed: + event_collector.push(record) + else: + print (ujson.dumps(record)) + + row_count += 1 + if args.limit != -1 and row_count >= args.limit: + break + +if args.parsed: + for push_data in event_collector: + print (push_data.strip()) diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py new file mode 100755 index 000000000..232edb9ce --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py @@ -0,0 +1,71 @@ +#!/usr/local/bin/python2.7 + +""" + Copyright (c) 2018-2019 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" + +import sys +import argparse +import requests +import syslog +import urllib3 +import telemetry +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +parser = argparse.ArgumentParser() +parser.add_argument('-e', '--endpoint', + help='Endpoint url to reach', + default="%s/api/v1/sensor" % telemetry.BASE_URL) +parser.add_argument('-i', '--insecure', + help='Insecure, skip certificate validation', + action="store_true", + default=False) +parser.add_argument('-c', '--config', + help='rule downloader configuration', + default="/usr/local/etc/suricata/rule-updater.config" + ) +args = parser.parse_args() + +exit_code = -1 +cnf = telemetry.get_config(args.config) +if cnf.token is not None: + params = {'timeout': 5, 'headers': {'Authorization': 'Bearer %s' % cnf.token}} + if args.insecure: + params['verify'] = False + try: + r = requests.head(args.endpoint, **params) + if r.status_code == 200: + # expected result, set exit code + exit_code = 0 + else: + syslog.syslog(syslog.LOG_ERR, 'unexpected result from %s (http_code %s)' % (args.endpoint, r.status_code)) + except requests.exceptions.ConnectionError: + syslog.syslog(syslog.LOG_ERR, 'connection error sending heardbeat to %s' % args.endpoint) +else: + syslog.syslog(syslog.LOG_ERR, 'telemetry token missing in %s' % args.config) + + +# exit +sys.exit(exit_code) diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_telemetry.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_telemetry.py new file mode 100755 index 000000000..ece86d1e3 --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_telemetry.py @@ -0,0 +1,119 @@ +#!/usr/local/bin/python2.7 + +""" + Copyright (c) 2018-2019 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" + +import sys +import os +import argparse +import requests +import time +import syslog +import urllib3 +import ujson +import telemetry.log +import telemetry.state + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +parser = argparse.ArgumentParser() +parser.add_argument('-e', '--endpoint', help='Endpoint url to reach', + default="%s/api/v1/event" % telemetry.BASE_URL) +parser.add_argument('-i', '--insecure', help='Insecure, skip certificate validation', + action="store_true", default=False) +parser.add_argument('-c', '--config', help='rule downloader configuration', + default="/usr/local/etc/suricata/rule-updater.config") +parser.add_argument('-l', '--log', help='log directory containing eve.json files', + default="/var/log/suricata/") +parser.add_argument('-s', '--state', help='persistent state (and lock) filename', + default="/usr/local/var/run/et_telemetry.state") +parser.add_argument('-d', '--days', help='Maximum number of days to look back on initial run', type=int, default=2) +args = parser.parse_args() + + +exit_code = -1 +send_start_time = time.time() +telemetry_state = telemetry.state.Telemetry(filename=args.state, init_last_days=args.days) +if not telemetry_state.is_running(): + cnf = telemetry.get_config(args.config) + if cnf.token is not None: + if os.path.isdir(args.log): + last_update = telemetry_state.get_last_update() + event_collector = telemetry.EventCollector() + row_count = 0 + max_timestamp = None + for record in telemetry.log.reader(args.log, last_update): + if max_timestamp is None or record['__timestamp__'] > max_timestamp: + max_timestamp = record['__timestamp__'] + event_collector.push(record) + row_count += 1 + + # data collected, log and push + if row_count > 0 and max_timestamp is not None: + syslog.syslog( + syslog.LOG_NOTICE, + 'telemetry data collected %d records in %.2f seconds @%s' % ( + row_count, time.time() - send_start_time, max_timestamp + ) + ) + for push_data in event_collector: + params = { + 'timeout': 5, + 'headers': {'Authorization': 'Bearer %s' % cnf.token}, + 'data': push_data.strip() + } + if args.insecure: + params['verify'] = False + + r = requests.post(args.endpoint, **params) + if r.status_code != 201: + syslog.syslog( + syslog.LOG_ERR, + 'unexpected result from %s (http_code %s)' % (args.endpoint, r.status_code) + ) + exit_code = -1 + break + else: + try: + ujson.loads(r.text) + exit_code = 0 + except ValueError: + syslog.syslog(syslog.LOG_ERR, 'telemetry unexpected response %s' % r.text[:256]) + exit_code = -1 + break + if exit_code == 0: + # update timestamp, last record processed + telemetry_state.set_last_update(max_timestamp) + else: + # no data + exit_code = 0 + else: + syslog.syslog(syslog.LOG_ERR, 'telemetry token missing in %s' % args.config) + else: + syslog.syslog(syslog.LOG_ERR, 'directory %s missing' % args.log) + + +sys.exit(exit_code) diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/sensor_info.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/sensor_info.py new file mode 100755 index 000000000..bb6abc456 --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/sensor_info.py @@ -0,0 +1,61 @@ +#!/usr/local/bin/python2.7 + +""" + Copyright (c) 2018-2019 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" + +import argparse +import requests +import urllib3 +import ujson +import telemetry +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +parser = argparse.ArgumentParser() +parser.add_argument('-e', '--endpoint', help='Endpoint url to reach', + default="%s/api/v1/sensorinfo" % telemetry.BASE_URL) +parser.add_argument('-i', '--insecure', help='Insecure, skip certificate validation', + action="store_true", default=False) +parser.add_argument('-c', '--config', help='rule downloader configuration', + default="/usr/local/etc/suricata/rule-updater.config") +args = parser.parse_args() + +cnf = telemetry.get_config(args.config) +if cnf.token is not None: + try: + req = requests.get(args.endpoint, headers={'Authorization': 'Bearer %s' % cnf.token}) + if req.status_code == 200: + response = ujson.loads(req.text) + response['status'] = 'ok' + else: + response = {'status': 'failed', 'response': req.text} + except requests.exceptions.SSLError as e: + response = {'status': 'failed', 'response': '%s' % e} + except ValueError: + response = {'status': 'failed', 'response': req.text} +else: + response = {'status': 'unconfigured'} + +print (ujson.dumps(response)) diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py new file mode 100644 index 000000000..ee66262f0 --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py @@ -0,0 +1,140 @@ +""" + Copyright (c) 2018-2019 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" +import os +import subprocess +import tempfile +import collections +try: + from configparser import ConfigParser +except ImportError: + from ConfigParser import ConfigParser +import netaddr +import ujson + + +BASE_URL = 'https://opnsense.emergingthreats.net' + + +def get_config(rule_update_config): + """ + retrieve device token, since we align our telemetry data to the existing rule download feature in OPNsense + it should be safe to assume rule-updater.config contains the token that is used. + :param rule_update_config: path to OPNsense rule update configuration + :return: token id or None if not found + """ + response = collections.namedtuple('sensor', 'token') + if os.path.exists(rule_update_config): + cnf = ConfigParser() + cnf.read(rule_update_config) + if cnf.has_section('__properties__'): + if cnf.has_option('__properties__', 'et_telemetry.token'): + response.token = cnf.get('__properties__', 'et_telemetry.token') + + return response + + +class EventCollector(object): + """ Event collector, responsible for extracting and anonymising from an eve.json stream + """ + def __init__(self): + self._tmp_handle = tempfile.NamedTemporaryFile() + self._local_networks = list() + self._get_local_networks() + + def _get_local_networks(self): + """ collect local attached networks for anonymization purposes + :return: None + """ + with tempfile.NamedTemporaryFile() as output_stream: + subprocess.call(['ifconfig', '-a'], stdout=output_stream, stderr=open(os.devnull, 'wb')) + output_stream.seek(0) + for line in output_stream: + if line.startswith(b'\tinet'): + parts = line.split() + if len(parts) > 3: + if parts[0] == 'inet6' and parts[2] == 'prefixlen': + # IPv6 addresses + self._local_networks.append( + netaddr.IPNetwork("%s/%s" % (parts[1].split('%')[0], parts[3])) + ) + elif parts[0] == 'inet' and len(parts) > 3 and parts[2] == 'netmask': + # IPv4 addresses + mask = int(parts[3], 16) + self._local_networks.append( + netaddr.IPNetwork("%s/%s" % (netaddr.IPAddress(parts[1]), netaddr.IPAddress(mask))) + ) + + def is_local_address(self, address): + """ check if provided address is local for this device + :param address: address (string) + :return: boolean + """ + addr_to_check = netaddr.IPAddress(address) + for local_network in self._local_networks: + if addr_to_check in local_network: + return True + return False + + def push(self, record): + """ cleanup and write record + :param record: parsed eve log record + :return: None + """ + to_push = dict() + for address in ['src_ip', 'dest_ip']: + if address in record: + if self.is_local_address(record[address]): + if record[address].find(':') > -1: + # replace local IPv6 address + to_push[address] = 'xxxx:xxxx:%s' % ':'.join(record[address].split(':')[-2:]) + else: + to_push[address] = 'xxx.xxx.xxx.%s' % record[address].split('.')[-1] + else: + # non local address + to_push[address] = record[address] + + # unfiltered output fields + for attr in ["timestamp", "flow_id", "in_iface", "event_type", + "vlan", "src_port", "dest_port", "proto", "alert"]: + if attr in record: + to_push[attr] = record[attr] + + self._tmp_handle.write(("%s\n" % ujson.dumps(to_push)).encode()) + + def get(self): + """ fetch all data from temp + :return: + """ + self._tmp_handle.seek(0) + return self._tmp_handle.read() + + def __iter__(self): + """ Iterate parsed events + :return: + """ + self._tmp_handle.seek(0) + for line in self._tmp_handle: + yield line diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/log.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/log.py new file mode 100644 index 000000000..67fa20592 --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/log.py @@ -0,0 +1,105 @@ +""" + Copyright (c) 2018-2019 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" +import os +import re +import glob +import datetime +import ujson + + +def reverse_log_reader(filename): + """ read log file in reverse order + :param filename: filename + :return: generator + """ + block_size = 81920 + input_stream = open(filename, 'rU') + input_stream.seek(0, os.SEEK_END) + file_byte_start = input_stream.tell() + + data = '' + while file_byte_start > 0: + if file_byte_start - block_size < 0: + block_size = file_byte_start + file_byte_start = 0 + else: + file_byte_start -= block_size + + input_stream.seek(file_byte_start) + + data = input_stream.read(block_size) + data + bol = data.rfind('\n') + eol = len(data) + + while bol > -1: + yield data[bol:eol] + eol = bol + bol = data.rfind('\n', 0, eol) + + data = data[:eol] if bol == -1 else '' + + if file_byte_start == 0 and bol == -1: + yield data + + +def parse_log_line(line): + """ parse eve log records, add __timestamp__ containing a timezoned datetime object + :param line: raw json string + :return: dict or None if unable to parse + """ + try: + record = ujson.loads(line.strip()) + except ValueError: + record = dict() + if 'timestamp' in record: + # convert timestamp + try: + tmp = list(map(lambda x: int(x), re.split('T|-|\:|\.|\+', record['timestamp']))) + tz = record['timestamp'][-5:] + ts = datetime.datetime(*tmp[:7]) + ts -= datetime.timedelta(hours=int(tz[2:3]), minutes=int(tz[-2:])) * int(tz[0:1] + '1') + record['__timestamp__'] = ts + return record + except ValueError: + pass + + return None + + +def reader(log_directory, last_update): + """ read and parse eve logs until timestamp is reached + :param log_directory: directory to search for eve.json* + :param last_update: datetime of last send event + :return: iterator + """ + for filename in sorted(glob.glob("%s/eve.json*" % log_directory)): + for line in reverse_log_reader(filename=filename): + record = parse_log_line(line) + if record: + if record['__timestamp__'] > last_update: + yield record + else: + return diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py new file mode 100644 index 000000000..d5ea56783 --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py @@ -0,0 +1,71 @@ +""" + Copyright (c) 2018-2019 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" +import fcntl +import datetime + + +class Telemetry(object): + def __init__(self, filename, init_last_days=2): + self._filename = filename + self._init_last_days = init_last_days + try: + self._file_handle = open(self._filename, 'a+') + fcntl.flock(self._file_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + # already running + self._file_handle = None + + def is_running(self): + return self._file_handle is None + + def get_last_update(self): + """ return last used timestamp + :return: datetime + """ + self._file_handle.seek(0) + try: + result = datetime.datetime.fromtimestamp(float(self._file_handle.readline())) + except ValueError: + result = datetime.datetime.now() - datetime.timedelta(days=self._init_last_days) + + return result + + def set_last_update(self, stamp): + """ set last timestamp + :param stamp: datetime object + :return: + """ + self._file_handle.seek(0) + self._file_handle.truncate() + self._file_handle.write("%s\n" % stamp.strftime('%s.%f')) + + def __del__(self): + """ close file handle on destruct + :return: + """ + if self._file_handle: + self._file_handle.close() + diff --git a/security/etpro_telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml b/security/etpro_telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml new file mode 100644 index 000000000..8b6da84fb --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml @@ -0,0 +1,58 @@ + + + + + Bearer %%et_telemetry.token%% + + + + botcc.portgrouped.rules + botcc.rules + ciarmy.rules + compromised.rules + drop.rules + dshield.rules + emerging-activex.rules + emerging-attack_response.rules + emerging-chat.rules + emerging-current_events.rules + emerging-deleted.rules + emerging-dns.rules + emerging-dos.rules + emerging-exploit.rules + emerging-ftp.rules + emerging-games.rules + emerging-icmp.rules + emerging-icmp_info.rules + emerging-imap.rules + emerging-inappropriate.rules + emerging-info.rules + emerging-malware.rules + emerging-misc.rules + emerging-mobile_malware.rules + emerging-netbios.rules + emerging-p2p.rules + emerging-policy.rules + emerging-pop3.rules + emerging-rpc.rules + emerging-scada.rules + emerging-scan.rules + emerging-shellcode.rules + emerging-smtp.rules + emerging-snmp.rules + emerging-sql.rules + emerging-telnet.rules + emerging-tftp.rules + emerging-trojan.rules + emerging-user_agents.rules + emerging-voip.rules + emerging-web_client.rules + emerging-web_server.rules + emerging-web_specific_apps.rules + emerging-worm.rules + tor.rules + + + + + diff --git a/security/etpro_telemetry/src/opnsense/service/conf/actions.d/actions_proofpoint.conf b/security/etpro_telemetry/src/opnsense/service/conf/actions.d/actions_proofpoint.conf new file mode 100644 index 000000000..41a3c0b48 --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/service/conf/actions.d/actions_proofpoint.conf @@ -0,0 +1,5 @@ +[et.status] +command:/usr/local/opnsense/scripts/etpro_telemetry/sensor_info.py +parameters: +type:script_output +message:proofpoint ids telemetry status diff --git a/security/etpro_telemetry/src/opnsense/www/img/proofpoint.svg b/security/etpro_telemetry/src/opnsense/www/img/proofpoint.svg new file mode 100644 index 000000000..564d1adfc --- /dev/null +++ b/security/etpro_telemetry/src/opnsense/www/img/proofpoint.svg @@ -0,0 +1,26 @@ + + + + + Asset 1 + + + + + + + + + + + + + + + + + diff --git a/security/etpro_telemetry/src/www/widgets/include/proofpoint_et.inc b/security/etpro_telemetry/src/www/widgets/include/proofpoint_et.inc new file mode 100644 index 000000000..abf77c3bc --- /dev/null +++ b/security/etpro_telemetry/src/www/widgets/include/proofpoint_et.inc @@ -0,0 +1,2 @@ + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+
+ + Deciso B.V. +