+ {{ lang._('After changing settings, please remember to apply them with the button below') }}
+
+
+
+
+
+
+
+
+{{ partial("layout_partials/base_dialog",['fields':formDialogDefaultPolicy,'id':'DialogDefaultPolicy','label':lang._('Edit List')])}}
+{{ partial("layout_partials/base_dialog",['fields':formDialogCustomPolicy,'id':'DialogCustomPolicy','label':lang._('Edit List')])}}
diff --git a/www/OPNProxy/src/opnsense/scripts/OPNProxy/download_cleanse_ut1.py b/www/OPNProxy/src/opnsense/scripts/OPNProxy/download_cleanse_ut1.py
new file mode 100755
index 000000000..75cf4d94e
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/scripts/OPNProxy/download_cleanse_ut1.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# coding=utf-8
+"""
+ Copyright (c) 2023 Ad Schellevis
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+"""
+import argparse
+import os
+import shutil
+import sys
+import tempfile
+import tarfile
+import io
+import requests
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('filename', help='output filename')
+ cmd_args = parser.parse_args()
+
+ req_opts = {
+ 'url': 'http://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz',
+ 'timeout': 120,
+ 'stream': True
+ }
+ try:
+ req = requests.get(**req_opts)
+ except Exception as e:
+ print("unable to download %s" % req_opts['url'])
+ sys.exit(99)
+
+ directory_map = {
+ 'blacklists/agressif': 'blacklists/aggressive',
+ 'blacklists/publicite': 'blacklists/advertisements',
+ 'blacklists/drogue': 'blacklists/drugs',
+ 'blacklists/tricheur': None,
+ 'blacklists/arjel': None,
+ 'blacklists/associations_religieuses': None,
+ 'blacklists/dialer': None,
+ 'blacklists/liste_bu': None,
+ 'blacklists/reaffected': None,
+ 'blacklists/strict_redirector': None,
+ 'blacklists/strong_redirector': None,
+ 'blacklists/sect': None,
+
+ }
+ filenames = ['urls', 'domains', 'README', 'global_usage', 'cc-by-sa-4-0.pdf', 'LICENSE.pdf']
+
+ if 200 <= req.status_code <= 299:
+ with tempfile.NamedTemporaryFile() as tmp_stream:
+ shutil.copyfileobj(req.raw, tmp_stream)
+ tmp_stream.seek(0)
+ tf = tarfile.open(fileobj=tmp_stream)
+ with tarfile.open(cmd_args.filename, "w:gz") as tar_handle:
+ for tf_file in tf.getmembers():
+ filename = os.path.basename(tf_file.name)
+ if tf_file.isreg() and filename in filenames:
+ target = tf_file.name
+ dirname = os.path.dirname(tf_file.name)
+ if dirname in directory_map:
+ if directory_map[dirname] is None:
+ continue
+ else:
+ target = "%s/%s" % (directory_map[dirname], filename)
+ fhandle = tf.extractfile(tf_file)
+ info = tarfile.TarInfo(target)
+ fhandle.seek(0, io.SEEK_END)
+ info.size = fhandle.tell()
+ fhandle.seek(0, io.SEEK_SET)
+ tar_handle.addfile(info, fhandle)
+
+ tar_handle.close()
diff --git a/www/OPNProxy/src/opnsense/scripts/OPNProxy/lib/__init__.py b/www/OPNProxy/src/opnsense/scripts/OPNProxy/lib/__init__.py
new file mode 100755
index 000000000..95988b247
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/scripts/OPNProxy/lib/__init__.py
@@ -0,0 +1,135 @@
+"""
+ Copyright (c) 2023 Ad Schellevis
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+"""
+import copy
+import tarfile
+import os
+import stat
+import syslog
+import time
+import requests
+from configparser import ConfigParser
+
+
+class Policy:
+ def __init__(self, policy_filename):
+ self._policy_config = policy_filename
+ self._domain_entries = dict()
+ self._policy_settings = dict()
+ self._tf = None
+ self.load()
+
+ def load(self):
+ """ load policy database
+ :return:
+ """
+ self._domain_entries = dict()
+ self._policy_settings = dict()
+ # collect all policies per domain, so we can safely overwrite existing content when it exists
+ cnf = ConfigParser()
+ cnf.read(self._policy_config)
+ if cnf.has_section('source'):
+ blocklist_filename = cnf.get('source', 'blocklist')
+ if cnf.has_option('source', 'blocklist_download_uri'):
+ blocklist_ttl = cnf.getint('source', 'blocklist_ttl')
+ if not os.path.isfile(blocklist_filename) or \
+ time.time() - os.stat(blocklist_filename)[stat.ST_MTIME] > blocklist_ttl:
+ try:
+ response = requests.get(cnf.get('source', 'blocklist_download_uri'), stream=True)
+ response.raise_for_status()
+ with open(blocklist_filename, 'wb') as handle:
+ for block in response.iter_content(1024):
+ handle.write(block)
+ except requests.exceptions.RequestException as e:
+ # we are unable to download a new blocklist, if a previous version still exists keep using that
+ syslog.syslog(syslog.LOG_ERR, 'unable to download new blocklist (%s)' % e)
+
+ if os.path.isfile(blocklist_filename) and tarfile.is_tarfile(blocklist_filename):
+ self._tf = tarfile.open(fileobj=open(blocklist_filename, "rb"))
+ else:
+ syslog.syslog(syslog.LOG_ERR, 'default policy rules not available (%s missing)' % blocklist_filename)
+
+ for section in cnf.sections():
+ if cnf.has_option(section, 'policy_type') and cnf.has_option(section, 'content'):
+ self._policy_settings[section] = {
+ 'action': cnf.get(section, 'action'),
+ 'id': section.split('_', 1)[-1],
+ 'applies_on': cnf.get(section, 'applies_on').split(','),
+ 'source_net': cnf.get(section, 'source_net').split(','),
+ 'policy_type': cnf.get(section, 'policy_type'),
+ 'description': cnf.get(section, 'description')
+ }
+ ittr_method = self._itr_default if cnf.get(section, 'policy_type') == "default" else self._itr_custom
+ split_char = ',' if cnf.get(section, 'policy_type') == "default" else '\n'
+ for is_wildcard, item in ittr_method(cnf.get(section, 'content').split(split_char)):
+ parts = item.split('/', 1)
+ domain = parts[0]
+ if domain not in self._domain_entries:
+ self._domain_entries[domain] = list()
+ self._domain_entries[domain].append([
+ section,
+ "/%s" % parts[1] if len(parts) > 1 else "/",
+ is_wildcard
+ ])
+
+ def _itr_default(self, items: list):
+ if self._tf:
+ for tf_file in self._tf.getmembers():
+ if tf_file.isreg():
+ fhandle = self._tf.extractfile(tf_file)
+ if tf_file.name.count('/') >= 2 and tf_file.name.split('/')[-2] in items:
+ filename = os.path.basename(tf_file.name)
+ if filename in ['urls', 'domains']:
+ for line in fhandle.read().decode().split('\n'):
+ line = line.strip()
+ if line:
+ # assume domains are wildcards (e.g. youtube.com --> .youtube.com)
+ yield line.find('/') == -1, line
+
+ @staticmethod
+ def _itr_custom(items: list):
+ for line in items:
+ if line.startswith('.') or line.startswith('*'):
+ # wildcard search, e.g. matches all subdomains of given domain, where * is the absolute toplevel (root)
+ yield True, line.lstrip('.')
+ else:
+ yield False, line
+
+ def __iter__(self):
+ for domain in self._domain_entries:
+ # prepare domain policies
+ policy = {
+ 'domain': domain,
+ 'items': []
+ }
+ for entry in self._domain_entries[domain]:
+ politem = copy.deepcopy(self._policy_settings[entry[0]])
+ politem['path'] = entry[1]
+ politem['wildcard'] = entry[2]
+ policy['items'].append(politem)
+ yield policy
+
+ def exists(self, domain):
+ return domain.split(':')[-1] in self._domain_entries
diff --git a/www/OPNProxy/src/opnsense/scripts/OPNProxy/policies_to_redis_proto.py b/www/OPNProxy/src/opnsense/scripts/OPNProxy/policies_to_redis_proto.py
new file mode 100755
index 000000000..0e4599d6c
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/scripts/OPNProxy/policies_to_redis_proto.py
@@ -0,0 +1,100 @@
+#!/usr/local/bin/python3
+# -*- coding: utf-8 -*-
+"""
+ Copyright (c) 2023 Ad Schellevis
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+"""
+import argparse
+import fcntl
+import time
+import ujson
+from lib import Policy
+import redis
+
+
+def redis_proto_parser(*args):
+ """
+ https://redis.io/topics/protocol
+ :return:
+ """
+ response = ["*%d\r\n$%d\r\n%s\r\n" % (len(args), len(args[0]), args[0])]
+ for item in args[1:]:
+ response.append("$%d\r\n%s\r\n" % (len(item), item))
+ return "".join(response)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--redis_host',
+ help='redis hostname to read keys from (default: 127.0.0.1)',
+ default='127.0.0.1'
+ )
+ parser.add_argument(
+ '--redis_port',
+ help='redis port number (default: 6379)',
+ type=int,
+ default=6379
+ )
+ parser.add_argument(
+ '--proxy_policies',
+ help='proxy policies configuration file',
+ default='/usr/local/etc/squid/proxy_policies.conf'
+ )
+ parser.add_argument('--output', help='output filename', default='/dev/stdout')
+
+ cmd_args = parser.parse_args()
+
+ try:
+ lck = open('/tmp/policies_to_redis_proto.LCK', 'w+')
+ fcntl.flock(lck, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except IOError:
+ # already running, exit status 99
+ sys.exit(99)
+
+ policy = Policy(cmd_args.proxy_policies)
+
+ # fetch current domain keys from redis
+ try:
+ existing_domains = redis.StrictRedis(
+ host=cmd_args.redis_host, port=cmd_args.redis_port, db=0, decode_responses=True
+ ).keys('domain:*')
+ except (redis.exceptions.ConnectionError, redis.exceptions.BusyLoadingError) as e:
+ existing_domains = list()
+
+ with open(cmd_args.output, 'w') as output_stream:
+ statistics = {'domains': 0, 'policies': 0, 'generated': time.time()}
+ # generate delete statements for non existing keys
+ for domain in existing_domains:
+ domain = domain.split(':')[-1]
+ if not policy.exists(domain):
+ output_stream.write(redis_proto_parser("DEL", "domain:%s" % domain))
+
+ # generate set statements for new data (upsert)
+ for item in policy:
+ statistics['domains'] += 1
+ statistics['policies'] += len(item['items'])
+ output_stream.write(redis_proto_parser("SET", "domain:%s" % item['domain'], ujson.dumps(item)))
+
+ output_stream.write(redis_proto_parser("SET", "domain_statistics", ujson.dumps(statistics)))
diff --git a/www/OPNProxy/src/opnsense/scripts/OPNProxy/redis_sync_users.py b/www/OPNProxy/src/opnsense/scripts/OPNProxy/redis_sync_users.py
new file mode 100755
index 000000000..3e22e0909
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/scripts/OPNProxy/redis_sync_users.py
@@ -0,0 +1,76 @@
+#!/usr/local/bin/python3
+# -*- coding: utf-8 -*-
+"""
+ Copyright (c) 2023 Ad Schellevis
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+"""
+import argparse
+import fcntl
+import sys
+import syslog
+import redis
+import ujson
+import xml.etree.ElementTree as ET
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--redis_host', help='redis hostname (default: 127.0.0.1)', default='127.0.0.1')
+ parser.add_argument('--redis_port', help='redis port number (default: 6379)', type=int, default=6379)
+ parser.add_argument('username', help='optional username', nargs='?', default=None)
+ args = parser.parse_args()
+
+ # wait for other redis_sync_users sync events to complete
+ lck = open('/tmp/redis_sync_users.LCK', 'w+')
+ fcntl.flock(lck, fcntl.LOCK_EX)
+
+ redisdb = redis.Redis(host=args.redis_host, port=args.redis_port, db=0)
+
+ # ideally we would flush config data using the template system first, but since user settings may change
+ # more rappidly we opt to read the raw source here.
+ try:
+ tree = ET.parse('/conf/config.xml')
+ xmlroot = tree.getroot()
+ except (FileNotFoundError, ET.ParseError):
+ syslog.syslog(syslog.LOG_ERR, 'enable to open /conf/config.xml')
+ sys.exit(1)
+
+ # merge group membership into user object and flush to redis
+ membership = dict()
+ for group in xmlroot.findall('./system/group'):
+ for member in group.findall('member'):
+ if member.text not in membership:
+ membership[member.text] = list()
+ membership[member.text].append(group.findtext('name'))
+
+ for user in xmlroot.findall('./system/user'):
+ if args.username is None or args.username == user.findtext('name'):
+ user_object = dict()
+ user_object['uid'] = user.findtext('name')
+ user_object['id'] = user.findtext('uid')
+ user_object['applies_on'] = ["u:%s" % user.findtext('name')]
+ if user_object['id'] in membership:
+ for group in membership[user_object['id']]:
+ user_object['applies_on'].append("g:%s" % group)
+ redisdb.set('user:%s' % user_object['uid'], ujson.dumps(user_object))
diff --git a/www/OPNProxy/src/opnsense/scripts/OPNProxy/squid_acl_helper.py b/www/OPNProxy/src/opnsense/scripts/OPNProxy/squid_acl_helper.py
new file mode 100755
index 000000000..17d4d87fa
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/scripts/OPNProxy/squid_acl_helper.py
@@ -0,0 +1,217 @@
+#!/usr/local/bin/python3
+# -*- coding: utf-8 -*-
+"""
+ Copyright (c) 2023 Ad Schellevis
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+"""
+import argparse
+import decimal
+import sys
+import syslog
+import traceback
+from urllib.parse import urlparse
+import redis
+import ujson
+import ipaddress
+
+
+class RedisAuth:
+ def __init__(self, host, port):
+ self._redis = redis.Redis(host=host, port=port, db=0)
+
+ def domain_policy_iterator(self, r_fqdn):
+ """ traverse domain policies
+ :param r_fqdn: fqdn
+ :return:
+ """
+ try:
+ tmp = self._redis.get("domain:%s" % r_fqdn)
+ if tmp:
+ domain_policy = ujson.loads(tmp.decode())
+ else:
+ return
+ except Exception as e:
+ # connectivity or parse issue, log and return
+ syslog.syslog(syslog.LOG_ERR, traceback.format_exc().replace('\n', ' '))
+ return
+
+ if type(domain_policy.get('items', None)) is list:
+ for policy in domain_policy['items']:
+ if type(policy) is dict:
+ for fieldname in ['id', 'path', 'wildcard', 'action', 'applies_on', 'source_net']:
+ if fieldname not in policy:
+ policy[fieldname] = None
+ yield policy
+
+ def get_user(self, uid):
+ if uid == "-":
+ return {'applies_on': set('-')}
+ try:
+ tmp = self._redis.get("user:%s" % uid)
+ if not tmp:
+ return None
+ udata = ujson.loads(tmp.decode())
+ # cleanse data
+ udata['applies_on'] = set(udata['applies_on']) if 'applies_on' in udata else set()
+ except Exception:
+ syslog.syslog(syslog.LOG_ERR, traceback.format_exc().replace('\n', ' '))
+ return None
+
+ return udata
+
+def in_network(src, networks):
+ if networks is None or type(networks) is not list or src == '-':
+ return True
+ try:
+ src_net = ipaddress.ip_network(src)
+ except ValueError:
+ syslog.syslog(syslog.LOG_ERR, traceback.format_exc().replace('\n', ' '))
+ return False
+ for network in networks:
+ try:
+ if src_net.overlaps(ipaddress.ip_network(network)):
+ return True
+ except ValueError:
+ syslog.syslog(syslog.LOG_ERR, traceback.format_exc().replace('\n', ' '))
+
+ return False
+
+def match_policy(acl, ident, src, method, uri, sslurlonly=False):
+ # default response, invalid user
+ match_res = {'message': "ERR message=\"no (valid) IDENT %s\"\n" % ident}
+ if uri.find('://') == -1:
+ base_domain = uri.split(':')[0]
+ request_path = '/'
+ else:
+ uri_parsed = urlparse(uri)
+ base_domain = uri_parsed.netloc.split(':')[0]
+ request_path = uri_parsed.path if uri_parsed.path else '/'
+
+ syslog.syslog(
+ syslog.LOG_NOTICE,
+ "ACL-REQ |%s| |%s| |%s| |%s| |%s| %s" % (acl, ident, src, method, uri, 'SNI only' if sslurlonly else '')
+ )
+ fqdn = base_domain
+ user_data = redis_auth.get_user(ident)
+ if user_data:
+ acl_decisions = dict()
+ # traverse domain upwards until either a policy is found or no matches are possible
+ # matches are prioritized on best path match and accept (higher) or deny.
+ while len(acl_decisions) == 0:
+ for this_policy in redis_auth.domain_policy_iterator(fqdn):
+ is_parent = base_domain != fqdn
+ match_parent = this_policy['path'] == '/' and is_parent and this_policy['wildcard']
+ match_main = request_path.find(this_policy['path']) == 0 and not is_parent
+ if (match_parent or match_main) and set(this_policy['applies_on']) & user_data['applies_on']:
+ if not in_network(src, this_policy['source_net']):
+ continue
+ tp = 0 if this_policy['action'] == 'deny' else 1
+ this_prio = decimal.Decimal("%d.%d" % (len(this_policy['path']), tp))
+ acl_decisions[this_prio] = this_policy
+ acl_decisions[this_prio]['domain'] = fqdn
+
+ if fqdn.find('.') == -1:
+ if fqdn == '*':
+ break
+ else:
+ # top level wildcard (add extra level)
+ fqdn = '*'
+ else:
+ fqdn = fqdn.split('.', maxsplit=1)[1]
+
+ match_res['user'] = user_data
+ match_res['user']['applies_on'] = list(user_data['applies_on'])
+
+ if not sslurlonly and method.lower() == 'connect':
+ # skip connect when full ssl bump is enabled
+ match_res['policy'] = {'action': 'allow', 'policy_type': 'fallback'}
+ match_res['message'] = "OK user=\"%s\"\n" % ident
+ elif len(acl_decisions) > 0:
+ acl_decision = acl_decisions[sorted(acl_decisions.keys(), reverse=True)[0]]
+ match_res['policy'] = acl_decision
+ if match_res['policy']['action'] == 'deny':
+ match_res['message'] = "ERR message=\"reason:%s policy_type:%s\" user=\"%s\"\n" % (
+ acl_decision['id'], acl_decision['policy_type'], ident
+ )
+ else:
+ match_res['message'] = "OK message=\"whitelisted %s\" user=\"%s\"\n" % (acl_decision['id'], ident)
+ elif ident != '-':
+ # network only authentication needs an explicit policy, user-based allows by default
+ match_res['policy'] = {'action': 'allow', 'policy_type': 'fallback'}
+ match_res['message'] = "OK user=\"%s\"\n" % ident
+
+ return match_res
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--test_user', help='test mode (singleshot), username')
+ parser.add_argument('--test_uri', help='test mode (singleshot), uri')
+ parser.add_argument('--test_src', help='test mode (singleshot), source address', default='-')
+ parser.add_argument('--redis_host', help='redis hostname (default: 127.0.0.1)', default='127.0.0.1')
+ parser.add_argument('--redis_port', help='redis port number (default: 6379)', type=int, default=6379)
+ parser.add_argument('--sslurlonly', help='Log SNI information only enabled', action="store_true", default=False)
+ parser.add_argument(
+ '--no_ident',
+ help='Do not expect iden/user information in the message line',
+ action="store_true",
+ default=False
+ )
+
+ args = parser.parse_args()
+ syslog.openlog('squid', facility=syslog.LOG_LOCAL2)
+ redis_auth = RedisAuth(args.redis_host, args.redis_port)
+ if args.test_user and args.test_uri:
+ # test mode, dump raw json object to stdout
+ result = match_policy(acl='-', ident=args.test_user, src=args.test_src, method='-', uri=args.test_uri)
+ print (ujson.dumps(result))
+ else:
+ # squid worker mode
+ while True:
+ try:
+ # accept messages like:
+ # my_ext_acl user 127.0.0.2 GET https://requested.domain/path/
+ line = sys.stdin.readline().strip()
+ if line == "":
+ sys.exit()
+ if line:
+ try:
+ acl_parts = line.split()
+ except ValueError:
+ sys.stdout.write("ERR message=\"missing input\"\n")
+ break
+ offset = -1 if args.no_ident else 0
+ result = match_policy(
+ acl=acl_parts[0],
+ ident='-' if args.no_ident else acl_parts[1],
+ src=acl_parts[2+offset],
+ method=acl_parts[3+offset],
+ uri=acl_parts[4+offset],
+ sslurlonly=args.sslurlonly
+ )
+ sys.stdout.write(result['message'])
+
+ sys.stdout.flush()
+ except IOError:
+ pass
diff --git a/www/OPNProxy/src/opnsense/service/conf/actions.d/actions_opnproxy.conf b/www/OPNProxy/src/opnsense/service/conf/actions.d/actions_opnproxy.conf
new file mode 100644
index 000000000..bafa0f699
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/service/conf/actions.d/actions_opnproxy.conf
@@ -0,0 +1,21 @@
+[apply_policies]
+command:
+ /usr/local/opnsense/scripts/OPNProxy/policies_to_redis_proto.py | redis-cli --pipe &&
+ /usr/local/sbin/squid -k reconfigure
+parameters:
+type:script
+message:download proxy policies and apply to redisdb
+description:OPNProxy apply policies
+
+
+[sync_users]
+command: /usr/local/opnsense/scripts/OPNProxy/redis_sync_users.py
+parameters:
+type:script
+message:synchronise proxy users
+
+[user.test]
+command: /usr/local/opnsense/scripts/OPNProxy/squid_acl_helper.py
+parameters: --test_user %s --test_uri %s --test_src %s
+type:script_output
+message:test user login
diff --git a/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/+TARGETS b/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/+TARGETS
new file mode 100644
index 000000000..b9419e8ae
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/+TARGETS
@@ -0,0 +1,2 @@
+proxy_policies.conf:/usr/local/etc/squid/proxy_policies.conf
+10-opnproxy-ext.auth.conf:/usr/local/etc/squid/auth/10-opnproxy-ext.auth.conf
diff --git a/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/10-opnproxy-ext.auth.conf b/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/10-opnproxy-ext.auth.conf
new file mode 100644
index 000000000..408895ad8
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/10-opnproxy-ext.auth.conf
@@ -0,0 +1,43 @@
+external_acl_type ext_opnproxy_helper_net ttl=30 negative_ttl=5 %ACL %SRC %METHOD %URI /usr/local/opnsense/scripts/OPNProxy/squid_acl_helper.py --no_ident {% if not helpers.empty('OPNsense.proxy.forward.sslurlonly') %} --sslurlonly {% endif %}
+
+acl opnproxy_ext_acl_net external ext_opnproxy_helper_net
+http_access allow opnproxy_ext_acl_net
+
+{% if not helpers.empty('OPNsense.proxy.forward.authentication.method') %}
+# Login based authentication
+external_acl_type ext_opnproxy_helper_usr ttl=30 negative_ttl=5 %ACL %LOGIN %SRC %METHOD %URI /usr/local/opnsense/scripts/OPNProxy/squid_acl_helper.py {% if not helpers.empty('OPNsense.proxy.forward.sslurlonly') %} --sslurlonly {% endif %}
+
+acl opnproxy_ext_acl_usr external ext_opnproxy_helper_usr
+http_access allow opnproxy_ext_acl_usr
+{% endif %}
+
+
+{% if not helpers.empty('OPNsense.proxy.forward.icap.enable') %}
+{% if not helpers.empty('OPNsense.proxy.forward.icap.ResponseURL') %}
+adaptation_access response_mod allow opnproxy_ext_acl_net
+{% if not helpers.empty('OPNsense.proxy.forward.authentication.method') %}
+adaptation_access response_mod allow opnproxy_ext_acl_usr
+{% endif %}
+{% endif %}
+{% if not helpers.empty('OPNsense.proxy.forward.icap.RequestURL') %}
+adaptation_access request_mod allow opnproxy_ext_acl_net
+{% if not helpers.empty('OPNsense.proxy.forward.authentication.method') %}
+adaptation_access request_mod allow opnproxy_ext_acl_usr
+{% endif %}
+{% endif %}
+{% endif %}
+
+{% if not helpers.empty('OPNsense.proxy.forward.authentication.method') %}
+# explicit disable default allow authenticated users clause
+http_access deny local_auth all
+{% if not helpers.empty('OPNsense.proxy.forward.icap.enable') %}
+{% if not helpers.empty('OPNsense.proxy.forward.icap.ResponseURL') %}
+adaptation_access response_mod deny local_auth
+{% endif %}
+{% if not helpers.empty('OPNsense.proxy.forward.icap.RequestURL') %}
+adaptation_access request_mod deny local_auth
+{% endif %}
+{% endif %}
+{% else %}
+http_access deny localnet
+{% endif %}
diff --git a/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/proxy_policies.conf b/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/proxy_policies.conf
new file mode 100644
index 000000000..ce330f22d
--- /dev/null
+++ b/www/OPNProxy/src/opnsense/service/templates/Deciso/Proxy/proxy_policies.conf
@@ -0,0 +1,30 @@
+{% for policy in helpers.toList('Deciso.Proxy.ACL.policies.policy') %}
+{% if policy.enabled|default('0') == '1' %}
+[policy_{{ policy['@uuid'] }}]
+policy_type=default
+description={{ policy.description }}
+content={{ policy.content }}
+applies_on={{ policy.applies_on|default('-') }}
+source_net={{ policy.source_net }}
+action={{ policy.action }}
+{% endif %}
+
+{% endfor %}
+
+{% for policy in helpers.toList('Deciso.Proxy.ACL.custom_policies.policy') %}
+{% if policy.enabled|default('0') == '1' %}
+[policy_{{ policy['@uuid'] }}]
+policy_type=custom
+description={{ policy.description }}
+content={{ policy.content.replace('\n', '\n\t') }}
+applies_on={{ policy.applies_on|default('-') }}
+source_net={{ policy.source_net }}
+action={{ policy.action }}
+{% endif %}
+
+{% endfor %}
+
+[source]
+blocklist=/usr/local/opnsense/data/proxy/blocklists.tar.gz
+blocklist_download_uri=https://rulesets.opnsense.org/proxy/blocklists.tar.gz
+blocklist_ttl=86300