drop python 2 support (closes #267)

.gitlab-ci.yml

@@ -8,7 +8,7 @@ stages:
 
 test:lint:
   stage: test
-  image: python:2.7
+  image: python:3.6
   script:
   - pip install flake8 pylint
   - make flake8
@@ -17,17 +17,6 @@ test:lint:
   except:
   - tags
 
-test:py2.7:
-  stage: test
-  image: python:2.7
-  script:
-  - pip install tox
-  - tox -e py27
-  tags:
-  - docker
-  except:
-  - tags
-
 test:py3.6:
   stage: test
   image: python:3.6
@@ -39,22 +28,6 @@ test:py3.6:
   except:
   - tags
 
-build:py2:
-  stage: build
-  image: python:2.7
-  script:
-  - /bin/bash tests/run_build.sh
-  tags:
-  - build
-  only:
-  - master@ziirish/burp-ui
-  - demo@ziirish/burp-ui
-  artifacts:
-    paths:
-    - dist/
-    - meta/
-    expire_in: 2 mos
-
 build:py3:
   stage: build
   image: python:3.6
@@ -64,6 +37,7 @@ build:py3:
   - build
   only:
   - master@ziirish/burp-ui
+  - demo@ziirish/burp-ui
   artifacts:
     paths:
     - dist/

burpui/__init__.py

@@ -9,17 +9,12 @@ jQuery/Bootstrap
 
 .. moduleauthor:: Ziirish <hi+burpui@ziirish.me>
 """
-import sys
 import warnings
 
 from .app import init
 
 warnings.simplefilter('always', RuntimeWarning)
 
-if sys.version_info < (3, 0):  # pragma: no cover
-    reload(sys)
-    sys.setdefaultencoding('utf-8')
-
 
 # backward compatibility
 create_app = init

burpui/_compat.py

@@ -8,55 +8,12 @@
 
 """
 import re
-import sys
 
-try:
-    import cPickle as pickle  # noqa
-except ImportError:
-    import pickle  # noqa
+import pickle  # noqa
 
-if sys.version_info[0] >= 3:
-    PY3 = True
-    from urllib.parse import unquote, quote, urlparse, urljoin  # noqa
-    text_type = str
-    string_types = (str,)
-
-    def iterkeys(d, *args, **kwargs):
-        return iter(d.keys(*args, **kwargs))
-
-    def itervalues(d, *args, **kwargs):
-        return iter(d.values(*args, **kwargs))
-
-    def iteritems(d, *args, **kwargs):
-        return iter(d.items(*args, **kwargs))
-
-    def iterlists(d, *args, **kwargs):
-        return iter(d.lists(*args, **kwargs))
-
-    def iterlistvalues(d, *args, **kwargs):
-        return iter(d.listvalues(*args, **kwargs))
-
-else:
-    PY3 = False
-    from urllib import unquote, quote  # noqa
-    from urlparse import urlparse, urljoin  # noqa
-    text_type = unicode
-    string_types = (str, unicode)
-
-    def iterkeys(d, *args, **kwargs):
-        return d.iterkeys(*args, **kwargs)
-
-    def itervalues(d, *args, **kwargs):
-        return d.itervalues(*args, **kwargs)
-
-    def iteritems(d, *args, **kwargs):
-        return d.iteritems(*args, **kwargs)
-
-    def iterlists(d, *args, **kwargs):
-        return d.iterlists(*args, **kwargs)
-
-    def iterlistvalues(d, *args, **kwargs):
-        return d.iterlistvalues(*args, **kwargs)
+from urllib.parse import unquote, quote, urlparse, urljoin  # noqa
+text_type = str
+string_types = (str,)
 
 
 def to_bytes(text):
@@ -72,7 +29,7 @@ def to_unicode(input_bytes, encoding='utf-8'):
         input_bytes = input_bytes.decode(encoding)
     elif re.match(r'\\u[0-9a-f]{4}', input_bytes):
         input_bytes = input_bytes.decode('unicode-escape')
-    return input_bytes or u''
+    return input_bytes or ''
 
 
 # maps module name -> attribute name -> original item

burpui/agent.py

@@ -34,13 +34,18 @@ try:
 except ImportError:
     USE_SENDFILE = False
 
-G_PORT = 10000
-G_BIND = u'::'
-G_SSL = False
-G_VERSION = 2
-G_SSLCERT = u''
-G_SSLKEY = u''
-G_PASSWORD = u'password'
+
+BUI_DEFAULTS = {
+    'Global': {
+        'port': 10000,
+        'bind': '::',
+        'ssl': False,
+        'sslcert': '',
+        'sslkey': '',
+        'backend': 'burp2',
+        'password': 'password',
+    },
+}
 
 DISCLOSURE = 5
 
@@ -55,21 +60,24 @@ class BurpHandler(BUIbackend):
     foreign = BUIbackend.__abstractmethods__
     BUIbackend.__abstractmethods__ = frozenset()
 
-    def __init__(self, vers=2, logger=None, conf=None):
-        self.vers = vers
+    def __init__(self, backend='burp2', logger=None, conf=None):
+        self.backend = backend
         self.logger = logger
 
         top = __name__
-        if '.' in top:
-            top = top.split('.')[0]
-        module = '{0}.misc.backend.burp{1}'.format(top, self.vers)
+        if '.' in self.backend:
+            module = self.backend
+        else:
+            if '.' in top:
+                top = top.split('.')[0]
+            module = '{0}.misc.backend.{1}'.format(top, self.backend)
         try:
             sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
             mod = __import__(module, fromlist=['Burp'])
             Client = mod.Burp
             self.backend = Client(conf=conf)
         except Exception as e:
-            self.logger.error('{}\n\nFailed loading backend for Burp version {}: {}'.format(traceback.format_exc(), self.vers, str(e)))
+            self.logger.error('{}\n\nFailed loading backend {}: {}'.format(traceback.format_exc(), self.backend, str(e)))
             sys.exit(2)
 
     def __getattribute__(self, name):
@@ -90,17 +98,6 @@ class BurpHandler(BUIbackend):
 
 class BUIAgent(BUIbackend, BUIlogging):
     BUIbackend.__abstractmethods__ = frozenset()
-    defaults = {
-        'Global': {
-            'port': G_PORT,
-            'bind': G_BIND,
-            'ssl': G_SSL,
-            'sslcert': G_SSLCERT,
-            'sslkey': G_SSLKEY,
-            'version': G_VERSION,
-            'password': G_PASSWORD,
-        },
-    }
 
     def __init__(self, conf=None, level=0, logfile=None, debug=False):
         self.debug = debug
@@ -143,18 +140,18 @@ class BUIAgent(BUIbackend, BUIlogging):
 
         # Raise exception if errors are encountered during parsing
         self.conf = config
-        self.conf.parse(conf, True, self.defaults)
+        self.conf.parse(conf, True, BUI_DEFAULTS)
         self.conf.default_section('Global')
         self.port = self.conf.safe_get('port', 'integer')
         self.bind = self.conf.safe_get('bind')
-        self.vers = self.conf.safe_get('version', 'integer')
+        self.backend = self.conf.safe_get('backend')
         self.ssl = self.conf.safe_get('ssl', 'boolean')
         self.sslcert = self.conf.safe_get('sslcert')
         self.sslkey = self.conf.safe_get('sslkey')
         self.password = self.conf.safe_get('password')
         self.conf.setdefault('BUI_AGENT', True)
 
-        self.client = BurpHandler(self.vers, self.logger, self.conf)
+        self.client = BurpHandler(self.backend, self.logger, self.conf)
         pool = Pool(10000)
         if not self.ssl:
             self.server = StreamServer((self.bind, self.port), self.handle, spawn=pool)
@@ -277,7 +274,7 @@ class BUIAgent(BUIbackend, BUIlogging):
                                 key = to_bytes(key)
                                 bytes_pickles = pickles
                                 digest = hmac.new(key, bytes_pickles, hashlib.sha1).hexdigest()
-                                if digest != j['digest']:
+                                if not hmac.compare_digest(digest, j['digest']):
                                     raise BUIserverException('Integrity check failed: {} != {}'.format(digest, j['digest']))
                                 # We need to replace the burpui datastructure
                                 # module by our own since it's the same but

burpui/api/clients.py

@@ -308,7 +308,7 @@ class ClientsReport(Resource):
 
     def _check_acl(self, server):
         # Manage ACL
-        if (not bui.standalone and not current_user.is_anonymous and
+        if (not bui.config['STANDALONE'] and not current_user.is_anonymous and
                 (not current_user.acl.is_admin() and
                  not current_user.acl.is_server_allowed(server))):
             self.abort(403, 'Sorry, you don\'t have any rights on this server')
@@ -392,7 +392,7 @@ class ClientsReport(Resource):
             if not current_user.is_anonymous and not current_user.acl.is_admin():
                 clients = [x for x in clients if current_user.acl.is_client_allowed(x['name'], server)]
             return bui.client.get_clients_report(clients, server)
-        if bui.standalone:
+        if bui.config['STANDALONE']:
             ret = res
         else:
             ret = res.get(server, {})
@@ -481,7 +481,7 @@ class ClientsStats(Resource):
 
         server = server or self.parser.parse_args()['serverName']
         try:
-            if (not bui.standalone and not current_user.is_anonymous and
+            if (not bui.config['STANDALONE'] and not current_user.is_anonymous and
                     (not current_user.acl.is_admin() and
                      not current_user.acl.is_server_allowed(server))):
                 self.abort(403, 'Sorry, you don\'t have any rights on this server')
@@ -603,7 +603,7 @@ class AllClients(Resource):
                 ret = [{'name': x, 'agent': server} for x in clients]
             return ret
 
-        if bui.standalone:
+        if bui.config['STANDALONE']:
             try:
                 clients = [x['name'] for x in bui.client.get_all_clients()]
             except BUIserverException:

burpui/api/custom/resource.py

@@ -13,10 +13,6 @@ import json
 
 from flask_restplus import Resource as ResourcePlus
 from flask_restplus.errors import abort
-from ..._compat import PY3
-
-if PY3:
-    basestring = str
 
 
 class Resource(ResourcePlus):
@@ -33,7 +29,7 @@ class Resource(ResourcePlus):
 
         See: :func:`~flask_restplus.errors.abort`
         """
-        if message and not isinstance(message, basestring):
+        if message and not isinstance(message, str):
             try:
                 message = json.dumps(message)  # pragma: no cover
             except:

burpui/api/misc.py

@@ -622,7 +622,7 @@ class History(Resource):
                 ret.append(feed)
             return ret
 
-        if bui.standalone:
+        if bui.config['STANDALONE']:
             if data:
                 clients_list = data.keys()
             else:
@@ -788,7 +788,7 @@ class History(Resource):
         events = []
         filtered = False
         if data:
-            if bui.standalone:
+            if bui.config['STANDALONE']:
                 events = data.get(client, [None])
             else:
                 events = data.get(server, {}).get(client, [None])

burpui/api/servers.py

@@ -65,7 +65,7 @@ class ServersStats(Resource):
         r = []
         check = False
 
-        if bui.standalone:
+        if bui.config['STANDALONE']:
             return r
 
         if not current_user.is_anonymous and not current_user.acl.is_admin():

burpui/app.py

@@ -421,7 +421,7 @@ def create_app(conf=None, verbose=0, logfile=None, **kwargs):
         g.now = round(time.time())
         g.date_format = session.get('dateFormat', 'llll')
         # make sure to store secure cookie if required
-        if app.scookie:
+        if app.config['BUI_SCOOKIE']:
             criteria = [
                 request.is_secure,
                 request.headers.get('X-Forwarded-Proto', 'http') == 'https'

burpui/cli.py

@@ -317,7 +317,7 @@ def compile_translation():
               help='Dry mode. Do not edit the files but display changes')
 def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
     """Setup burp client for burp-ui."""
-    if app.vers != 2:
+    if app.config['BACKEND'] != 'burp2':
         click.echo(
             click.style(
                 'Sorry, you can only setup the Burp 2 client',
@@ -327,7 +327,7 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
         )
         sys.exit(1)
 
-    if not app.standalone:
+    if not app.config['STANDALONE']:
         click.echo(
             click.style(
                 'Sorry, only the standalone mode is supported',
@@ -782,7 +782,7 @@ password = abcdefgh
               help='Show you some tips')
 def diag(client, host, tips):
     """Check Burp-UI is correctly setup."""
-    if app.vers != 2:
+    if app.config['BACKEND'] != 'burp2':
         click.echo(
             click.style(
                 'Sorry, you can only setup the Burp 2 client',
@@ -792,7 +792,7 @@ def diag(client, host, tips):
         )
         sys.exit(1)
 
-    if not app.standalone:
+    if not app.config['STANDALONE']:
         click.echo(
             click.style(
                 'Sorry, only the standalone mode is supported',
@@ -1124,15 +1124,13 @@ def sysinfo(verbose, load):
         except Exception as e:
             msg = str(e)
 
-    backend_version = app.vers
-    if not app.standalone:
-        backend_version = 'multi'
+    backend_version = app.config['BACKEND']
 
     colors = {
         'True': 'green',
         'False': 'red',
     }
-    embedded_ws = str(app.websocket)
+    embedded_ws = str(app.config['WITH_WS'])
     available_ws = str(WS_AVAILABLE)
 
     click.echo('Python version:      {}.{}.{}'.format(sys.version_info[0], sys.version_info[1], sys.version_info[2]))
@@ -1140,13 +1138,13 @@ def sysinfo(verbose, load):
     click.echo('OS:                  {}:{} ({})'.format(platform.system(), platform.release(), os.name))
     if platform.system() == 'Linux':
         click.echo('Distribution:        {} {} {}'.format(*platform.dist()))
-    click.echo('Single mode:         {}'.format(app.standalone))
+    click.echo('Single mode:         {}'.format(app.config['STANDALONE']))
     click.echo('Backend version:     {}'.format(backend_version))
     click.echo('WebSocket embedded:  {}'.format(click.style(embedded_ws, fg=colors[embedded_ws])))
     click.echo('WebSocket available: {}'.format(click.style(available_ws, colors[available_ws])))
     click.echo('Config file:         {}'.format(app.config.conffile))
     if load:
-        if not app.standalone and not msg:
+        if not app.config['STANDALONE'] and not msg:
             click.echo('Agents:')
             for agent, obj in iteritems(app.client.servers):
                 client_version = server_version = 'unknown'

burpui/datastructures.py

@@ -12,7 +12,7 @@
 from copy import deepcopy
 from itertools import repeat
 
-from ._compat import iterkeys, itervalues, iteritems, iterlists, PY3
+from six import iterkeys, itervalues, iteritems, iterlists
 
 
 def is_immutable(self):
@@ -39,36 +39,7 @@ def iter_multi_items(mapping):
 
 
 def native_itermethods(names):
-    if PY3:
-        return lambda x: x
-
-    def setviewmethod(cls, name):
-        viewmethod_name = 'view%s' % name
-
-        def viewmethod(self, *a, **kw):
-            return ViewItems(self, name, 'view_%s' % name, *a, **kw)
-
-        viewmethod.__doc__ = \
-            '"""`%s()` object providing a view on %s"""' % (viewmethod_name, name)
-        setattr(cls, viewmethod_name, viewmethod)
-
-    def setitermethod(cls, name):
-        itermethod = getattr(cls, name)
-        setattr(cls, 'iter%s' % name, itermethod)
-
-        def listmethod(self, *a, **kw):
-            return list(itermethod(self, *a, **kw))
-
-        listmethod.__doc__ = \
-            'Like :py:meth:`iter%s`, but returns a list.' % name
-        setattr(cls, name, listmethod)
-
-    def wrap(cls):
-        for name in names:
-            setitermethod(cls, name)
-            setviewmethod(cls, name)
-        return cls
-    return wrap
+    return lambda x: x
 
 
 class _Missing(object):

burpui/extensions.py

@@ -11,10 +11,7 @@ import os
 import re
 import warnings
 
-from ._compat import PY3, to_unicode
-
-if PY3:  # pragma: no cover
-    basestring = str
+from ._compat import to_unicode
 
 
 def parse_db_setting(string):
@@ -284,7 +281,7 @@ def create_celery(myapp, warn=True):
         from .exceptions import BUIserverException
         host, oport, pwd = get_redis_server(myapp)
         odb = 2
-        if isinstance(myapp.use_celery, basestring):
+        if isinstance(myapp.use_celery, str):
             try:
                 (_, _, pwd, host, port, db) = parse_db_setting(myapp.use_celery)
                 if not port:

burpui/misc/backend/burp1.py

@@ -24,13 +24,9 @@ from ..parser.burp1 import Parser
 from ...utils import human_readable as _hr, BUIcompress, utc_to_local
 from ...security import sanitize_string
 from ...exceptions import BUIserverException
-from ..._compat import unquote, PY3, to_unicode, to_bytes
+from ..._compat import unquote, to_unicode, to_bytes
 
-if PY3:  # pragma: no cover
-    from shlex import quote
-else:
-    from pipes import quote
-    from itertools import imap as map
+from shlex import quote
 
 G_BURPPORT = 4972
 G_BURPHOST = u'::1'

burpui/misc/parser/openssl.py

@@ -11,15 +11,10 @@ import codecs
 import logging
 import subprocess
 
-from ..._compat import PY3
-
 from hashlib import md5
 from six import iteritems
 from OpenSSL import crypto
 
-if PY3:
-    long = int
-
 
 class OSSLAuth(object):
     """OpenSSL wrapper"""
@@ -88,7 +83,7 @@ class OSSLAuth(object):
 
         revoked = self.crl.get_revoked() or []
         for rvk in revoked:
-            if client_crt.get_serial_number() == long(rvk.get_serial(), 16):
+            if client_crt.get_serial_number() == int(rvk.get_serial(), 16):
                 return True
 
         return False

burpui/misc/parser/utils.py

@@ -948,18 +948,12 @@ class File(dict):
         self._dirty = True
         return self.options.pop(*args)
 
-    def __cmp__(self, dic):
-        return cmp(self.options, dic)
-
     def __contains__(self, item):
         return item in self.options
 
     def __iter__(self):
         return iter(self.options)
 
-    def __unicode__(self):
-        return unicode(self.__repr__())
-
     @property
     def raw(self):
         if self._raw and not self._changed and not self.changed:
@@ -1707,10 +1701,6 @@ class Config(File):
         self._dirty = True
         return self.get_default(True).pop(*args)
 
-    def __cmp__(self, dic):
-        self._refresh()
-        return cmp(self.options, dic)
-
     def __contains__(self, item):
         self._refresh()
         return item in self.options
@@ -1718,6 +1708,3 @@ class Config(File):
     def __iter__(self):
         self._refresh()
         return iter(self.options)
-
-    def __unicode__(self):
-        return unicode(repr(self))

burpui/routes.py

@@ -83,7 +83,7 @@ And here is the main site
 def calendar(server=None, client=None):
     server = server or request.args.get('serverName')
     client = client or request.args.get('clientName')
-    if not bui.standalone:
+    if not bui.config['STANDALONE']:
         servers = not server and not client
         clients = bool(server)
         cli = bool(client)
@@ -276,7 +276,7 @@ def live_monitor(server=None, name=None):
     """Live status monitor view"""
     server = server or request.args.get('serverName')
     running = bui.client.is_one_backup_running()
-    if bui.standalone:
+    if bui.config['STANDALONE']:
         if not running:
             flash(_('Sorry, there are no running backups'), 'warning')
             return redirect(url_for('.home'))
@@ -549,7 +549,7 @@ def about():
 @login_required
 def home():
     """Home page"""
-    if bui.standalone:
+    if bui.config['STANDALONE']:
         return redirect(url_for('.clients'))
     else:
         server = request.args.get('serverName')

burpui/tasks.py

@@ -21,7 +21,6 @@ from time import gmtime, strftime, sleep
 # Try to load modules from our current env first
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
 
-from burpui._compat import PY3  # noqa
 from burpui.config import config  # noqa
 from burpui.ext.async import celery  # noqa
 from burpui.ext.cache import cache  # noqa
@@ -36,9 +35,6 @@ try:
 except ImportError:
     WS_AVAILABLE = False
 
-if not PY3:
-    from itertools import imap as map
-
 if config.get('WITH_SQL'):
     from burpui.ext.sql import db
 else:
@@ -51,10 +47,6 @@ ME = __name__
 LOCK_EXPIRE = 60 * 30  # Lock expires in 30 minutes
 
 BEAT_SCHEDULE = {
-    'ping-backend-hourly': {
-        'task': '{}.ping_backend'.format(ME),
-        'schedule': crontab(minute='15'),  # run every hour
-    },
     'backup-running-4-minutely': {
         'task': '{}.backup_running'.format(ME),
         'schedule': timedelta(seconds=15),  # run every 15 seconds
@@ -129,24 +121,6 @@ def wait_for(lock_name, value, wait=10, timeout=LOCK_EXPIRE):
     return old_lock
 
 
-@celery.task(ignore_result=True)
-def ping_backend():
-    if bui.standalone:
-        bui.client.status()
-    else:
-        def __status(server):
-            (serv, back) = server
-            try:
-                return bui.client.status(agent=serv)
-            except BUIserverException:
-                return False
-
-        list(map(
-            __status,
-            iteritems(bui.client.servers)
-        ))
-
-
 @celery.task(bind=True, ignore_result=True)
 def backup_running(self):
     # run one at the time, if one task was already running, we just discard
@@ -182,7 +156,7 @@ def get_all_backups(self):
         return None
     try:
         backups = {}
-        if bui.standalone:
+        if bui.config['STANDALONE']:
             for cli in bui.client.get_all_clients():
                 backups[cli['name']] = bui.client.get_client(cli['name'])
         else:
@@ -203,7 +177,7 @@ def get_all_clients_reports(self):
         return None
     try:
         reports = {}
-        if bui.standalone:
+        if bui.config['STANDALONE']:
             reports = bui.client.get_clients_report(bui.client.get_all_clients())
         else:
             for serv in bui.client.servers:

burpui/utils.py

@@ -18,19 +18,14 @@ import logging
 
 from uuid import UUID
 from inspect import currentframe, getouterframes
-from ._compat import PY3, string_types
 
 NOTIF_OK = 0
 NOTIF_WARN = 1
 NOTIF_ERROR = 2
 NOTIF_INFO = 3
 
-if PY3:
-    long = int  # pragma: no cover
-    basestring = str  # pragma: no cover
 
-
-class human_readable(long):
+class human_readable(int):
     """define a human_readable class to allow custom formatting
     format specifiers supported :
         em : formats the size as bits in IEC format i.e. 1024 bits (128 bytes) = 1Kib
@@ -48,7 +43,7 @@ class human_readable(long):
             if fmt[-1].lower() in \
                     ['b', 'c', 'd', 'o', 'x', 'n', 'e', 'f', 'g', '%']:
                 # Numeric format.
-                return long(self).__format__(fmt)
+                return int(self).__format__(fmt)
             else:
                 return str(self).__format__(fmt)
 
@@ -82,81 +77,44 @@ class human_readable(long):
         return "{0:{1}}".format(t, width) if width != "" else t
 
 
-if PY3:  # pragma: no cover
-    class BUIlogger(logging.Logger):
-        padding = 0
-        """Logger class for more convenience"""
-        def makeRecord(self, name, level, fn, lno, msg,
-                       args, exc_info, func=None, extra=None, sinfo=None):
-            """
-            Try to guess where was call the function
-            """
-            cf = currentframe()
-            caller = getouterframes(cf)
-            cpt = 0
-            size = len(caller)
-            me = __file__
-            if me.endswith('.pyc'):
-                me = me[:-1]
-            # It's easy to get the _logger parent function because it's the
-            # following frame
-            while cpt < size - 1:
-                (_, filename, _, function_name, _, _) = caller[cpt]
-                if function_name == '_logger' and filename == me:
-                    cpt += 1
-                    break
-                cpt += 1
-            cpt += self.padding
-            (frame, filename, line_number, function_name, lines, index) = \