fix: try to parse timestamp according to the timezone #284

CHANGELOG.rst

@@ -5,6 +5,7 @@ Current
 -------
 
 - **BREAKING**: the *single* and *version* options within the ``[Global]`` section have been removed in favor of a new unified *backend* option
+- **BREAKING**: a change introduced by `#284 <https://git.ziirish.me/ziirish/burp-ui/issues/284>`_ may return wrong timestamps for backups made with burp-server <= 2.1.10 if your current burp-server is >= 2.1.10
 - Add: new `audit logging <https://git.ziirish.me/ziirish/burp-ui/issues/260>`_ system
 - Add: new ``bui-monitor`` processes pool + ``async`` backend to parallelize some requests `#278 <https://git.ziirish.me/ziirish/burp-ui/issues/278>`_ 
 - Add: new `listen` and `listen_status` options in burp-2.2.10 `#279 <https://git.ziirish.me/ziirish/burp-ui/issues/279>`_ 

burpui/misc/backend/burp1.py

@@ -237,51 +237,47 @@ class Burp(BUIbackend):
             self.logger.error('Cannot contact burp server at {0}:{1}'.format(self.host, self.port))
             raise BUIserverException('Cannot contact burp server at {0}:{1}'.format(self.host, self.port))
 
-    def _get_all_backup_logs(self, client, forward=False):
+    def _get_all_backup_logs(self, client, forward=False, deep=False):
         ret = []
         backups = self.get_client(client)
         queue = []
         for back in backups:
-            queue.append(self._get_backup_logs(back['number'], client, forward))
+            queue.append(self._get_backup_logs(back['number'], client, forward, deep))
 
         ret = sorted(queue, key=lambda x: x['number'])
         return ret
 
-    def _get_backup_logs(self, number, client, forward=False):
+    def _get_backup_logs(self, number, client, forward=False, deep=False):
         if not client or not number:
             return {}
 
         filemap = self.status('c:{0}:b:{1}\n'.format(client, number))
-        found = False
         ret = {}
         for line in filemap:
             if line == 'backup_stats':
-                found = True
+                ret = self._parse_backup_stats(number, client, forward)
                 break
-
-        if not found:
+        else:
             cli = None
             if forward:
                 cli = client
 
             filemap = self.status('c:{0}:b:{1}:f:log.gz\n'.format(client, number))
             ret = self._parse_backup_log(filemap, number, cli)
-        else:
-            ret = self._parse_backup_stats(number, client, forward)
 
         ret['encrypted'] = False
         if 'files_enc' in ret and ret['files_enc']['total'] > 0:
             ret['encrypted'] = True
         return ret
 
-    def get_backup_logs(self, number, client, forward=False, agent=None):
+    def get_backup_logs(self, number, client, forward=False, deep=False, agent=None):
         """See :func:`burpui.misc.backend.interface.BUIbackend.get_backup_logs`"""
         if not client or not number:
             return {} if number and number != -1 else []
 
         if number == -1:
-            return self._get_all_backup_logs(client, forward)
-        return self._get_backup_logs(number, client, forward)
+            return self._get_all_backup_logs(client, forward, deep)
+        return self._get_backup_logs(number, client, forward, deep)
 
     def _parse_backup_stats(self, number, client, forward=False, stats=None, agent=None):
         """The :func:`burpui.misc.backend.burp1.Burp._parse_backup_stats`

burpui/misc/backend/burp2.py

@@ -17,22 +17,17 @@ from collections import OrderedDict
 from .burp1 import Burp as Burp1
 from .interface import BUIbackend
 from .utils.burp2 import Monitor
-from .utils.constant import BURP_REVERSE_COUNTERS
+from .utils.constant import BURP_REVERSE_COUNTERS, BURP_STATUS_FORMAT_V2
 from ..parser.burp2 import Parser
 from ...utils import human_readable as _hr, utc_to_local
 from ...exceptions import BUIserverException
 from ..._compat import to_unicode
 
+from threading import RLock as _RLock
 try:
     from gevent.lock import RLock
-
 except ImportError:
-    class RLock(object):
-        def __enter__(self):
-            return self
-
-        def __exit__(*x):
-            pass
+    RLock = _RLock
 
 
 # Some functions are the same as in Burp1 backend
@@ -133,11 +128,12 @@ class Burp(Burp1):
         with self.plock:
             return self.monitor.status(query, timeout, cache)
 
-    def _get_backup_logs(self, number, client, forward=False):
+    def _get_backup_logs(self, number, client, forward=False, deep=False):
         """See
         :func:`burpui.misc.backend.interface.BUIbackend.get_backup_logs`
         """
         ret = {}
+        ret2 = {}
         if not client or not number:
             return ret
 
@@ -151,6 +147,10 @@ class Burp(Burp1):
             return ret
         if 'backup_stats' in logs:
             ret = self._parse_backup_stats(number, client, forward)
+        if 'backup' in logs and deep:
+            ret2 = self._parse_backup_log(number, client)
+
+        ret.update(ret2)
 
         ret['encrypted'] = False
         if 'files_enc' in ret and ret['files_enc']['total'] > 0:
@@ -165,6 +165,8 @@ class Burp(Burp1):
         ret['protocol'] = 1
         ret['is_windows'] = False
         ret['server_version'] = None
+        if not data:
+            return ret
         try:
             log = data['clients'][0]['backups'][0]['logs']['backup']
         except KeyError:
@@ -177,7 +179,7 @@ class Burp(Burp1):
             'protocol': re.compile(r'Protocol: (\d)$'),
             'is_windows': re.compile(r'Client is Windows$'),
         }
-        expressions_list = ret.keys()
+        expressions_list = list(ret.keys())
         catching_expressions = ['client_version', 'server_version', 'protocol']
         casting_expressions = {
             'protocol': int,
@@ -187,16 +189,21 @@ class Burp(Burp1):
             return val
 
         for line in log:
-            for expression in expressions_list:
+            expressions = expressions_list
+            for expression in expressions:
                 if expression in catching_expressions:
                     catch = regex[expression].search(line)
                     if catch:
                         cast = casting_expressions.get(expression, __dummy)
                         ret[expression] = cast(catch.group(1))
+                        # don't search this expression twice
+                        expressions_list.remove(expression)
                         break
                 else:
                     if expression in regex and regex[expression].search(line):
                         ret[expression] = True
+                        # don't search this expression twice
+                        expressions_list.remove(expression)
                         break
         return ret
 
@@ -213,8 +220,8 @@ class Burp(Burp1):
 
         :returns: a dict with some useful details
         """
-        query = self.status('c:{0}:b:{1}:l:backup\n'.format(client, number))
-        return self._do_parse_backup_log(query, client)
+        data = self.status('c:{0}:b:{1}:l:backup\n'.format(client, number))
+        return self._do_parse_backup_log(data, client)
 
     def _do_parse_backup_stats(self, data, result, number, client, forward=False, agent=None):
         ret = {}
@@ -523,7 +530,7 @@ class Burp(Burp1):
             clients = self.status('c:{}'.format(name))
             client = clients['clients'][0]
             return client['backups'][0]
-        except (KeyError, BUIserverException):
+        except (KeyError, TypeError, BUIserverException):
             return None
 
     def _guess_os(self, name):
@@ -591,7 +598,15 @@ class Burp(Burp1):
                 cli['last'] = 'never'
             else:
                 infos = infos[0]
-                cli['last'] = infos['timestamp']
+                if self.server_version and self.server_version < BURP_STATUS_FORMAT_V2:
+                    cli['last'] = infos['timestamp']
+                # only do deep inspection when server >= BURP_STATUS_FORMAT_V2
+                elif self.deep_inspection:
+                    logs = self.get_backup_logs(infos['number'], client['name'])
+                    cli['last'] = logs['start']
+                else:
+                    cli['last'] = utc_to_local(infos['timestamp'])
+
             ret.append(cli)
         return ret
 

burpui/misc/backend/interface.py

@@ -22,6 +22,7 @@ G_BURPCONFCLI = '/etc/burp/burp.conf'
 G_BURPCONFSRV = '/etc/burp/burp-server.conf'
 G_TMPDIR = '/tmp/bui'
 G_TIMEOUT = 15
+G_DEEP_INSPECTION = False
 G_ZIP64 = True
 G_INCLUDES = ['/etc/burp']
 G_ENFORCE = False
@@ -62,6 +63,7 @@ class BUIbackend(object, metaclass=ABCMeta):
         self.includes = G_INCLUDES
         self.revoke = G_REVOKE
         self.enforce = G_ENFORCE
+        self.deep_inspection = G_DEEP_INSPECTION
         self.running = []
         burp_opts = {
             'bport': G_BURPPORT,
@@ -72,6 +74,7 @@ class BUIbackend(object, metaclass=ABCMeta):
             'bconfsrv': G_BURPCONFSRV,
             'timeout': G_TIMEOUT,
             'tmpdir': G_TMPDIR,
+            'deep_inspection': G_DEEP_INSPECTION,
         }
         self.defaults = {
             'Burp': burp_opts,
@@ -129,6 +132,11 @@ class BUIbackend(object, metaclass=ABCMeta):
                 'integer'
             )
 
+            self.deep_inspection = conf.safe_get(
+                'deep_inspection',
+                'boolean'
+            )
+
             # Experimental options
             self.zip64 = conf.safe_get(
                 'zip64',
@@ -262,7 +270,7 @@ class BUIbackend(object, metaclass=ABCMeta):
         raise NotImplementedError("Sorry, the current Backend does not implement this method!")  # pragma: no cover
 
     @abstractmethod
-    def get_backup_logs(self, number, client, forward=False, agent=None):
+    def get_backup_logs(self, number, client, forward=False, deep=False, agent=None):
         """The :func:`burpui.misc.backend.interface.BUIbackend.get_backup_logs`
         function is used to retrieve the burp logs depending the burp-server
         version.
@@ -276,6 +284,9 @@ class BUIbackend(object, metaclass=ABCMeta):
         :param forward: Is the client name needed in later process
         :type forward: bool
 
+        :param deep: Enable deep log inspection
+        :type deep: bool
+
         :param agent: What server to ask (only in multi-agent mode)
         :type agent: str
 

burpui/misc/backend/parallel.py

@@ -16,6 +16,7 @@ import logging
 
 from .burp2 import Burp as Burp2
 from .interface import BUIbackend
+from .utils.constant import BURP_STATUS_FORMAT_V2
 from ..parser.burp2 import Parser
 from ...exceptions import BUIserverException
 from ..._compat import to_unicode, to_bytes
@@ -48,7 +49,7 @@ class Parallel:
         self.password = conf.safe_get('password', section='Parallel', defaults=BUI_DEFAULTS)
         self.timeout = conf.safe_get('timeout', 'integer', section='Parallel', defaults=BUI_DEFAULTS)
 
-        self.logger.info(f'Monitor {self.host}:{self.port} - ssl: {self.ssl}')
+        self.logger.debug(f'Monitor {self.host}:{self.port} - ssl: {self.ssl}')
 
         self.connected = False
 
@@ -58,6 +59,10 @@ class Parallel:
                 ctx = ssl.SSLContext()
                 ctx.verify_mode = ssl.CERT_NONE
                 ctx.check_hostname = False
+                ctx.options |= (
+                    ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
+                )  # RFC 7540 Section 9.2: MUST be TLS >=1.2
+                ctx.options |= ssl.OP_NO_COMPRESSION  # RFC 7540 Section 9.2.1: MUST disable compression
                 ctx.load_default_certs()
                 self.client_stream = await trio.open_ssl_over_tcp_stream(self.host, self.port, ssl_context=ctx)
             else:
@@ -239,11 +244,14 @@ class Burp(Burp2):
         """See :func:`burpui.misc.backend.interface.BUIbackend.status`"""
         return trio.run(self._async_status, query, timeout, cache)
 
-    async def _async_get_backup_logs(self, number, client, forward=False, store=None, limit=None):
+    async def _async_get_backup_logs(self, number, client, forward=False, deep=False, store=None, limit=None):
         async def _do_stuff():
             nonlocal client
             nonlocal number
             nonlocal forward
+            nonlocal deep
+            bucket1 = []
+            bucket2 = []
             ret = {}
             query = await self._async_status('c:{0}:b:{1}\n'.format(client, number))
             if not query:
@@ -253,8 +261,16 @@ class Burp(Burp2):
             except KeyError:
                 self.logger.warning('No logs found')
                 return ret
-            if 'backup_stats' in logs:
-                ret = await self._async_parse_backup_stats(number, client, forward)
+            async with trio.open_nursery() as nursery:
+                if 'backup_stats' in logs:
+                    nursery.start_soon(self._async_parse_backup_stats, number, client, forward, None, bucket1)
+                if 'backup' in logs and deep:
+                    nursery.start_soon(self._async_parse_backup_log, number, client, bucket2)
+
+            if bucket1:
+                ret = bucket1[0]
+            if bucket2:
+                ret.update(bucket2[0])
 
             ret['encrypted'] = False
             if 'files_enc' in ret and ret['files_enc']['total'] > 0:
@@ -272,19 +288,19 @@ class Burp(Burp2):
         else:
             return res
 
-    async def _async_get_all_backup_logs(self, client, forward=False):
+    async def _async_get_all_backup_logs(self, client, forward=False, deep=False):
         ret = []
         backups = await self._async_get_client(client)
         queue = []
         limit = trio.CapacityLimiter(self.concurrency)
         async with trio.open_nursery() as nursery:
             for back in backups:
-                nursery.start_soon(self._async_get_backup_logs, back['number'], client, forward, queue, limit)
+                nursery.start_soon(self._async_get_backup_logs, back['number'], client, forward, deep, queue, limit)
 
         ret = sorted(queue, key=lambda x: x['number'])
         return ret
 
-    def get_backup_logs(self, number, client, forward=False, agent=None):
+    def get_backup_logs(self, number, client, forward=False, deep=False, agent=None):
         """See
         :func:`burpui.misc.backend.interface.BUIbackend.get_backup_logs`
         """
@@ -292,12 +308,15 @@ class Burp(Burp2):
             return {} if number and number != -1 else []
 
         if number == -1:
-            return trio.run(self._async_get_all_backup_logs, client, forward)
-        return trio.run(self._async_get_backup_logs, number, client, forward)
+            return trio.run(self._async_get_all_backup_logs, client, forward, deep)
+        return trio.run(self._async_get_backup_logs, number, client, forward, deep)
 
-    async def _async_parse_backup_log(self, number, client):
+    async def _async_parse_backup_log(self, number, client, bucket=None):
         query = await self._async_status('c:{0}:b:{1}:l:backup\n'.format(client, number))
-        return self._do_parse_backup_log(query, client)
+        res = self._do_parse_backup_log(query, client)
+        if bucket is not None:
+            bucket.append(res)
+        return res
 
     def _parse_backup_log(self, number, client):
         """The :func:`burpui.misc.backend.burp2.Burp._parse_backup_log`
@@ -314,7 +333,7 @@ class Burp(Burp2):
         """
         return trio.run(self._async_parse_backup_log, number, client)
 
-    async def _async_parse_backup_stats(self, number, client, forward=False, agent=None):
+    async def _async_parse_backup_stats(self, number, client, forward=False, agent=None, bucket=None):
         """The :func:`burpui.misc.backend.parallel.Burp._async_parse_backup_stats`
         function is used to parse the burp logs.
 
@@ -338,7 +357,10 @@ class Burp(Burp2):
         query = await self._async_status(
             'c:{0}:b:{1}:l:backup_stats\n'.format(client, number)
         )
-        return self._do_parse_backup_stats(query, backup, number, client, forward, agent)
+        ret = self._do_parse_backup_stats(query, backup, number, client, forward, agent)
+        if bucket is not None:
+            bucket.append(ret)
+        return ret
 
     # def get_clients_report(self, clients, agent=None):
 
@@ -434,7 +456,47 @@ class Burp(Burp2):
         """
         return trio.run(self._async_guess_os, name)
 
-    # def get_all_clients(self, agent=None):
+    async def _async_get_all_clients(self):
+        ret = []
+        query = await self._async_status()
+        if not query or 'clients' not in query:
+            return ret
+
+        async def __compute_client_data(client, limit, queue):
+            async with limit:
+                cli = {}
+                cli['name'] = client['name']
+                cli['state'] = self._status_human_readable(client['run_status'])
+                infos = client['backups']
+                if cli['state'] in ['running']:
+                    cli['last'] = 'now'
+                elif not infos:
+                    cli['last'] = 'never'
+                else:
+                    infos = infos[0]
+                    logs = await self._async_get_backup_logs(infos['number'], client['name'])
+                    cli['last'] = logs['start']
+                queue.append(cli)
+
+        clients = query['clients']
+        limiter = trio.CapacityLimiter(self.concurrency)
+
+        async with trio.open_nursery() as nursery:
+            for client in clients:
+                nursery.start_soon(__compute_client_data, client, limiter, ret)
+
+        return ret
+
+    def get_all_clients(self, agent=None):
+        """See
+        :func:`burpui.misc.backend.interface.BUIbackend.get_all_clients`
+        """
+        # don't need async processing if burp-server < BURP_STATUS_FORMAT_V2
+        if not self.deep_inspection or (self.server_version and
+                                        self.server_version < BURP_STATUS_FORMAT_V2):
+            return Burp2.get_all_clients(self)
+        # the deep inspection can take advantage of async processing
+        return trio.run(self._async_get_all_clients)
 
     # def get_client_status(self, name=None, agent=None):
 

docs/advanced_usage.rst

@@ -258,7 +258,7 @@ Some of these options are also available in the `bui-agent`_ configuration file.
 Backends
 --------
 
-`Burp-UI`_ ships with three different backends:
+`Burp-UI`_ ships with four different backends:
 
 - `Burp1`_
 - `Burp2`_
@@ -432,6 +432,15 @@ Options
     tmpdir = /tmp
     # how many time to wait for the monitor to answer (in seconds)
     timeout = 5
+    # since burp-2.1.10, timestamps have local offsets, if we detect a burp-server
+    # version greater than 2.1.10 we'll suppose every backup was made with that
+    # version. If this is not the case, you may end-up with wrongly computed backup
+    # dates in the clients overview. For that reason, you can enable the
+    # 'deep_inspection' option which will check every backup logs in order to
+    # find out which server version was used.
+    # The drawback is this process requires some extra work that may slow-down
+    # burp-ui.
+    deep_inspection = false
 
 
 Each option is commented, but here is a more detailed documentation:

docs/upgrading.rst

@@ -10,6 +10,24 @@ Each section presents major/breaking changes, new requirements and new options.
 For a complete list of changes, you may refer to the
 `CHANGELOG <changelog.html>`_ page.
 
+v0.7.0
+------
+
+- **Breaking** - the *single* and *version* options within the ``[Global]``
+  section have been removed in favor of a new unified *backend* option. See the
+  `Backends <advanced_usage.html#backends>`__ section of the documentation for
+  details.
+- **Breaking** - there was a bug when using burp-server >= 2.1.10 where
+  timestamps where wrongly computed on the global clients view due to the fact
+  timestamps have offsets since burp-server 2.1.10. The new behaviour is to
+  suppose every timestamp have offset whenever we detect your current
+  burp-server version is >= 2.1.10. If backups were made with an older
+  burp-server, the timestamps will be wrong. You can avoid that by enabling
+  the ``deep_inspection`` option in the ``[Burp]`` section (see the
+  `Backend options <advanced_usage.html#options>`__ for details).
+  The drawback of enabling the ``deep_inspection`` is this requires some extra
+  work that may slow down burp-ui.
+
 v0.6.0
 ------
 

share/burpui/etc/burpui.sample.cfg

@@ -182,6 +182,15 @@ bconfsrv = /etc/burp/burp-server.conf
 tmpdir = /tmp/bui
 # how many time to wait for the monitor to answer (in seconds)
 timeout = 15
+# since burp-2.1.10, timestamps have local offsets, if we detect a burp-server
+# version greater than 2.1.10 we'll suppose every backup was made with that
+# version. If this is not the case, you may end-up with wrongly computed backup
+# dates in the clients overview. For that reason, you can enable the
+# 'deep_inspection' option which will check every backup logs in order to
+# find out which server version was used.
+# The drawback is this process requires some extra work that may slow-down
+# burp-ui.
+deep_inspection = false
 
 # parallel backend specific options
 [Parallel]