Commit f8f8e94c authored by Ziirish's avatar Ziirish

fix: try to parse timestamp according to the timezone #284

parent a718e7be
Pipeline #1510 passed with stage
in 1 minute and 21 seconds
......@@ -5,6 +5,7 @@ Current
-------
- **BREAKING**: the *single* and *version* options within the ``[Global]`` section have been removed in favor of a new unified *backend* option
- **BREAKING**: a change introduced by `#284 <https://git.ziirish.me/ziirish/burp-ui/issues/284>`_ may return wrong timestamps for backups made with burp-server <= 2.1.10 if your current burp-server is >= 2.1.10
- Add: new `audit logging <https://git.ziirish.me/ziirish/burp-ui/issues/260>`_ system
- Add: new ``bui-monitor`` processes pool + ``async`` backend to parallelize some requests `#278 <https://git.ziirish.me/ziirish/burp-ui/issues/278>`_
- Add: new `listen` and `listen_status` options in burp-2.2.10 `#279 <https://git.ziirish.me/ziirish/burp-ui/issues/279>`_
......
......@@ -237,51 +237,47 @@ class Burp(BUIbackend):
self.logger.error('Cannot contact burp server at {0}:{1}'.format(self.host, self.port))
raise BUIserverException('Cannot contact burp server at {0}:{1}'.format(self.host, self.port))
def _get_all_backup_logs(self, client, forward=False):
def _get_all_backup_logs(self, client, forward=False, deep=False):
ret = []
backups = self.get_client(client)
queue = []
for back in backups:
queue.append(self._get_backup_logs(back['number'], client, forward))
queue.append(self._get_backup_logs(back['number'], client, forward, deep))
ret = sorted(queue, key=lambda x: x['number'])
return ret
def _get_backup_logs(self, number, client, forward=False):
def _get_backup_logs(self, number, client, forward=False, deep=False):
if not client or not number:
return {}
filemap = self.status('c:{0}:b:{1}\n'.format(client, number))
found = False
ret = {}
for line in filemap:
if line == 'backup_stats':
found = True
ret = self._parse_backup_stats(number, client, forward)
break
if not found:
else:
cli = None
if forward:
cli = client
filemap = self.status('c:{0}:b:{1}:f:log.gz\n'.format(client, number))
ret = self._parse_backup_log(filemap, number, cli)
else:
ret = self._parse_backup_stats(number, client, forward)
ret['encrypted'] = False
if 'files_enc' in ret and ret['files_enc']['total'] > 0:
ret['encrypted'] = True
return ret
def get_backup_logs(self, number, client, forward=False, agent=None):
def get_backup_logs(self, number, client, forward=False, deep=False, agent=None):
"""See :func:`burpui.misc.backend.interface.BUIbackend.get_backup_logs`"""
if not client or not number:
return {} if number and number != -1 else []
if number == -1:
return self._get_all_backup_logs(client, forward)
return self._get_backup_logs(number, client, forward)
return self._get_all_backup_logs(client, forward, deep)
return self._get_backup_logs(number, client, forward, deep)
def _parse_backup_stats(self, number, client, forward=False, stats=None, agent=None):
"""The :func:`burpui.misc.backend.burp1.Burp._parse_backup_stats`
......
......@@ -17,22 +17,17 @@ from collections import OrderedDict
from .burp1 import Burp as Burp1
from .interface import BUIbackend
from .utils.burp2 import Monitor
from .utils.constant import BURP_REVERSE_COUNTERS
from .utils.constant import BURP_REVERSE_COUNTERS, BURP_STATUS_FORMAT_V2
from ..parser.burp2 import Parser
from ...utils import human_readable as _hr, utc_to_local
from ...exceptions import BUIserverException
from ..._compat import to_unicode
from threading import RLock as _RLock
try:
from gevent.lock import RLock
except ImportError:
class RLock(object):
def __enter__(self):
return self
def __exit__(*x):
pass
RLock = _RLock
# Some functions are the same as in Burp1 backend
......@@ -133,11 +128,12 @@ class Burp(Burp1):
with self.plock:
return self.monitor.status(query, timeout, cache)
def _get_backup_logs(self, number, client, forward=False):
def _get_backup_logs(self, number, client, forward=False, deep=False):
"""See
:func:`burpui.misc.backend.interface.BUIbackend.get_backup_logs`
"""
ret = {}
ret2 = {}
if not client or not number:
return ret
......@@ -151,6 +147,10 @@ class Burp(Burp1):
return ret
if 'backup_stats' in logs:
ret = self._parse_backup_stats(number, client, forward)
if 'backup' in logs and deep:
ret2 = self._parse_backup_log(number, client)
ret.update(ret2)
ret['encrypted'] = False
if 'files_enc' in ret and ret['files_enc']['total'] > 0:
......@@ -165,6 +165,8 @@ class Burp(Burp1):
ret['protocol'] = 1
ret['is_windows'] = False
ret['server_version'] = None
if not data:
return ret
try:
log = data['clients'][0]['backups'][0]['logs']['backup']
except KeyError:
......@@ -177,7 +179,7 @@ class Burp(Burp1):
'protocol': re.compile(r'Protocol: (\d)$'),
'is_windows': re.compile(r'Client is Windows$'),
}
expressions_list = ret.keys()
expressions_list = list(ret.keys())
catching_expressions = ['client_version', 'server_version', 'protocol']
casting_expressions = {
'protocol': int,
......@@ -187,16 +189,21 @@ class Burp(Burp1):
return val
for line in log:
for expression in expressions_list:
expressions = expressions_list
for expression in expressions:
if expression in catching_expressions:
catch = regex[expression].search(line)
if catch:
cast = casting_expressions.get(expression, __dummy)
ret[expression] = cast(catch.group(1))
# don't search this expression twice
expressions_list.remove(expression)
break
else:
if expression in regex and regex[expression].search(line):
ret[expression] = True
# don't search this expression twice
expressions_list.remove(expression)
break
return ret
......@@ -213,8 +220,8 @@ class Burp(Burp1):
:returns: a dict with some useful details
"""
query = self.status('c:{0}:b:{1}:l:backup\n'.format(client, number))
return self._do_parse_backup_log(query, client)
data = self.status('c:{0}:b:{1}:l:backup\n'.format(client, number))
return self._do_parse_backup_log(data, client)
def _do_parse_backup_stats(self, data, result, number, client, forward=False, agent=None):
ret = {}
......@@ -523,7 +530,7 @@ class Burp(Burp1):
clients = self.status('c:{}'.format(name))
client = clients['clients'][0]
return client['backups'][0]
except (KeyError, BUIserverException):
except (KeyError, TypeError, BUIserverException):
return None
def _guess_os(self, name):
......@@ -591,7 +598,15 @@ class Burp(Burp1):
cli['last'] = 'never'
else:
infos = infos[0]
cli['last'] = infos['timestamp']
if self.server_version and self.server_version < BURP_STATUS_FORMAT_V2:
cli['last'] = infos['timestamp']
# only do deep inspection when server >= BURP_STATUS_FORMAT_V2
elif self.deep_inspection:
logs = self.get_backup_logs(infos['number'], client['name'])
cli['last'] = logs['start']
else:
cli['last'] = utc_to_local(infos['timestamp'])
ret.append(cli)
return ret
......
......@@ -22,6 +22,7 @@ G_BURPCONFCLI = '/etc/burp/burp.conf'
G_BURPCONFSRV = '/etc/burp/burp-server.conf'
G_TMPDIR = '/tmp/bui'
G_TIMEOUT = 15
G_DEEP_INSPECTION = False
G_ZIP64 = True
G_INCLUDES = ['/etc/burp']
G_ENFORCE = False
......@@ -62,6 +63,7 @@ class BUIbackend(object, metaclass=ABCMeta):
self.includes = G_INCLUDES
self.revoke = G_REVOKE
self.enforce = G_ENFORCE
self.deep_inspection = G_DEEP_INSPECTION
self.running = []
burp_opts = {
'bport': G_BURPPORT,
......@@ -72,6 +74,7 @@ class BUIbackend(object, metaclass=ABCMeta):
'bconfsrv': G_BURPCONFSRV,
'timeout': G_TIMEOUT,
'tmpdir': G_TMPDIR,
'deep_inspection': G_DEEP_INSPECTION,
}
self.defaults = {
'Burp': burp_opts,
......@@ -129,6 +132,11 @@ class BUIbackend(object, metaclass=ABCMeta):
'integer'
)
self.deep_inspection = conf.safe_get(
'deep_inspection',
'boolean'
)
# Experimental options
self.zip64 = conf.safe_get(
'zip64',
......@@ -262,7 +270,7 @@ class BUIbackend(object, metaclass=ABCMeta):
raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover
@abstractmethod
def get_backup_logs(self, number, client, forward=False, agent=None):
def get_backup_logs(self, number, client, forward=False, deep=False, agent=None):
"""The :func:`burpui.misc.backend.interface.BUIbackend.get_backup_logs`
function is used to retrieve the burp logs depending the burp-server
version.
......@@ -276,6 +284,9 @@ class BUIbackend(object, metaclass=ABCMeta):
:param forward: Is the client name needed in later process
:type forward: bool
:param deep: Enable deep log inspection
:type deep: bool
:param agent: What server to ask (only in multi-agent mode)
:type agent: str
......
......@@ -16,6 +16,7 @@ import logging
from .burp2 import Burp as Burp2
from .interface import BUIbackend
from .utils.constant import BURP_STATUS_FORMAT_V2
from ..parser.burp2 import Parser
from ...exceptions import BUIserverException
from ..._compat import to_unicode, to_bytes
......@@ -48,7 +49,7 @@ class Parallel:
self.password = conf.safe_get('password', section='Parallel', defaults=BUI_DEFAULTS)
self.timeout = conf.safe_get('timeout', 'integer', section='Parallel', defaults=BUI_DEFAULTS)
self.logger.info(f'Monitor {self.host}:{self.port} - ssl: {self.ssl}')
self.logger.debug(f'Monitor {self.host}:{self.port} - ssl: {self.ssl}')
self.connected = False
......@@ -58,6 +59,10 @@ class Parallel:
ctx = ssl.SSLContext()
ctx.verify_mode = ssl.CERT_NONE
ctx.check_hostname = False
ctx.options |= (
ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
) # RFC 7540 Section 9.2: MUST be TLS >=1.2
ctx.options |= ssl.OP_NO_COMPRESSION # RFC 7540 Section 9.2.1: MUST disable compression
ctx.load_default_certs()
self.client_stream = await trio.open_ssl_over_tcp_stream(self.host, self.port, ssl_context=ctx)
else:
......@@ -239,11 +244,14 @@ class Burp(Burp2):
"""See :func:`burpui.misc.backend.interface.BUIbackend.status`"""
return trio.run(self._async_status, query, timeout, cache)
async def _async_get_backup_logs(self, number, client, forward=False, store=None, limit=None):
async def _async_get_backup_logs(self, number, client, forward=False, deep=False, store=None, limit=None):
async def _do_stuff():
nonlocal client
nonlocal number
nonlocal forward
nonlocal deep
bucket1 = []
bucket2 = []
ret = {}
query = await self._async_status('c:{0}:b:{1}\n'.format(client, number))
if not query:
......@@ -253,8 +261,16 @@ class Burp(Burp2):
except KeyError:
self.logger.warning('No logs found')
return ret
if 'backup_stats' in logs:
ret = await self._async_parse_backup_stats(number, client, forward)
async with trio.open_nursery() as nursery:
if 'backup_stats' in logs:
nursery.start_soon(self._async_parse_backup_stats, number, client, forward, None, bucket1)
if 'backup' in logs and deep:
nursery.start_soon(self._async_parse_backup_log, number, client, bucket2)
if bucket1:
ret = bucket1[0]
if bucket2:
ret.update(bucket2[0])
ret['encrypted'] = False
if 'files_enc' in ret and ret['files_enc']['total'] > 0:
......@@ -272,19 +288,19 @@ class Burp(Burp2):
else:
return res
async def _async_get_all_backup_logs(self, client, forward=False):
async def _async_get_all_backup_logs(self, client, forward=False, deep=False):
ret = []
backups = await self._async_get_client(client)
queue = []
limit = trio.CapacityLimiter(self.concurrency)
async with trio.open_nursery() as nursery:
for back in backups:
nursery.start_soon(self._async_get_backup_logs, back['number'], client, forward, queue, limit)
nursery.start_soon(self._async_get_backup_logs, back['number'], client, forward, deep, queue, limit)
ret = sorted(queue, key=lambda x: x['number'])
return ret
def get_backup_logs(self, number, client, forward=False, agent=None):
def get_backup_logs(self, number, client, forward=False, deep=False, agent=None):
"""See
:func:`burpui.misc.backend.interface.BUIbackend.get_backup_logs`
"""
......@@ -292,12 +308,15 @@ class Burp(Burp2):
return {} if number and number != -1 else []
if number == -1:
return trio.run(self._async_get_all_backup_logs, client, forward)
return trio.run(self._async_get_backup_logs, number, client, forward)
return trio.run(self._async_get_all_backup_logs, client, forward, deep)
return trio.run(self._async_get_backup_logs, number, client, forward, deep)
async def _async_parse_backup_log(self, number, client):
async def _async_parse_backup_log(self, number, client, bucket=None):
query = await self._async_status('c:{0}:b:{1}:l:backup\n'.format(client, number))
return self._do_parse_backup_log(query, client)
res = self._do_parse_backup_log(query, client)
if bucket is not None:
bucket.append(res)
return res
def _parse_backup_log(self, number, client):
"""The :func:`burpui.misc.backend.burp2.Burp._parse_backup_log`
......@@ -314,7 +333,7 @@ class Burp(Burp2):
"""
return trio.run(self._async_parse_backup_log, number, client)
async def _async_parse_backup_stats(self, number, client, forward=False, agent=None):
async def _async_parse_backup_stats(self, number, client, forward=False, agent=None, bucket=None):
"""The :func:`burpui.misc.backend.parallel.Burp._async_parse_backup_stats`
function is used to parse the burp logs.
......@@ -338,7 +357,10 @@ class Burp(Burp2):
query = await self._async_status(
'c:{0}:b:{1}:l:backup_stats\n'.format(client, number)
)
return self._do_parse_backup_stats(query, backup, number, client, forward, agent)
ret = self._do_parse_backup_stats(query, backup, number, client, forward, agent)
if bucket is not None:
bucket.append(ret)
return ret
# def get_clients_report(self, clients, agent=None):
......@@ -434,7 +456,47 @@ class Burp(Burp2):
"""
return trio.run(self._async_guess_os, name)
# def get_all_clients(self, agent=None):
async def _async_get_all_clients(self):
ret = []
query = await self._async_status()
if not query or 'clients' not in query:
return ret
async def __compute_client_data(client, limit, queue):
async with limit:
cli = {}
cli['name'] = client['name']
cli['state'] = self._status_human_readable(client['run_status'])
infos = client['backups']
if cli['state'] in ['running']:
cli['last'] = 'now'
elif not infos:
cli['last'] = 'never'
else:
infos = infos[0]
logs = await self._async_get_backup_logs(infos['number'], client['name'])
cli['last'] = logs['start']
queue.append(cli)
clients = query['clients']
limiter = trio.CapacityLimiter(self.concurrency)
async with trio.open_nursery() as nursery:
for client in clients:
nursery.start_soon(__compute_client_data, client, limiter, ret)
return ret
def get_all_clients(self, agent=None):
"""See
:func:`burpui.misc.backend.interface.BUIbackend.get_all_clients`
"""
# don't need async processing if burp-server < BURP_STATUS_FORMAT_V2
if not self.deep_inspection or (self.server_version and
self.server_version < BURP_STATUS_FORMAT_V2):
return Burp2.get_all_clients(self)
# the deep inspection can take advantage of async processing
return trio.run(self._async_get_all_clients)
# def get_client_status(self, name=None, agent=None):
......
......@@ -258,7 +258,7 @@ Some of these options are also available in the `bui-agent`_ configuration file.
Backends
--------
`Burp-UI`_ ships with three different backends:
`Burp-UI`_ ships with four different backends:
- `Burp1`_
- `Burp2`_
......@@ -432,6 +432,15 @@ Options
tmpdir = /tmp
# how many time to wait for the monitor to answer (in seconds)
timeout = 5
# since burp-2.1.10, timestamps have local offsets, if we detect a burp-server
# version greater than 2.1.10 we'll suppose every backup was made with that
# version. If this is not the case, you may end-up with wrongly computed backup
# dates in the clients overview. For that reason, you can enable the
# 'deep_inspection' option which will check every backup logs in order to
# find out which server version was used.
# The drawback is this process requires some extra work that may slow-down
# burp-ui.
deep_inspection = false
Each option is commented, but here is a more detailed documentation:
......
......@@ -10,6 +10,24 @@ Each section presents major/breaking changes, new requirements and new options.
For a complete list of changes, you may refer to the
`CHANGELOG <changelog.html>`_ page.
v0.7.0
------
- **Breaking** - the *single* and *version* options within the ``[Global]``
section have been removed in favor of a new unified *backend* option. See the
`Backends <advanced_usage.html#backends>`__ section of the documentation for
details.
- **Breaking** - there was a bug when using burp-server >= 2.1.10 where
timestamps where wrongly computed on the global clients view due to the fact
timestamps have offsets since burp-server 2.1.10. The new behaviour is to
suppose every timestamp have offset whenever we detect your current
burp-server version is >= 2.1.10. If backups were made with an older
burp-server, the timestamps will be wrong. You can avoid that by enabling
the ``deep_inspection`` option in the ``[Burp]`` section (see the
`Backend options <advanced_usage.html#options>`__ for details).
The drawback of enabling the ``deep_inspection`` is this requires some extra
work that may slow down burp-ui.
v0.6.0
------
......
......@@ -182,6 +182,15 @@ bconfsrv = /etc/burp/burp-server.conf
tmpdir = /tmp/bui
# how many time to wait for the monitor to answer (in seconds)
timeout = 15
# since burp-2.1.10, timestamps have local offsets, if we detect a burp-server
# version greater than 2.1.10 we'll suppose every backup was made with that
# version. If this is not the case, you may end-up with wrongly computed backup
# dates in the clients overview. For that reason, you can enable the
# 'deep_inspection' option which will check every backup logs in order to
# find out which server version was used.
# The drawback is this process requires some extra work that may slow-down
# burp-ui.
deep_inspection = false
# parallel backend specific options
[Parallel]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment