refactoring to reduce duplicate code

parent d38beea8
Pipeline #1485 canceled with stages
......@@ -12,6 +12,8 @@ import os
import time
import json
from collections import OrderedDict
from .burp1 import Burp as Burp1
from .interface import BUIbackend
from .utils.burp2 import Monitor
......@@ -155,32 +157,53 @@ class Burp(Burp1):
ret['encrypted'] = True
return ret
def _guess_backup_protocol(self, number, client):
"""The :func:`burpui.misc.backend.burp2.Burp._guess_backup_protocol`
function helps you determine if the backup is protocol 2 or 1
:param number: Backup number to work on
:type number: int
:param client: Client name to work on
:type client: str
:returns: 1 or 2
"""
query = self.status('c:{0}:b:{1}:l:backup\n'.format(client, number))
@staticmethod
def _do_parse_backup_log(data, client):
# tests ordered as the logs order
ret = OrderedDict()
ret['client_version'] = None
ret['protocol'] = 1
ret['is_windows'] = False
ret['server_version'] = None
try:
log = query['clients'][0]['backups'][0]['logs']['backup']
for line in log:
if re.search(r'Protocol: 2$', line):
return 2
log = data['clients'][0]['backups'][0]['logs']['backup']
except KeyError:
# Assume protocol 1 in all cases unless explicitly found Protocol 2
return 1
return 1
return ret
# pre-compile regex since they'll be called on every log line
regex = {
'client_version': re.compile(r'Client version: (\d+\.\d+\.\d+)$'),
'server_version': re.compile(r"WARNING: Client '{}' version '\d+\.\d+\.\d+' does not match server version '(\d+\.\d+\.\d+)'. An upgrade is recommended.$".format(client)),
'protocol': re.compile(r'Protocol: (\d)$'),
'is_windows': re.compile(r'Client is Windows$'),
}
expressions_list = ret.keys()
catching_expressions = ['client_version', 'server_version', 'protocol']
casting_expressions = {
'protocol': int,
}
def _parse_backup_stats(self, number, client, forward=False, agent=None):
"""The :func:`burpui.misc.backend.burp2.Burp._parse_backup_stats`
function is used to parse the burp logs.
def __dummy(val):
return val
for line in log:
for expression in expressions_list:
if expression in catching_expressions:
catch = regex[expression].search(line)
if catch:
cast = casting_expressions.get(expression, __dummy)
ret[expression] = cast(catch.group(1))
break
else:
if expression in regex and regex[expression].search(line):
ret[expression] = True
break
return ret
def _parse_backup_log(self, number, client):
"""The :func:`burpui.misc.backend.burp2.Burp._parse_backup_log`
function helps you determine if the backup is protocol 2 or 1 and various
useful details.
:param number: Backup number to work on
:type number: int
......@@ -188,18 +211,13 @@ class Burp(Burp1):
:param client: Client name to work on
:type client: str
:param forward: Is the client name needed in later process
:type forward: bool
:param agent: What server to ask (only in multi-agent mode)
:type agent: str
:returns: Dict containing the backup log
:returns: a dict with some useful details
"""
query = self.status('c:{0}:b:{1}:l:backup\n'.format(client, number))
return self._do_parse_backup_log(query, client)
def _do_parse_backup_stats(self, data, result, number, client, forward=False, agent=None):
ret = {}
backup = {'os': self._guess_os(client), 'number': int(number)}
if forward:
backup['name'] = client
translate = {
'time_start': 'start',
'time_end': 'end',
......@@ -239,13 +257,10 @@ class Burp(Burp1):
'bytes_estimated',
'bytes'
]
query = self.status(
'c:{0}:b:{1}:l:backup_stats\n'.format(client, number)
)
if not query:
if not data:
return ret
try:
back = query['clients'][0]['backups'][0]
back = data['clients'][0]['backups'][0]
except KeyError:
self.logger.warning('No backup found')
return ret
......@@ -274,26 +289,53 @@ class Burp(Burp1):
if name in translate:
name = translate[name]
if counter['name'] in single:
backup[name] = counter['count']
result[name] = counter['count']
else:
backup[name] = {}
result[name] = {}
for (key, val) in counts.items():
if val in counter:
backup[name][key] = counter[val]
result[name][key] = counter[val]
else:
backup[name][key] = 0
if 'start' in backup and 'end' in backup:
backup['duration'] = backup['end'] - backup['start']
result[name][key] = 0
if 'start' in result and 'end' in result:
result['duration'] = result['end'] - result['start']
# convert utc timestamp to local
# example: 1468850307 -> 1468857507
backup['start'] = utc_to_local(backup['start'])
backup['end'] = utc_to_local(backup['end'])
result['start'] = utc_to_local(result['start'])
result['end'] = utc_to_local(result['end'])
# Needed for graphs
if 'received' not in backup:
backup['received'] = 1
if 'received' not in result:
result['received'] = 1
return result
def _parse_backup_stats(self, number, client, forward=False, agent=None):
"""The :func:`burpui.misc.backend.burp2.Burp._parse_backup_stats`
function is used to parse the burp logs.
:param number: Backup number to work on
:type number: int
:param client: Client name to work on
:type client: str
:param forward: Is the client name needed in later process
:type forward: bool
:param agent: What server to ask (only in multi-agent mode)
:type agent: str
return backup
:returns: Dict containing the backup log
"""
ret = {}
backup = {'os': self._guess_os(client), 'number': int(number)}
if forward:
backup['name'] = client
query = self.status(
'c:{0}:b:{1}:l:backup_stats\n'.format(client, number)
)
return self._do_parse_backup_stats(query, backup, number, client, forward, agent)
# TODO: support old clients
# NOTE: this should now be partly done since we fallback to the Burp1 code
......@@ -678,30 +720,12 @@ class Burp(Burp1):
except KeyError:
return False
def get_tree(self, name=None, backup=None, root=None, level=-1, agent=None):
"""See :func:`burpui.misc.backend.interface.BUIbackend.get_tree`"""
def _format_tree(self, data, top, level):
ret = []
if not name or not backup:
return ret
if not root:
top = ''
else:
top = to_unicode(root)
# we know this operation may take a while so we arbitrary increase the
# read timeout
timeout = None
if top == '*':
timeout = max(self.timeout, 300)
query = self.status(
'c:{0}:b:{1}:p:{2}\n'.format(name, backup, top),
timeout
)
if not query:
if not data:
return ret
try:
backup = query['clients'][0]['backups'][0]
backup = data['clients'][0]['backups'][0]
except KeyError:
return ret
for entry in backup['browse']['entries']:
......@@ -735,6 +759,27 @@ class Burp(Burp1):
ret.append(data)
return ret
def get_tree(self, name=None, backup=None, root=None, level=-1, agent=None):
"""See :func:`burpui.misc.backend.interface.BUIbackend.get_tree`"""
if not name or not backup:
return []
if not root:
top = ''
else:
top = to_unicode(root)
# we know this operation may take a while so we arbitrary increase the
# read timeout
timeout = None
if top == '*':
timeout = max(self.timeout, 300)
query = self.status(
'c:{0}:b:{1}:p:{2}\n'.format(name, backup, top),
timeout
)
return self._format_tree(query, top, level)
def get_client_version(self, agent=None):
"""See
:func:`burpui.misc.backend.interface.BUIbackend.get_client_version`
......
......@@ -298,9 +298,14 @@ class Burp(Burp2):
return trio.run(self._async_get_all_backup_logs, client, forward)
return trio.run(self._async_get_backup_logs, number, client, forward)
def _guess_backup_protocol(self, number, client):
"""The :func:`burpui.misc.backend.burp2.Burp._guess_backup_protocol`
function helps you determine if the backup is protocol 2 or 1
async def _async_parse_backup_log(self, number, client):
query = await self._async_status('c:{0}:b:{1}:l:backup\n'.format(client, number))
return self._do_parse_backup_log(query, client)
def _parse_backup_log(self, number, client):
"""The :func:`burpui.misc.backend.burp2.Burp._parse_backup_log`
function helps you determine if the backup is protocol 2 or 1 and various
useful details.
:param number: Backup number to work on
:type number: int
......@@ -308,18 +313,9 @@ class Burp(Burp2):
:param client: Client name to work on
:type client: str
:returns: 1 or 2
:returns: a dict with some useful details
"""
query = self.status('c:{0}:b:{1}:l:backup\n'.format(client, number))
try:
log = query['clients'][0]['backups'][0]['logs']['backup']
for line in log:
if re.search(r'Protocol: 2$', line):
return 2
except KeyError:
# Assume protocol 1 in all cases unless explicitly found Protocol 2
return 1
return 1
return trio.run(self._async_parse_backup_log, number, client)
async def _async_parse_backup_stats(self, number, client, forward=False, agent=None):
"""The :func:`burpui.misc.backend.parallel.Burp._async_parse_backup_stats`
......@@ -343,100 +339,10 @@ class Burp(Burp2):
backup = {'os': await self._async_guess_os(client), 'number': int(number)}
if forward:
backup['name'] = client
translate = {
'time_start': 'start',
'time_end': 'end',
'time_taken': 'duration',
'bytes': 'totsize',
'bytes_received': 'received',
'bytes_estimated': 'estimated_bytes',
'files': 'files',
'files_encrypted': 'files_enc',
'directories': 'dir',
'soft_links': 'softlink',
'hard_links': 'hardlink',
'meta_data': 'meta',
'meta_data_encrypted': 'meta_enc',
'special_files': 'special',
'efs_files': 'efs',
'vss_headers': 'vssheader',
'vss_headers_encrypted': 'vssheader_enc',
'vss_footers': 'vssfooter',
'vss_footers_encrypted': 'vssfooter_enc',
'total': 'total',
'grand_total': 'total',
}
counts = {
'new': 'count',
'changed': 'changed',
'unchanged': 'same',
'deleted': 'deleted',
'total': 'scanned',
'scanned': 'scanned',
}
single = [
'time_start',
'time_end',
'time_taken',
'bytes_received',
'bytes_estimated',
'bytes'
]
query = await self._async_status(
'c:{0}:b:{1}:l:backup_stats\n'.format(client, number)
)
if not query:
return ret
try:
back = query['clients'][0]['backups'][0]
except KeyError:
self.logger.warning('No backup found')
return ret
if 'backup_stats' not in back['logs']:
self.logger.warning('No stats found for backup')
return ret
stats = None
try:
stats = json.loads(''.join(back['logs']['backup_stats']))
except:
stats = back['logs']['backup_stats']
if not stats:
return ret
# server was upgraded but backup comes from an older version
if 'counters' not in stats:
return Burp1._parse_backup_stats(
self,
number,
client,
forward,
stats
)
counters = stats['counters']
for counter in counters:
name = counter['name']
if name in translate:
name = translate[name]
if counter['name'] in single:
backup[name] = counter['count']
else:
backup[name] = {}
for (key, val) in counts.items():
if val in counter:
backup[name][key] = counter[val]
else:
backup[name][key] = 0
if 'start' in backup and 'end' in backup:
backup['duration'] = backup['end'] - backup['start']
# convert utc timestamp to local
# example: 1468850307 -> 1468857507
backup['start'] = utc_to_local(backup['start'])
backup['end'] = utc_to_local(backup['end'])
# Needed for graphs
if 'received' not in backup:
backup['received'] = 1
return backup
return self._do_parse_backup_stats(query, backup, number, client, forward, agent)
# def get_clients_report(self, clients, agent=None):
......@@ -646,42 +552,7 @@ class Burp(Burp2):
'c:{0}:b:{1}:p:{2}\n'.format(name, backup, top),
timeout
)
if not query:
return ret
try:
backup = query['clients'][0]['backups'][0]
except KeyError:
return ret
for entry in backup['browse']['entries']:
data = {}
base = None
dirn = None
if top == '*':
base = os.path.basename(entry['name'])
dirn = os.path.dirname(entry['name'])
if entry['name'] == '.':
continue
else:
data['name'] = base or entry['name']
data['mode'] = self._human_st_mode(entry['mode'])
if re.match('^(d|l)', data['mode']):
data['type'] = 'd'
data['folder'] = True
else:
data['type'] = 'f'
data['folder'] = False
data['inodes'] = entry['nlink']
data['uid'] = entry['uid']
data['gid'] = entry['gid']
data['parent'] = dirn or top
data['size'] = '{0:.1eM}'.format(_hr(entry['size']))
data['date'] = entry['mtime']
data['fullname'] = os.path.join(top, entry['name']) if top != '*' \
else entry['name']
data['level'] = level
data['children'] = []
ret.append(data)
return ret
return self._format_tree(query, top, level)
def get_tree(self, name=None, backup=None, root=None, level=-1, agent=None):
"""See :func:`burpui.misc.backend.interface.BUIbackend.get_tree`"""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment