Skip to content
Commits on Source (60)
Expand all Hide whitespace changes
Inline Side-by-side
CHANGELOG.rst
View file @ 8a178930
Changelog
=========
0.4.1 (12/15/2016)
------------------
- **BREAKING**: Use the new Flask's embedded server by default means no more SSL (HTTPS) support without a dedicated application server
- Fix: issue `#156 <https://git.ziirish.me/ziirish/burp-ui/issues/156>`_
- Fix: issue `#157 <https://git.ziirish.me/ziirish/burp-ui/issues/157>`_
- Fix: issue `#165 <https://git.ziirish.me/ziirish/burp-ui/issues/165>`_
- Fix: issue `#176 <https://git.ziirish.me/ziirish/burp-ui/issues/176>`_
- Fix: issue `#181 <https://git.ziirish.me/ziirish/burp-ui/issues/181>`_
- Fix: issue `#182 <https://git.ziirish.me/ziirish/burp-ui/issues/182>`_
- Various bugfix
- `Full changelog <https://git.ziirish.me/ziirish/burp-ui/compare/v0.4.0...v0.4.1>`__
0.4.0 (11/23/2016)
------------------
......
CONTRIBUTORS
View file @ 8a178930
......@@ -7,6 +7,7 @@ Wade Fitzpatrick
Nigel Hathaway
Graham Keeling (main author of Burp)
larsen0815
Benjamin SANS (main author)
Johannes Lerch
slarti5191
Robert Tichy
Benjamin `ziirish` SANS (main author)
Dockerfile
View file @ 8a178930
......@@ -2,7 +2,7 @@ FROM debian:jessie
MAINTAINER hi+burpui@ziirish.me
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y supervisor logrotate locales wget python2.7-dev git gunicorn python-pip cron libffi-dev netcat \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y supervisor logrotate locales gunicorn cron netcat openssl \
&& update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \
&& locale-gen en_US.UTF-8 \
&& dpkg-reconfigure -f noninteractive locales \
......
MANIFEST.in
View file @ 8a178930
include LICENSE
include README.rst
include CHANGELOG.rst
include MANIFEST.in
include CONTRIBUTORS
include burpui/VERSION
include burpui/RELEASE
include requirements.txt
include test-requirements.txt
include share/burpui/etc/burpui.sample.cfg
include share/burpui/etc/buiagent.sample.cfg
include contrib/debian/init.sh
include contrib/debian/bui-celery.init
include contrib/centos/init.sh
include contrib/gunicorn.d/burp-ui
include contrib/gunicorn/burpui_config.py
include bower.json
include .bowerrc
include babel.cfg
graft contrib
graft burpui
graft migrations
global-exclude *.pyc
burpui/__main__.py
View file @ 8a178930
......@@ -22,21 +22,22 @@ sys.path.insert(0, os.path.join(ROOT, '..'))
def parse_args(mode=True, name=None):
if not name:
name = 'burp-ui'
parser = ArgumentParser(prog=name)
mname = 'burp-ui'
parser = ArgumentParser(prog=mname)
parser.add_argument('-v', '--verbose', dest='log', help='increase output verbosity (e.g., -vv is more verbose than -v)', action='count')
parser.add_argument('-d', '--debug', dest='debug', help='enable debug mode', action='store_true') # alias for -v
parser.add_argument('-d', '--debug', dest='debug', help='enable debug mode', action='store_true')
parser.add_argument('-V', '--version', dest='version', help='print version and exit', action='store_true')
parser.add_argument('-c', '--config', dest='config', help='burp-ui configuration file', metavar='<CONFIG>')
parser.add_argument('-l', '--logfile', dest='logfile', help='output logs in defined file', metavar='<FILE>')
parser.add_argument('-i', '--migrations', dest='migrations', help='migrations directory', metavar='<MIGRATIONSDIR>')
parser.add_argument('remaining', nargs=REMAINDER)
if mode:
parser.add_argument('-m', '--mode', dest='mode', help='application mode', metavar='<agent|server|celery|manage>')
parser.add_argument('-m', '--mode', dest='mode', help='application mode', metavar='<agent|server|celery|manage|legacy>')
options, unknown = parser.parse_known_args()
if mode and options.mode and options.mode not in ['celery', 'manage']:
if mode and options.mode and options.mode not in ['celery', 'manage', 'server']:
options = parser.parse_args()
unknown = []
if options.version:
from burpui.app import __title__, __version__, __release__
......@@ -46,44 +47,69 @@ def parse_args(mode=True, name=None):
print(ver)
sys.exit(0)
return options
return options, unknown
def main():
"""
Main function
"""
options = parse_args(mode=True)
options, unknown = parse_args(mode=True)
if not options.mode or options.mode == 'server':
server(options)
server(options, unknown)
elif options.mode == 'agent':
agent(options)
elif options.mode == 'celery':
celery()
elif options.mode == 'manage':
manage()
elif options.mode == 'legacy':
legacy(options, unknown)
else:
print('Wrong mode!')
sys.exit(1)
def server(options=None):
from burpui import create_app
def server(options=None, unknown=None):
from burpui.utils import lookup_file
if unknown is None:
unknown = []
if not options:
options = parse_args(mode=False)
options, unknown = parse_args(mode=False)
env = os.environ
if options.config:
conf = lookup_file(options.config, guess=False)
else:
conf = lookup_file()
if 'BUI_CONFIG' in env:
conf = env['BUI_CONFIG']
else:
conf = lookup_file()
check_config(conf)
server = create_app(conf, options.log, options.logfile, False, debug=options.debug)
if os.path.isdir('burpui'):
env['FLASK_APP'] = 'burpui/cli.py'
else:
env['FLASK_APP'] = 'burpui.cli'
env['BUI_CONFIG'] = conf
env['BUI_VERBOSE'] = str(options.log)
if options.logfile:
env['BUI_LOGFILE'] = options.logfile
if options.debug:
env['BUI_DEBUG'] = '1'
env['FLASK_DEBUG'] = '1'
env['BUI_MODE'] = 'server'
args = [
'flask',
'run'
]
args += unknown
args += [x for x in options.remaining if x != '--']
server.manual_run()
os.execvpe(args[0], args, env)
def agent(options=None):
......@@ -96,7 +122,7 @@ def agent(options=None):
patch_json()
if not options:
options = parse_args(mode=False, name='bui-agent')
options, _ = parse_args(mode=False, name='bui-agent')
conf = ['buiagent.cfg', 'buiagent.sample.cfg']
if options.config:
......@@ -114,7 +140,7 @@ def celery():
parser = ArgumentParser('bui-celery')
parser.add_argument('-c', '--config', dest='config', help='burp-ui configuration file', metavar='<CONFIG>')
parser.add_argument('-m', '--mode', dest='mode', help='application mode', metavar='<agent|server|worker|manage>')
parser.add_argument('-m', '--mode', dest='mode', help='application mode', metavar='<agent|server|worker|manage|legacy>')
parser.add_argument('remaining', nargs=REMAINDER)
options, unknown = parser.parse_known_args()
......@@ -127,22 +153,24 @@ def celery():
conf = env['BUI_CONFIG']
else:
conf = lookup_file()
check_config(conf)
# make conf path absolute
if not conf.startswith('/'):
curr = os.getcwd()
conf = os.path.join(curr, conf)
check_config(conf)
os.chdir(ROOT)
env['BUI_MODE'] = 'celery'
env['BUI_CONFIG'] = conf
args = [
'celery',
'worker',
'-A',
'celery_worker.celery'
'worker.celery'
]
args += unknown
args += [x for x in options.remaining if x != '--']
......@@ -156,7 +184,7 @@ def manage():
parser = ArgumentParser('bui-manage')
parser.add_argument('-c', '--config', dest='config', help='burp-ui configuration file', metavar='<CONFIG>')
parser.add_argument('-i', '--migrations', dest='migrations', help='migrations directory', metavar='<MIGRATIONSDIR>')
parser.add_argument('-m', '--mode', dest='mode', help='application mode', metavar='<agent|server|worker|manage>')
parser.add_argument('-m', '--mode', dest='mode', help='application mode', metavar='<agent|server|worker|manage|legacy>')
parser.add_argument('remaining', nargs=REMAINDER)
options, unknown = parser.parse_known_args()
......@@ -176,6 +204,7 @@ def manage():
else:
migrations = lookup_file('migrations', directory=True)
env['BUI_MODE'] = 'manage'
env['BUI_CONFIG'] = conf
if migrations:
env['BUI_MIGRATIONS'] = migrations
......@@ -193,6 +222,47 @@ def manage():
os.execvpe(args[0], args, env)
def legacy(options=None, unknown=None):
from burpui.utils import lookup_file
if unknown is None:
unknown = []
if not options:
options, unknown = parse_args(mode=False, name='burpui-legacy')
env = os.environ
if options.config:
conf = lookup_file(options.config, guess=False)
else:
if 'BUI_CONFIG' in env:
conf = env['BUI_CONFIG']
else:
conf = lookup_file()
check_config(conf)
env['BUI_MODE'] = 'legacy'
env['BUI_CONFIG'] = conf
if os.path.isdir('burpui'):
env['FLASK_APP'] = 'burpui/cli.py'
else:
env['FLASK_APP'] = 'burpui.cli'
env['BUI_VERBOSE'] = str(options.log)
if options.logfile:
env['BUI_LOGFILE'] = options.logfile
if options.debug:
env['BUI_DEBUG'] = '1'
env['FLASK_DEBUG'] = '1'
args = [
'flask',
'legacy'
]
args += unknown
args += [x for x in options.remaining if x != '--']
os.execvpe(args[0], args, env)
def check_config(conf):
if not conf:
raise IOError('No configuration file found')
......
burpui/_compat.py
View file @ 8a178930
......@@ -33,9 +33,10 @@ def patch_item(module, attr, newitem, newmodule=None):
olditem = getattr(module, attr, NONE)
if olditem is not NONE:
saved.setdefault(module.__name__, {}).setdefault(attr, olditem)
if newmodule:
if newmodule and not getattr(newmodule, 'ori_' + attr, None):
setattr(newmodule, 'ori_' + attr, olditem)
setattr(module, attr, newitem)
if not getattr(newmodule, 'ori_' + attr, None):
setattr(module, attr, newitem)
def patch_module(name, items=None):
......
burpui/api/async.py
View file @ 8a178930
......@@ -14,6 +14,7 @@ import struct
from . import api, cache_key
from .misc import History
from .custom import Resource
from .client import ClientTreeAll, node_fields
from .clients import RunningBackup, ClientsReport
from ..exceptions import BUIserverException
from ..server import BUIServer # noqa
......@@ -208,35 +209,36 @@ def cleanup_expired_sessions():
@celery.task
def cleanup_restore():
tasks = Task.query.filter_by(task='perform_restore').all()
tasks = db.session.query(Task).filter(Task.task == 'perform_restore').filter(datetime.utcnow() > Task.expire).all()
# tasks = Task.query.filter_by(task='perform_restore').all()
for rec in tasks:
if rec.expire and datetime.utcnow() > rec.expire:
logger.info('Task expired: {}'.format(rec))
task = perform_restore.AsyncResult(rec.uuid)
try:
if task.state != 'SUCCESS':
logger.warn(
'Task is not done yet or did not end '
'successfully: {}'.format(task.state)
)
task.revoke(terminate=True)
continue
if not task.result:
logger.warn('The task did not return anything')
continue
server = task.result.get('server')
path = task.result.get('path')
if path:
if server:
if not bui.client.del_file(path, agent=server):
logger.warn("'{}' already removed".format(path))
else:
if os.path.isfile(path):
os.unlink(path)
finally:
db.session.delete(rec)
db.session.commit()
task.revoke()
# if rec.expire and datetime.utcnow() > rec.expire:
logger.info('Task expired: {}'.format(rec))
task = perform_restore.AsyncResult(rec.uuid)
try:
if task.state != 'SUCCESS':
logger.warn(
'Task is not done yet or did not end '
'successfully: {}'.format(task.state)
)
task.revoke(terminate=True)
continue
if not task.result:
logger.warn('The task did not return anything')
continue
server = task.result.get('server')
path = task.result.get('path')
if path:
if server:
if not bui.client.del_file(path, agent=server):
logger.warn("'{}' already removed".format(path))
else:
if os.path.isfile(path):
os.unlink(path)
finally:
db.session.delete(rec)
db.session.commit()
task.revoke()
@celery.task(bind=True)
......@@ -304,6 +306,41 @@ def perform_restore(self, client, backup,
return ret
@celery.task(bind=True)
def load_all_tree(self, client, backup, server=None, user=None):
key = 'load_all_tree-{}-{}-{}'.format(client, backup, server)
ret = cache.cache.get(key)
if ret:
return {
'client': client,
'backup': backup,
'server': server,
'user': user,
'tree': ret
}
lock_name = '{}-{}'.format(self.name, server)
# TODO: maybe do something with old_lock someday
wait_for(lock_name, self.request.id)
try:
ret = ClientTreeAll._get_tree_all(client, backup, server)
except BUIserverException as exp:
raise Exception(str(exp))
finally:
release_lock(lock_name)
cache.cache.set(key, ret, 3600)
return {
'client': client,
'backup': backup,
'server': server,
'user': user,
'tree': ret
}
def force_scheduling_now():
"""Force scheduling some tasks now"""
get_all_backups.delay()
......@@ -879,3 +916,170 @@ class AsyncClientsReport(ClientsReport):
# redirect anymore if the redirection is problematic
return redirect(url_for('api.clients_report', server=server))
return self._get_clients_reports(res, server)
@ns.route('/browseall/<name>/<int:backup>',
'/<server>/browsall/<name>/<int:backup>',
endpoint='async_client_tree_all')
@ns.doc(
params={
'server': 'Which server to collect data from when in' +
' multi-agent mode',
'name': 'Client name',
'backup': 'Backup number',
},
)
class AsyncClientTreeAll(Resource):
"""The :class:`burpui.api.async.AsyncClientTreeAll` resource allows you to
retrieve a list of all the files in a given backup through the celery
worker.
This resource is part of the :mod:`burpui.api.client` module.
An optional ``GET`` parameter called ``serverName`` is supported when
running in multi-agent mode.
"""
parser = ns.parser()
parser.add_argument(
'serverName',
help='Which server to collect data from when in multi-agent mode'
)
@ns.expect(parser)
@ns.doc(
responses={
202: 'Accepted',
405: 'Method not allowed',
403: 'Insufficient permissions',
500: 'Internal failure',
},
)
def post(self, server=None, name=None, backup=None):
"""Launch the tasks that will gather all nodes of a given backup
**POST** method provided by the webservice.
This method returns a :mod:`flask.Response` object.
:param server: Which server to collect data from when in multi-agent
mode
:type server: str
:param name: The client we are working on
:type name: str
:param backup: The backup we are working on
:type backup: int
"""
args = self.parser.parse_args()
server = server or args.get('serverName')
if not bui.client.get_attr('batch_list_supported', False, server):
self.abort(
405,
'Sorry, the requested backend does not support this method'
)
# Manage ACL
if (bui.acl and
(not bui.acl.is_client_allowed(self.username,
name,
server) and not
self.is_admin)):
self.abort(403, 'Sorry, you are not allowed to view this client')
task = load_all_tree.apply_async(
args=[
name,
backup,
server,
self.username
]
)
return {'id': task.id, 'name': 'load_all_tree'}, 202
@ns.route('/browse-status/<task_id>', endpoint='async_browse_status')
@ns.doc(
params={
'task_id': 'The task ID to process',
}
)
class AsyncBrowseStatus(Resource):
"""The :class:`burpui.api.async.AsyncBrowseStatus` resource allows you to
follow a browse task.
This resource is part of the :mod:`burpui.api.async` module.
"""
@ns.doc(
responses={
200: 'Success',
500: 'Task failed',
},
)
def get(self, task_id):
"""Returns the state of the given task"""
task = load_all_tree.AsyncResult(task_id)
if task.state == 'FAILURE':
task.revoke()
err = str(task.result)
self.abort(502, err)
if task.state == 'SUCCESS':
if not task.result:
self.abort(500, 'The task did not return anything')
server = task.result.get('server')
return {
'state': task.state,
'location': url_for(
'.async_do_browse_all',
task_id=task_id,
server=server
)
}
return {'state': task.state}
@ns.route('/get-browse/<task_id>',
'/<server>/get-browse/<task_id>',
endpoint='async_do_browse_all')
@ns.doc(
params={
'task_id': 'The task ID to process',
}
)
class AsyncDoBrowseAll(Resource):
"""The :class:`burpui.api.async.AsyncDoBrowseAll` resource allows you to
retrieve the tree generated by the given task.
This resource is part of the :mod:`burpui.api.async` module.
"""
@ns.marshal_list_with(node_fields, code=200, description='Success')
@ns.doc(
responses={
400: 'Incomplete task',
403: 'Insufficient permissions',
500: 'Task failed',
},
)
def get(self, task_id, server=None):
"""Returns the generated archive"""
task = load_all_tree.AsyncResult(task_id)
if task.state != 'SUCCESS':
if task.state == 'FAILURE':
self.abort(
500,
'Unsuccessful task: {}'.format(task.result.get('error'))
)
self.abort(400, 'Task not processed yet: {}'.format(task.state))
user = task.result.get('user')
dst_server = task.result.get('server')
resp = task.result.get('tree')
if self.username != user or (dst_server and dst_server != server):
self.abort(403, 'Unauthorized access')
task.revoke()
return resp
burpui/api/client.py
View file @ 8a178930
......@@ -428,6 +428,13 @@ class ClientTreeAll(Resource):
self.abort(403, 'Sorry, you are not allowed to view this client')
try:
json = self._get_tree_all(name, backup, server)
except BUIserverException as e:
self.abort(500, str(e))
return json
@staticmethod
def _get_tree_all(name, backup, server):
json = bui.client.get_tree(name, backup, '*', agent=server)
tree = {}
rjson = []
......@@ -487,10 +494,7 @@ class ClientTreeAll(Resource):
for fullname in roots:
rjson.append(tree[fullname])
json = rjson
except BUIserverException as e:
self.abort(500, str(e))
return json
return rjson
@ns.route('/report/<name>',
......
burpui/api/settings.py
View file @ 8a178930
......@@ -308,6 +308,7 @@ class ClientSettings(Resource):
parser_delete = ns.parser()
parser_delete.add_argument('revoke', type=boolean, help='Whether to revoke the certificate or not', default=False, nullable=True)
parser_delete.add_argument('delcert', type=boolean, help='Whether to delete the certificate or not', default=False, nullable=True)
parser_delete.add_argument('keepconf', type=boolean, help='Whether to keep the conf or not', default=False, nullable=True)
@api.disabled_on_demo()
@api.acl_admin_required(message='Sorry, you don\'t have rights to access the setting panel')
......@@ -365,13 +366,15 @@ class ClientSettings(Resource):
args = self.parser_delete.parse_args()
delcert = args.get('delcert', False)
revoke = args.get('revoke', False)
# clear the cache when we remove a client
api.cache.clear()
if bui.config['WITH_CELERY']:
from .async import force_scheduling_now
force_scheduling_now()
return bui.client.delete_client(client, delcert=delcert, revoke=revoke, agent=server), 200
keepconf = args.get('keepconf', False)
if not keepconf:
# clear the cache when we remove a client
api.cache.clear()
if bui.config['WITH_CELERY']:
from .async import force_scheduling_now
force_scheduling_now()
return bui.client.delete_client(client, keepconf=keepconf, delcert=delcert, revoke=revoke, agent=server), 200
@ns.route('/path-expander',
......
burpui/app.py
View file @ 8a178930
......@@ -74,17 +74,59 @@ def get_redis_server(myapp):
return host, port, pwd
def create_db(myapp):
def create_db(myapp, cli=False):
"""Create the SQLAlchemy instance if possible
:param myapp: Application context
:type myapp: :class:`burpui.server.BUIServer`
"""
if myapp.config['WITH_SQL']:
from .ext.sql import db
myapp.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db.init_app(myapp)
return db
try:
from .ext.sql import db
from .models import test_database
from sqlalchemy.exc import OperationalError
from sqlalchemy_utils.functions import database_exists, \
create_database
myapp.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
if not database_exists(myapp.config['SQLALCHEMY_DATABASE_URI']) and \
not cli:
try:
create_database(
myapp.config['SQLALCHEMY_DATABASE_URI']
)
db.init_app(myapp)
with myapp.app_context():
db.create_all()
db.session.commit()
return db
except OperationalError as exp:
myapp.logger.error(
'An error occured, disabling SQL support: '
'{}'.format(str(exp))
)
myapp.config['WITH_SQL'] = False
return None
db.init_app(myapp)
if not cli:
with myapp.app_context():
try:
test_database()
except OperationalError as exp:
if 'no such table' in str(exp):
myapp.logger.critical(
'Your database seem out of sync, you may want '
'to run \'bui-manage db upgrade\'. Disabling '
'SQL support for now.'
)
myapp.config['WITH_SQL'] = False
return None
return db
except ImportError:
myapp.logger.critical(
'Unable to load requirements, you may want to run \'pip '
'install "burp-ui[sql]"\'. Disabling SQL support for now.'
)
myapp.config['WITH_SQL'] = False
return None
......@@ -97,6 +139,7 @@ def create_celery(myapp, warn=True):
"""
if myapp.config['WITH_CELERY']:
from .ext.async import celery
from .exceptions import BUIserverException
host, oport, pwd = get_redis_server(myapp)
odb = 2
if isinstance(myapp.use_celery, basestring):
......@@ -135,7 +178,11 @@ def create_celery(myapp, warn=True):
def __call__(self, *args, **kwargs):
with myapp.app_context():
return TaskBase.__call__(self, *args, **kwargs)
try:
return TaskBase.__call__(self, *args, **kwargs)
except BUIserverException:
# ignore unhandled exceptions in the celery worker
pass
celery.Task = ContextTask
......@@ -154,7 +201,7 @@ def create_celery(myapp, warn=True):
def create_app(conf=None, verbose=0, logfile=None, gunicorn=True,
unittest=False, debug=False):
unittest=False, debug=False, cli=False):
"""Initialize the whole application.
:param conf: Configuration file to use
......@@ -175,6 +222,9 @@ def create_app(conf=None, verbose=0, logfile=None, gunicorn=True,
:param debug: Enable debug mode
:type debug: bool
:param cli: Are we running the CLI
:type cli: bool
:returns: A :class:`burpui.server.BUIServer` object
"""
from flask import g
......@@ -284,7 +334,7 @@ def create_app(conf=None, verbose=0, logfile=None, gunicorn=True,
logger.info('Using configuration: {}'.format(app.config['CFG']))
app.setup(app.config['CFG'], unittest)
app.setup(app.config['CFG'], unittest, cli)
if debug:
app.config.setdefault('TEMPLATES_AUTO_RELOAD', True)
......@@ -321,12 +371,14 @@ def create_app(conf=None, verbose=0, logfile=None, gunicorn=True,
if app.storage and app.storage.lower() != 'default':
try:
# Session setup
if not app.session_db or app.session_db.lower() != 'none':
if not app.session_db or \
app.session_db.lower() not in ['none']:
from redis import Redis
from .ext.session import sess
host, port, pwd = get_redis_server(app)
db = 0
if app.session_db and app.session_db.lower() != 'default':
if app.session_db and \
app.session_db.lower() not in ['redis', 'default']:
try:
(_, _, pwd, host, port, db) = \
parse_db_setting(app.session_db)
......@@ -349,10 +401,12 @@ def create_app(conf=None, verbose=0, logfile=None, gunicorn=True,
sess.init_app(app)
session_manager.backend = red
# Cache setup
if not app.cache_db or app.cache_db.lower() != 'none':
if not app.cache_db or \
app.cache_db.lower() not in ['none']:
host, port, pwd = get_redis_server(app)
db = 1
if app.cache_db and app.cache_db.lower() != 'default':
if app.cache_db and \
app.cache_db.lower() not in ['redis', 'default']:
try:
(_, _, pwd, host, port, db) = \
parse_db_setting(app.cache_db)
......@@ -403,7 +457,7 @@ def create_app(conf=None, verbose=0, logfile=None, gunicorn=True,
babel.init_app(app)
# Create SQLAlchemy if enabled
create_db(app)
create_db(app, cli)
# We initialize the API
api.version = __version__
......
burpui/cli.py
View file @ 8a178930
......@@ -16,38 +16,52 @@ import subprocess
from .app import create_app
DEBUG = os.environ.get('BUI_DEBUG')
DEBUG = os.environ.get('BUI_DEBUG') or os.environ.get('FLASK_DEBUG') or False
if DEBUG and DEBUG.lower() in ['true', 'yes', '1']:
DEBUG = True
else:
DEBUG = False
VERBOSE = os.environ.get('BUI_VERBOSE')
VERBOSE = os.environ.get('BUI_VERBOSE') or 0
if VERBOSE:
try:
VERBOSE = int(VERBOSE)
except ValueError:
VERBOSE = 0
else:
VERBOSE = 0
# UNITTEST is used to skip the burp-2 requirements for modes != server
UNITTEST = os.environ.get('BUI_MODE') not in ['server', 'manage', 'celery', 'legacy']
CLI = os.environ.get('BUI_MODE') not in ['server', 'legacy']
app = create_app(
conf=os.environ.get('BUI_CONFIG'),
verbose=VERBOSE,
logfile=os.environ.get('BUI_LOGFILE'),
debug=DEBUG,
gunicorn=False
gunicorn=False,
unittest=UNITTEST,
cli=CLI
)
if app.config['WITH_SQL']:
try:
from .app import create_db
from .ext.sql import db
from flask_migrate import Migrate
app.config['WITH_SQL'] = True
create_db(app, True)
mig_dir = os.getenv('BUI_MIGRATIONS')
if mig_dir:
migrate = Migrate(app, db, mig_dir)
else:
migrate = Migrate(app, db)
except ImportError:
pass
@app.cli.command()
def legacy():
"""Legacy server for backward compatibility"""
app.manual_run()
@app.cli.command()
......@@ -64,6 +78,8 @@ if app.config['WITH_SQL']:
@click.argument('name')
def create_user(backend, password, ask, verbose, name):
"""Create a new user."""
app.load_modules(False)
click.echo(click.style('[*] Adding \'{}\' user...'.format(name), fg='blue'))
try:
handler = getattr(app, 'uhandler')
......@@ -179,6 +195,8 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, dry):
)
sys.exit(1)
app.load_modules(False)
from .misc.parser.utils import Config
from .app import get_redis_server
import difflib
......@@ -224,61 +242,88 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, dry):
_edit_conf('bconfsrv', bconfsrv, 'burpconfsrv')
if redis:
if ('redis' not in app.conf.options['Production'] or
'redis' in app.conf.options['Production'] and
app.conf.options['Production']['redis'] != redis) and \
app.redis != redis:
app.conf.options['Production']['redis'] = redis
rhost, rport, _ = get_redis_server(app)
DEVNULL = open(os.devnull, 'wb')
ret = subprocess.call(['/bin/nc', '-z', '-w5', str(rhost), str(rport)], stdout=DEVNULL, stderr=subprocess.STDOUT)
if ret == 0:
app.conf.options['Production']['celery'] = 'true'
app.conf.options['Production']['storage'] = 'redis'
try:
# detect missing modules
import redis as redis_client # noqa
import celery # noqa
if ('redis' not in app.conf.options['Production'] or
'redis' in app.conf.options['Production'] and
app.conf.options['Production']['redis'] != redis) and \
app.redis != redis:
app.conf.options['Production']['redis'] = redis
rhost, rport, _ = get_redis_server(app)
DEVNULL = open(os.devnull, 'wb')
ret = subprocess.call(['/bin/nc', '-z', '-w5', str(rhost), str(rport)], stdout=DEVNULL, stderr=subprocess.STDOUT)
if ret == 0:
app.conf.options['Production']['celery'] = 'true'
app.conf.options['Production']['storage'] = 'redis'
app.conf.options['Production']['cache'] = 'redis'
else:
click.echo(
click.style(
'Unable to contact the redis server, disabling it',
fg='yellow'
)
)
app.conf.options['Production']['storage'] = 'default'
app.conf.options['Production']['cache'] = 'default'
if app.use_celery:
app.conf.options['Production']['celery'] = 'false'
app.conf.options['Production']['cache'] = 'redis'
else:
app.conf.options.write()
app.conf._refresh(True)
except ImportError:
click.echo(
click.style(
'Unable to contact the redis server, disabling it',
'Unable to activate redis & celery. Did you ran the '
'\'pip install burp-ui[celery]\' and '
'\'pip install burp-ui[gunicorn-extra]\' commands first?',
fg='yellow'
)
)
app.conf.options['Production']['storage'] = 'default'
app.conf.options['Production']['cache'] = 'default'
if app.use_celery:
app.conf.options['Production']['celery'] = 'false'
app.conf.options.write()
app.conf._refresh(True)
if database:
if ('database' not in app.conf.options['Production'] or
'database' in app.conf.options['Production'] and
app.conf.options['Production']['database'] != database) and \
app.database != database:
app.conf.options['Production']['database'] = database
app.conf.options.write()
app.conf._refresh(True)
try:
from .ext.sql import db # noqa
if ('database' not in app.conf.options['Production'] or
'database' in app.conf.options['Production'] and
app.conf.options['Production']['database'] != database) and \
app.database != database:
app.conf.options['Production']['database'] = database
app.conf.options.write()
app.conf._refresh(True)
except ImportError:
click.echo(
click.style(
'It looks like some dependencies are missing. Did you ran '
'the \'pip install burp-ui[sql]\' command first?',
fg='yellow'
)
)
if dry:
temp = app.conf.options.filename
app.conf.options.filename = orig
after = []
try:
with open(temp) as fil:
after = fil.readlines()
os.unlink(temp)
if not os.path.exists(temp) or os.path.getsize(temp) == 0:
after = conf_orig
else:
with open(temp) as fil:
after = fil.readlines()
os.unlink(temp)
except:
pass
diff = difflib.unified_diff(conf_orig, after, fromfile=orig, tofile='{}.new'.format(orig))
out = ''
for line in diff:
out += _color_diff(line)
click.echo_via_pager(out)
if out:
click.echo_via_pager(out)
bconfcli = bconfcli or app.conf.options['Burp2'].get('bconfcli') or \
getattr(app.client, 'burpconfcli')
......@@ -293,7 +338,7 @@ port = 4971
status_port = 4972
server = ::1
password = abcdefgh
cname = bui
cname = {0}
protocol = 1
pidfile = /tmp/burp.client.pid
syslog = 0
......@@ -304,7 +349,7 @@ server_can_restore = 0
cross_all_filesystems=0
ca_burp_ca = /usr/sbin/burp_ca
ca_csr_dir = /etc/burp/CA-client
ssl_cert_ca = /etc/burp/ssl_cert_ca.pem
ssl_cert_ca = /etc/burp/ssl_cert_ca-client-{0}.pem
ssl_cert = /etc/burp/ssl_cert-bui-client.pem
ssl_key = /etc/burp/ssl_cert-bui-client.key
ssl_key_password = password
......@@ -315,7 +360,7 @@ exclude_fs = tmpfs
nobackup = .nobackup
exclude_comp=bz2
exclude_comp=gz
"""
""".format(client)
if dry:
(_, dest_bconfcli) = tempfile.mkstemp()
......@@ -324,11 +369,9 @@ exclude_comp=gz
parser = app.client.get_parser()
confcli = Config()
data, path, _ = parser._readfile(dest_bconfcli, insecure=True)
parsed = parser._parse_lines(data, path, 'srv')
confcli.add_file(parsed, dest_bconfcli)
confcli = Config(dest_bconfcli, parser, 'srv')
confcli.set_default(dest_bconfcli)
confcli.parse()
if confcli.get('cname') != client:
confcli['cname'] = client
......@@ -340,7 +383,8 @@ exclude_comp=gz
(_, dstfile) = tempfile.mkstemp()
else:
dstfile = bconfcli
parser.store_conf(confcli, dstfile, insecure=True, source=bconfcli)
confcli.store(dest=dstfile, insecure=True)
if dry:
before = []
after = []
......@@ -365,7 +409,8 @@ exclude_comp=gz
out = ''
for line in diff:
out += _color_diff(line)
click.echo_via_pager(out)
if out:
click.echo_via_pager(out)
if not os.path.exists(bconfsrv):
click.echo(
......@@ -377,11 +422,9 @@ exclude_comp=gz
)
sys.exit(1)
confsrv = Config()
data, path, _ = parser._readfile(bconfsrv, insecure=True)
parsed = parser._parse_lines(data, path, 'srv')
confsrv.add_file(parsed, bconfsrv)
confsrv = Config(bconfsrv, parser, 'srv')
confsrv.set_default(bconfsrv)
confsrv.parse()
if host not in ['::1', '127.0.0.1']:
bind = confsrv.get('status_address')
......@@ -429,7 +472,8 @@ exclude_comp=gz
(_, dstfile) = tempfile.mkstemp()
else:
dstfile = bconfsrv
parser.store_conf(confsrv, dstfile, insecure=True, source=bconfsrv)
confsrv.store(dest=dstfile, insecure=True)
if dry:
before = []
after = []
......@@ -448,9 +492,21 @@ exclude_comp=gz
out = ''
for line in diff:
out += _color_diff(line)
click.echo_via_pager(out)
if out:
click.echo_via_pager(out)
if confsrv.get('clientconfdir'):
bconfagent = os.path.join(confsrv.get('clientconfdir'), client)
else:
click.echo(
click.style(
'Unable to find "clientconfdir" option, you will have to '
'setup the agent by your own',
fg='yellow'
)
)
bconfagent = os.devnull
bconfagent = os.path.join(parser.clientconfdir, client)
if not os.path.exists(bconfagent):
agenttpl = """
......@@ -463,18 +519,17 @@ password = abcdefgh
else:
before = []
after = ['{}\n'.format(x) for x in agenttpl.splitlines()]
diff = difflib.unified_diff(before, after, fromfile='None', tofile=agenttpl)
diff = difflib.unified_diff(before, after, fromfile='None', tofile=bconfagent)
out = ''
for line in diff:
out += _color_diff(line)
click.echo_via_pager(out)
if out:
click.echo_via_pager(out)
else:
confagent = Config()
data, path, _ = parser._readfile(bconfagent, insecure=True)
parsed = parser._parse_lines(data, path, 'cli')
confagent.add_file(parsed, bconfagent)
confagent = Config(bconfagent, parser, 'cli')
confagent.set_default(bconfagent)
confagent.parse()
if confagent.get('password') != confcli.get('password'):
click.echo(
......
burpui/misc/backend/burp1.py
View file @ 8a178930
......@@ -346,7 +346,7 @@ class Burp(BUIbackend):
self.logger.error('Cannot guess burp server address')
return False
def status(self, query='\n', agent=None):
def status(self, query='\n', timeout=None, agent=None):
"""See :func:`burpui.misc.backend.interface.BUIbackend.status`"""
result = []
try:
......@@ -1089,11 +1089,11 @@ class Burp(BUIbackend):
return []
return self.parser.path_expander(path, source, client)
def delete_client(self, client=None, delcert=False, revoke=False, agent=None):
def delete_client(self, client=None, keepconf=False, delcert=False, revoke=False, agent=None):
"""See :func:`burpui.misc.backend.interface.BUIbackend.delete_client`"""
if not client:
return [[2, "No client provided"]]
return self.parser.remove_client(client, delcert, revoke)
return self.parser.remove_client(client, keepconf, delcert, revoke)
def clients_list(self, agent=None):
"""See :func:`burpui.misc.backend.interface.BUIbackend.clients_list`"""
......
burpui/misc/backend/burp2.py
View file @ 8a178930
......@@ -167,7 +167,7 @@ class Burp(Burp1):
)
confsrv = G_BURPCONFSRV
if not self.burpbin:
if not self.burpbin and self.app.strict:
# The burp binary is mandatory for this backend
raise Exception(
'This backend *CAN NOT* work without a burp binary'
......@@ -182,12 +182,13 @@ class Burp(Burp1):
"'%s' is not a directory",
tmpdir
)
if tmpdir == G_TMPDIR:
if tmpdir == G_TMPDIR and self.app.strict:
raise IOError(
"Cannot use '{}' as tmpdir".format(tmpdir)
)
tmpdir = G_TMPDIR
if os.path.exists(tmpdir) and not os.path.isdir(tmpdir):
if os.path.exists(tmpdir) and not os.path.isdir(tmpdir) and \
self.app.strict:
raise IOError(
"Cannot use '{}' as tmpdir".format(tmpdir)
)
......@@ -204,15 +205,16 @@ class Burp(Burp1):
cmd,
universal_newlines=True
).rstrip()
if version < BURP_MINIMAL_VERSION:
if version < BURP_MINIMAL_VERSION and self.app.strict:
raise Exception(
'Your burp version ({}) does not fit the minimal'
' requirements: {}'.format(version, BURP_MINIMAL_VERSION)
)
except subprocess.CalledProcessError as exp:
raise Exception(
'Unable to determine your burp version: {}'.format(str(exp))
)
if self.app.strict:
raise Exception(
'Unable to determine your burp version: {}'.format(str(exp))
)
self.client_version = version.replace('burp-', '')
......@@ -269,19 +271,18 @@ class Burp(Burp1):
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=False,
universal_newlines=True,
bufsize=0
)
# wait a little bit in case the process dies on a network error
time.sleep(0.5)
if not self._proc_is_alive():
raise Exception('Unable to spawn burp process')
raise OSError('Unable to spawn burp process')
_, write, _ = select([], [self.proc.stdin], [], self.timeout)
if self.proc.stdin not in write:
self._kill_burp()
raise OSError('Unable to setup burp client')
self.proc.stdin.write('j:pretty-print-off\n')
jso = self._read_proc_stdout()
self.proc.stdin.write('j:pretty-print-off\n'.encode('utf-8'))
jso = self._read_proc_stdout(self.timeout)
if self._is_warning(jso):
self.logger.info(jso['warning'])
......@@ -355,7 +356,7 @@ class Burp(Burp1):
return hur
def _read_proc_stdout(self):
def _read_proc_stdout(self, timeout):
"""reads the burp process stdout and returns a document or None"""
doc = u''
jso = None
......@@ -363,10 +364,10 @@ class Burp(Burp1):
try:
if not self._proc_is_alive():
raise Exception('process died while reading its output')
read, _, _ = select([self.proc.stdout], [], [], self.timeout)
read, _, _ = select([self.proc.stdout], [], [], timeout)
if self.proc.stdout not in read:
raise TimeoutError('Read operation timed out')
doc += self.proc.stdout.readline().rstrip('\n')
doc += self.proc.stdout.readline().decode('utf-8').rstrip('\n')
jso = self._is_valid_json(doc)
# if the string is a valid json and looks like a logline, we
# simply ignore it
......@@ -382,9 +383,10 @@ class Burp(Burp1):
break
return jso
def status(self, query='c:\n', agent=None):
def status(self, query='c:\n', timeout=None, agent=None):
"""See :func:`burpui.misc.backend.interface.BUIbackend.status`"""
try:
timeout = timeout or self.timeout
query = sanitize_string(query.rstrip())
self.logger.info("query: '{}'".format(query))
query = '{0}\n'.format(query)
......@@ -394,8 +396,8 @@ class Burp(Burp1):
_, write, _ = select([], [self.proc.stdin], [], self.timeout)
if self.proc.stdin not in write:
raise TimeoutError('Write operation timed out')
self.proc.stdin.write(query)
jso = self._read_proc_stdout()
self.proc.stdin.write(query.encode('utf-8'))
jso = self._read_proc_stdout(timeout)
if self._is_warning(jso):
self.logger.warning(jso['warning'])
self.logger.debug('Nothing interesting to return')
......@@ -411,7 +413,9 @@ class Burp(Burp1):
except (OSError, Exception) as exp:
msg = 'Cannot launch burp process: {}'.format(str(exp))
self.logger.error(msg)
raise BUIserverException(msg)
if self.app.strict:
raise BUIserverException(msg)
return None
def get_backup_logs(self, number, client, forward=False, agent=None):
"""See
......@@ -854,7 +858,16 @@ class Burp(Burp1):
except (UnicodeDecodeError, AttributeError):
top = root
query = self.status('c:{0}:b:{1}:p:{2}\n'.format(name, backup, top))
# we know this operation may take a while so we arbitrary increase the
# read timeout
timeout = None
if top == '*':
timeout = max(self.timeout, 120)
query = self.status(
'c:{0}:b:{1}:p:{2}\n'.format(name, backup, top),
timeout
)
if not query:
return ret
try:
......
burpui/misc/backend/interface.py
View file @ 8a178930
......@@ -47,13 +47,16 @@ class BUIbackend(with_metaclass(ABCMeta, object)):
"""
@abstractmethod
def status(self, query='\n', agent=None):
def status(self, query='\n', timeout=None, agent=None):
"""The :func:`burpui.misc.backend.interface.BUIbackend.status` method is
used to send queries to the Burp server
:param query: Query to send to the server
:type query: str
:param timeout: Query timeout in seconds
:type timeout: int
:param agent: What server to ask (only in multi-agent mode)
:type agent: str
......@@ -642,13 +645,16 @@ class BUIbackend(with_metaclass(ABCMeta, object)):
raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover
@abstractmethod
def delete_client(self, client=None, delcert=False, revoke=False, agent=None):
def delete_client(self, client=None, keepconf=False, delcert=False, revoke=False, agent=None):
"""The :func:`burpui.misc.backend.interface.BUIbackend.delete_client`
function is used to delete a client from burp's configuration.
:param client: The name of the client to remove
:type client: str
:param keepconf: Whether to keep the conf (in order to just revoke/delete the certs for instance)
:type keepconf: bool
:param delcert: Whether to delete the associated certificate
:type delcert: bool
......
burpui/misc/backend/multi.py
View file @ 8a178930
......@@ -419,12 +419,16 @@ class NClient(BUIbackend):
res = '[]'
err = None
notimeout = False
timeout = self.timeout
if not data:
raise BUIserverException('Missing data')
data['password'] = self.password
# manage long running operations
if data['func'] in ['restore_files', 'get_file', 'del_file']:
notimeout = True
if data['func'] == 'get_tree' and data['args'].get('root') == '*':
# arbitrary raise timeout
timeout = max(timeout, 120)
try:
# don't need a context manager here
if data['func'] == 'get_file':
......@@ -434,7 +438,7 @@ class NClient(BUIbackend):
if not self.setup(gsock.sock, gsock, raw):
return res
return gsock.sock
with Gsocket(self.host, self.port, self.ssl, self.timeout, notimeout) as (sock, gsock):
with Gsocket(self.host, self.port, self.ssl, timeout, notimeout) as (sock, gsock):
try:
raw = json.dumps(data)
if not self.setup(gsock.sock, gsock, raw):
......
burpui/misc/parser/burp1.py
View file @ 8a178930
This diff is collapsed.
burpui/misc/parser/doc.py
View file @ 8a178930
......@@ -754,7 +754,9 @@ class Doc(BUIparser):
u'server_script_post': __(u"Path to a script to run on the server"
" before the client disconnects. The"
" arguments to it are 'post', '(client"
" command)', 'reserved3' to 'reserved5', and"
" command)', '(client name), '(0 or 1 for"
" success or failure)', '(timer script exit"
" code)', and"
" then arguments defined by"
" server_script_post_arg. This command and"
" related options can be overriddden by the"
......@@ -778,7 +780,9 @@ class Doc(BUIparser):
" each successfully authenticated connection"
" but before any work is carried out. The"
" arguments to it are 'pre', '(client"
" command)', 'reserved3' to 'reserved5', and"
" command)', '(client name)', '(0 or 1 for"
" success or failure)', '(timer script exit"
" code)', and"
" then arguments defined by"
" server_script_pre_arg. If the script"
" returns non-zero, the task asked for by the"
......
burpui/misc/parser/interface.py
View file @ 8a178930
......@@ -127,7 +127,7 @@ class BUIparser(with_metaclass(ABCMeta, object)):
@abstractmethod
def store_conf(self, data, conf=None, client=None, mode='srv',
insecure=False, source=None):
insecure=False):
""":func:`burpui.misc.parser.interface.BUIparser.store_conf` is used to
store the configuration from the web-ui into the actual configuration
files.
......@@ -149,9 +149,6 @@ class BUIparser(with_metaclass(ABCMeta, object)):
:param insecure: Used for the CLI
:type insecure: bool
:param source: Used for the CLI
:type source: str
:returns: A list of notifications to return to the UI (success or
failure)
......@@ -210,13 +207,16 @@ class BUIparser(with_metaclass(ABCMeta, object)):
) # pragma: no cover
@abstractmethod
def remove_client(self, client=None, delcert=False, revoke=False):
def remove_client(self, client=None, keepconf=False, delcert=False, revoke=False):
""":func:`burpui.misc.parser.interface.BUIparser.remove_client` is used
to delete a client from burp's configuration.
:param client: The name of the client to remove
:type client: str
:param keepconf: Whether to keep the conf (in order to just revoke/delete the cert)
:param keepconf: bool
:param delcert: Whether to delete the associated certificate
:type delcert: bool
......
burpui/misc/parser/utils.py
View file @ 8a178930
This diff is collapsed.