add: new unit tests

burpui/cli.py

@@ -318,7 +318,7 @@ def compile_translation():
 def setup_burp(bconfcli, bconfsrv, client, host, redis, database,
                plugins, monitor, concurrency, backend, dry):
     """Setup burp client for burp-ui."""
-    if app.config['BACKEND'] not in ['burp2', 'parallel']:
+    if app.config['BACKEND'] not in ['burp2', 'parallel'] and not backend:
         click.echo(
             click.style(
                 "Sorry, you can only setup the 'burp2' and the 'parallel' backends",
@@ -360,7 +360,7 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database,
         msg = str(e)
 
     if msg:
-        _die(msg, 'setup_burp')
+        _die(msg, 'setup-burp')
 
     from .misc.parser.utils import Config
     from .app import get_redis_server

burpui/misc/backend/utils/burp2.py

@@ -84,7 +84,8 @@ class Monitor(object):
                 try:
                     version = subprocess.check_output(
                         cmd,
-                        universal_newlines=True
+                        universal_newlines=True,
+                        stderr=subprocess.DEVNULL
                     ).rstrip()
                 except subprocess.CalledProcessError:
                     pass

burpui/misc/parser/burp2.py

@@ -30,7 +30,7 @@ class Parser(Burp1):
                 'status_port': 'max_status_children',
                 'max_status_children': 'status_port',
             }
-            if self.backend and getattr(self.backend, 'server_version', '') >= '2.1.10':
+            if self.backend and (getattr(self.backend, 'server_version', None) or '') >= '2.1.10':
                 self._pair_associations = {
                     'listen': 'max_children',
                     'max_children': 'listen',
@@ -48,7 +48,7 @@ class Parser(Burp1):
                 'status_port',
                 'max_status_children',
             ]
-            if self.backend and getattr(self.backend, 'server_version', '') >= '2.1.10':
+            if self.backend and (getattr(self.backend, 'server_version', None) or '') >= '2.1.10':
                 self._pair_srv = [
                     'listen',
                     'max_children',

burpui/misc/parser/utils.py

@@ -1611,7 +1611,7 @@ class Config(File):
     def _get(self, key, default=None, raw=False):
         self._refresh()
         try:
-            if key in self._options_for_type('pair'):
+            if key in self._options_for_type('pair') and not raw:
                 obj = self.options[key].get(key)
             else:
                 obj = self.options[key]

setup.cfg

@@ -10,6 +10,8 @@ omit =
     */burpui/_rtfd.py
     */burpui/datastructures.py
     */burpui/misc/parser/openssl.py
+    */burpui/misc/backend/parallel.py
+    */burpui/misc/backend/multi.py
     */burpui/api/async.py
     */burpui/tasks.py
     */burpui/engines/agent.py

tests/burp/CA.cnf

+# simple config for burp_ca
+
+RANDFILE                = /dev/urandom
+CA_DIR                  = /var/lib/burp/CA
+
+
+[ ca ]
+dir                     = $ENV::CA_DIR
+database                = $dir/index.txt
+serial                  = $dir/serial.txt
+certs                   = $dir/certs
+new_certs_dir           = $dir/newcerts
+crlnumber               = $dir/crlnumber.txt
+
+unique_subject          = no
+
+default_md              = sha256
+default_days            = 7300
+default_crl_days        = 7300
+
+#????
+name_opt                = ca_default
+cert_opt                = ca_default
+
+x509_extensions         = usr_cert
+copy_extensions         = copy
+policy                  = policy_anything
+
+[ usr_cert ]
+basicConstraints        = CA:FALSE
+
+[ policy_anything ]
+commonName              = supplied

tests/burp/burp-server.conf

+# This is an example config file for the burp server.
+
+mode = server
+
+# The default addresses to listen on depend upon compile time options.
+# They may be overridden here.
+#address = 0.0.0.0
+port = 4971
+max_children = 5
+# Optionally configure additional ports.
+# port = 5971
+# max_children = 6
+
+# Think carefully before changing the status port address, as it can be used
+# to view the contents of backups.
+#status_address = 127.0.0.1
+# If you do not wish to run a status server at all, comment status_port out.
+status_port = 4972
+max_status_children = 15
+# Optionally configure additional status_ports.
+# status_port = 5972
+# max_status_children = 6
+
+directory = /var/spool/burp
+dedup_group = global
+clientconfdir = clientconfdir
+# Choose the protocol to use.
+# 0 to decide automatically, 1 to force protocol1 mode (file level granularity
+# with a pseudo mirrored storage on the server and optional rsync). 2 forces
+# protocol2 mode (inline deduplication with variable length blocks).
+# Like many other settings, this can be set per client in the clientconfdir
+# files.
+# protocol = 0
+pidfile = /run/burp/burp.server.pid
+hardlinked_archive = 0
+working_dir_recovery_method = delete
+umask = 0022
+syslog = 1
+stdout = 0
+# The following options can restrict what the client can do.
+# Restore clients can override all of these expect for force_backup.
+client_can_delete = 1
+# Set client_can_force_backup to 0 to only allow timed backups.
+client_can_force_backup = 1
+client_can_list = 1
+# Set client_can_restore to 0 if you want restores to only be initialised by
+# the server.
+client_can_restore = 1
+client_can_verify = 1
+# Ratelimit throttles the send speed. Specified in Megabits per second (Mb/s).
+# ratelimit = 1.5
+# Network timeout defaults to 7200 seconds (2 hours).
+# network_timeout = 7200
+
+# Server storage compression. Default is zlib9. Set to zlib0 to turn it off.
+#compression = zlib9
+
+# When the client version does not match the server version, log a warning.
+# Set to 0 to turn it off.
+version_warn = 1
+
+# More configuration files can be read, using syntax like the following
+# (without the leading '# ').
+# . path/to/more/conf
+# Location of autoupgrade files to serve to clients. Leave it commented out
+# to not autoupgrade clients.
+# autoupgrade_dir = /etc/burp/autoupgrade/server
+
+# You can have as many 'keep' lines as you like.
+# For example, if running backups daily, setting 7, 4, 6 will keep
+# 7 daily backups, 4 weekly, and 6 four-weekly backups.
+keep = 7
+# keep = 4
+# keep = 6
+
+# Run as different user/group.
+user = burp
+group = backup
+
+# CA options.
+# If you want your server to be a certificate authority and generate its own
+# certificates, uncomment the following lines. If the directory specified in
+# ca_conf does not exist, the server will create, populate it, and the paths
+# indicated by ssl_cert_ca, ssl_cert, ssl_key and ssl_dhfile below will be
+# overwritten. See docs/burp_ca.txt for more information.
+ca_conf = CA.cnf
+ca_name = burpCA
+ca_server_name = burpserver
+ca_burp_ca = /usr/sbin/burp_ca
+
+# Check for revoked certificates in the certificate revocation list.
+# Turn this off if you use the old ssl_extra_checks_script server script.
+ca_crl_check = 1
+
+# SSL certificate authority - same file on both server and client
+ssl_cert_ca = /var/lib/burp/ssl/server/ssl_cert_ca.pem
+
+# Server SSL certificate
+ssl_cert = /var/lib/burp/ssl/server/ssl_cert-server.pem
+
+# Server SSL key
+ssl_key = /var/lib/burp/ssl/server/ssl_cert-server.key
+
+# Server SSL ciphers
+#ssl_ciphers =
+
+# Server SSL compression. Default is zlib5. Set to zlib0 to turn it off.
+#ssl_compression = zlib5
+
+# SSL key password, for loading a certificate with encryption.
+#ssl_key_password = password
+
+# Server DH file.
+ssl_dhfile = /var/lib/burp/ssl/server/dhfile.pem
+
+timer_script = /usr/share/burp/scripts/timer_script
+# Ensure that 20 hours elapse between backups
+# Available units:
+# s (seconds), m (minutes), h (hours), d (days), w (weeks), n (months)
+timer_arg = 20h
+# Allow backups to start in the evenings and nights during weekdays
+timer_arg = Mon,Tue,Wed,Thu,Fri,00,01,02,03,04,05,19,20,21,22,23
+# Allow more hours at the weekend.
+timer_arg = Sat,Sun,00,01,02,03,04,05,06,07,08,17,18,19,20,21,22,23
+# Note that, if you specify no timebands, the default timer script will never
+# allow backups.
+
+# Uncomment the notify_success_* lines for email notifications of backups that
+# succeeded.
+# In the subject line, the following are substituted:
+# %b - "backup"/"restore"/"verify"
+# %c - client name
+# %w - number of warnings, if any
+#notify_success_script = /usr/share/burp/scripts/notify_script
+#notify_success_arg = sendmail -t
+#notify_success_arg = To: youremail@example.com
+#notify_success_arg = From: burp
+#notify_success_arg = Subject: %b succeeded: %c %w
+# Uncomment the following to have success notifications only if there were
+# warnings.
+#notify_success_warnings_only = 1
+# Uncomment the following to have success notifications only if there were
+# new or changed files.
+#notify_success_changes_only = 1
+
+# Uncomment the following for email notifications of backups that failed.
+#notify_failure_script = /usr/share/burp/scripts/notify_script
+#notify_failure_arg = sendmail -t
+#notify_failure_arg = To: youremail@example.com
+#notify_failure_arg = From: burp
+#notify_failure_arg = Subject: %b failed: %c %w
+
+# The server can run scripts on each connection after authentication and before
+# disconnecting.
+#server_script_pre = /usr/share/burp/scripts/ssl_extra_checks_script
+#server_script_pre_arg = /etc/burp/crl
+#server_script_pre_arg = /etc/burp/burp-server.conf
+#server_script_pre_arg = /usr/share/burp/scripts/server-pre-script.local
+# Set server_script_pre_notify to 1 to have notifications on server_script_pre
+# returning non-zero. Most people will want to leave this off - it could
+# result in a lot of emails because clients normally connect once every 20
+# minutes. Requires notify_failure_script to be set above.
+#server_script_pre_notify = 0
+#server_script_post =
+#server_script_post_arg =
+#server_script_post_arg =
+#server_script_post_run_on_fail=0
+# As for server_script_pre_notify, but for post.
+#server_script_post_notify = 0
+
+# Clients that are able to list and restore files belonging to any other
+# client. If this is too permissive, you may set a restore_client for
+# individual original clients in the individual clientconfdir files.
+# restore_client = someotherclient
+
+# Whether or not the server process should cache the tree when a monitor client
+# is browsing a backup. Advantage: speed. Disadvantage: more memory is used.
+monitor_browse_cache = 1
+
+#restore_client = hydrogen
+
+# Source external configurations
+. conf.d/*.conf

tests/burp/burp.conf

+# This is an example config file for the burp client.
+
+mode = client
+port = 4971
+# A different port to use for restores - see the man page for more options.
+#port_restore = 5971
+status_port = 4972
+server = 127.0.0.1
+password = abcdefgh
+cname = testclient
+# Choose the protocol to use.
+# 0 to decide automatically, 1 to force protocol1 mode (file level granularity
+# with a pseudo mirrored storage on the server and optional rsync). 2 forces
+# protocol2 mode (inline deduplication with variable length blocks).
+# protocol = 0
+pidfile = /var/run/burp.client.pid
+syslog = 0
+stdout = 1
+progress_counter = 1
+# Ratelimit throttles the send speed. Specified in Megabits per second (Mb/s).
+# ratelimit = 1.5
+# Network timeout defaults to 7200 seconds (2 hours).
+# network_timeout = 7200
+# The directory to which autoupgrade files will be downloaded.
+# To never autoupgrade, leave it commented out.
+# autoupgrade_dir=/etc/burp/autoupgrade/client
+# OS path component for the autoupgrade directory on the server.
+# autoupgrade_os=test_os
+# Wait a random number of seconds between 0 and the given number before
+# contacting the server on a timed backup.
+# randomise = 1200
+
+# Set server_can_restore to 0 if you do not want the server to be able to
+# initiate a restore.
+server_can_restore = 0
+
+# Set server_can_override_includes to 0 if you do not want the server to be
+# able to override the local include/exclude list. The default is 1.
+# server_can_override_includes = 1
+
+# Set an encryption password if you do not trust the server with your data.
+# Note that this will mean that network deltas will not be possible. Each time
+# a file changes, the whole file will be transferred on the next backup.
+# encryption_password = My^$pAsswIrD%@
+
+# More configuration files can be read, using syntax like the following
+# (without the leading '# ').
+# . path/to/more/conf
+
+# Run as different user/group.
+# user=graham
+# group=nogroup
+
+cross_filesystem=/home
+cross_all_filesystems=0
+
+# Uncomment the following lines to automatically generate a certificate signing
+# request and send it to the server.
+ca_burp_ca = /usr/sbin/burp_ca
+ca_csr_dir = /etc/burp/CA-client
+
+# SSL certificate authority - same file on both server and client
+ssl_cert_ca = /etc/burp/ssl_cert_ca.pem
+
+# Client SSL certificate
+ssl_cert = /etc/burp/ssl_cert-client.pem
+
+# Client SSL key
+ssl_key = /etc/burp/ssl_cert-client.key
+
+# Client SSL ciphers
+#ssl_ciphers = 
+
+# Client SSL compression. Default is zlib5. Set to zlib0 to turn it off.
+#ssl_compression = zlib5
+
+# SSL key password, for loading a certificate with encryption.
+#ssl_key_password = password
+
+# Common name in the certificate that the server gives us
+ssl_peer_cn = burpserver
+
+# Example syntax for pre/post scripts
+#backup_script_pre=/path/to/a/script
+#backup_script_post=/path/to/a/script
+#restore_script_pre=/path/to/a/script
+#restore_script_post=/path/to/a/script
+
+# The following options specify exactly what to backup.
+# The server will override them if there is at least one 'include=' line on
+# the server side and 'server_can_override_includes=1'.
+include = /home
+#exclude = /home/graham/testdir/librsync-0.9.7/testsuite
+#include = /home/graham/testdir/librsync-0.9.7/testsuite/deep
+#include = /home/graham/xdir
+#exclude = /home/graham/testdir/libr
+# Exclude file names ending in '.vdi' or '.vmdk' (case insensitive)
+#exclude_ext = vdi
+#exclude_ext = vmd
+# Exlude file path matching a regular expression
+# (note that 'include_regex' is not yet implemented)
+#exclude_regex = \.cache
+# Exclude various temporary file systems. You may want to add devfs, devpts,
+# proc, ramfs, etc.
+exclude_fs = sysfs
+exclude_fs = tmpfs
+# Exclude files based on size. Defaults are 0, which means no limit.
+#min_file_size = 0 Mb
+#max_file_size = 0 Mb
+# The content of directories containing a filesystem entry named like this
+# will not be backed up.
+nobackup = .nobackup
+# By default, burp backups up the fifos themselves, rather than reading from
+# them. These two options let you choose a particular fifo to read, or read
+# from all fifos.
+#read_fifo=/path/to/a/fifo
+#read_all_fifos=0
+# The same for block device nodes.
+#read_blockdev=/path/to/a/blockdev
+#read_all_blockdevs=0
+# Exclude files from compression by extension.
+exclude_comp=bz2
+exclude_comp=gz
+# When backing up, whether to enable O_NOATIME when opening files and
+# directories. The default is atime=0, which enables O_NOATIME.
+#atime=1
+# When enabled, this causes problems in the phase1 scan (such as an 'include'
+# being missing) to be treated as fatal errors. The default is 0.
+#scan_problem_raises_error=1

tests/burp/clientconfdir/incexc/example

+# If you add at least one 'include=' line, the server will override the
+# rest of the client options below, which define exactly what to backup.
+# Setting any of the other options here will then also take effect on the
+# client.
+# (This file needs to be included in the clientconfdir file for the client,
+# using the '. path/to/this/file' syntax. Alternatively, these options can
+# be added to the clientconfdir file directly).
+
+include=/home
+exclude=/home/dontwant
+exclude_ext=vdi
+exclude_regex=/\.cache/
+exclude_fs=tmpfs
+exclude_comp=gz
+min_file_size=0
+max_file_size=0
+cross_filesystem=/some/path
+cross_all_filesystems=0
+nobackup=.nobackup
+read_fifo=/some/path/to/a/fifo
+read_all_fifos=0
+split_vss=1
+strip_vss=0
+atime=0
+scan_problem_raises_error=0

tests/burp/clientconfdir/testclient

+password = abcdefgh
+
+# More configuration files can be read, using syntax like the following
+# (without the leading '# ').
+. incexc/*

tests/burp/conf.d/empty.conf

+# Please leave this file empty

tests/burp/conf.d/ipv4.conf

+address = 0.0.0.0
+status_address = 127.0.0.1

tests/burp/conf.d/reset_keep.conf

+keep := 3
+keep =  2

tests/conftest.py

@@ -9,10 +9,11 @@ import shutil
 sys.path.append('{0}/..'.format(os.path.join(os.path.dirname(os.path.realpath(__file__)))))
 
 from burpui import create_app as BUIinit  # noqa
-from burpui.misc.parser.burp1 import Parser  # noqa
+from burpui.misc.parser.burp2 import Parser  # noqa
 
 PWD = os.path.dirname(os.path.realpath(__file__))
 
+
 @pytest.fixture
 def app():
     conf = os.path.join(PWD, 'configs/test_api_prefs.cfg')
@@ -28,13 +29,15 @@ def app():
         db.session.commit()
     yield bui
 
+
 @pytest.fixture
 def parser(app):
     tmpdir = tempfile.mkdtemp()
+    shutil.rmtree(tmpdir)  # remove the dir since copytree will recreate it
     shutil.copytree(os.path.join(PWD, 'burp'), tmpdir)
     confsrv = os.path.join(tmpdir, 'burp-server.conf')
     confcli = os.path.join(tmpdir, 'burp.conf')
-    parser = Parser(app)
+    parser = Parser(app.client)
     parser.init_app(confsrv, confcli)
 
     yield parser

tests/unit/test_parser.py

+import os
+import tempfile
+
+from burpui.misc.parser.utils import OptionMulti, OptionPair
+
+def test_confsrv(parser):
+    confsrv = parser.server_conf
+    stdout = confsrv.get('stdout')
+    keep = confsrv.get('keep')
+    keep_raw = confsrv.get_raw('keep')
+    port = confsrv.get('port')
+    port_raw = confsrv.get_raw('port')
+    assert stdout == 0
+    assert keep == [3, 2]
+    assert isinstance(keep_raw, OptionMulti)
+    assert keep_raw.dump() == 'keep := 3\nkeep = 2'
+    assert port == [4971]
+    assert isinstance(port_raw, OptionPair)
+    assert port_raw.dump() == 'port = 4971\nmax_children = 5'
+
+def test_save_conf(parser):
+    (tmp, tmp_dest) = tempfile.mkstemp()
+    os.close(tmp)
+    confsrv = parser.server_conf
+    confsrv['stdout'] = 1
+    confsrv.store(confsrv.default, tmp_dest, True)
+    with open(tmp_dest) as conf:
+        assert 'stdout = 1\n' in conf.readlines()
+    os.unlink(tmp_dest)