add a new --pool-size option to the setup-burp command

burpui/cli.py

@@ -316,13 +316,15 @@ def compile_translation():
 @click.option('-m', '--monitor', default=None,
               help='bui-monitor configuration file')
 @click.option('-C', '--concurrency', default=None, type=click.INT,
-              help='Number of concurrent parallel processes to spawn')
+              help='Number of concurrent requests addressed to the monitor')
+@click.option('-P', '--pool-size', 'pool', default=None, type=click.INT,
+              help='Number of burp-client processes to spawn in the monitor')
 @click.option('-B', '--backend', default=None,
               help='Switch to another backend', type=click.Choice(['burp2', 'parallel']))
 @click.option('-n', '--dry', is_flag=True,
               help='Dry mode. Do not edit the files but display changes')
 def setup_burp(bconfcli, bconfsrv, client, host, redis, database,
-               plugins, monitor, concurrency, backend, dry):
+               plugins, monitor, concurrency, pool, backend, dry):
     """Setup burp client for burp-ui."""
     if app.config['BACKEND'] not in ['burp2', 'parallel'] and not backend:
         err("Sorry, you can only setup the 'burp2' and the 'parallel' backends")
@@ -340,6 +342,13 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database,
                 ' count might cause you some troubles'
             )
 
+    if pool and concurrency:
+        if concurrency > pool:
+            warn(
+                'Warning: setting a concurrency level higher than the available'
+                ' processes in the monitor is not recommended'
+            )
+
     is_parallel = app.config['BACKEND'] == 'parallel' or (backend and backend == 'parallel')
 
     # enforce burp2 backend for the configuration
@@ -451,9 +460,9 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database,
     if refresh:
         app.conf._refresh(True)
 
-    if monitor and concurrency:
+    if monitor and pool:
         refresh = False
-        refresh |= _edit_conf('pool', concurrency, None, 'Global', None, monconf)
+        refresh |= _edit_conf('pool', pool, None, 'Global', None, monconf)
         refresh |= _edit_conf('bconfcli', bconfcli, None, 'Burp', None, monconf)
         if refresh:
             monconf._refresh(True)
@@ -709,18 +718,19 @@ exclude_comp=gz
                 )
             )
 
+    MAX_STATUS_CHILDREN = pool if pool is not None else 15
     status_port = confsrv.get('status_port', [4972])
     do_warn = False
     if 'max_status_children' not in confsrv:
         info(
             'We need to set the number of \'max_status_children\'. '
-            'Setting it to 15.'
+            'Setting it to {}.'.format(MAX_STATUS_CHILDREN)
         )
-        confsrv['max_status_children'] = 15
+        confsrv['max_status_children'] = MAX_STATUS_CHILDREN
     elif confsrv.version and confsrv.version < BURP_BIND_MULTIPLE:
         max_status_children = confsrv.get('max_status_children')
-        if not max_status_children or max_status_children < 15:
-            confsrv['max_status_children'] = 15
+        if not max_status_children or max_status_children < MAX_STATUS_CHILDREN:
+            confsrv['max_status_children'] = MAX_STATUS_CHILDREN
             do_warn = True
     else:
         max_status_children = confsrv.get('max_status_children', [])
@@ -730,16 +740,16 @@ exclude_comp=gz
                     bind_index = idx
                     break
         if bind_index >= 0 and len(max_status_children) >= bind_index:
-            if max_status_children[bind_index] < 15:
-                confsrv['max_status_children'][bind_index] = 15
+            if max_status_children[bind_index] < MAX_STATUS_CHILDREN:
+                confsrv['max_status_children'][bind_index] = MAX_STATUS_CHILDREN
                 do_warn = True
         else:
-            if max_status_children[-1] < 15:
-                confsrv['max_status_children'][-1] = 15
+            if max_status_children[-1] < MAX_STATUS_CHILDREN:
+                confsrv['max_status_children'][-1] = MAX_STATUS_CHILDREN
                 do_warn = True
 
     if do_warn:
-        warn('We need to raise the number of \'max_status_children\'.')
+        warn('We need to raise the number of \'max_status_children\' to {}.'.format(MAX_STATUS_CHILDREN))
 
     if 'restore_client' not in confsrv:
         confsrv['restore_client'] = client

docker/docker-alpine/assets/init

@@ -22,6 +22,7 @@ DATABASE_URL=${DATABASE_URL:-postgresql://burpui:burpui@pgsql/burpuidb}
 GUNICORN_WORKERS=${GUNICORN_WORKERS:-$(getconf _NPROCESSORS_ONLN)}
 GUNICORN_WORKER_CLASS=${GUNICORN_WORKER_CLASS:-sync}
 ASYNC_POOL_SIZE=${ASYNC_POOL_SIZE:-$(getconf _NPROCESSORS_ONLN)}
+ASYNC_CONCURRENCY=${ASYNC_CONCURRENCY:-$(test $ASYNC_POOL_SIZE -gt 2 && echo 2 || echo 1)}
 TIMEZONE=${TIMEZONE:-Europe/Paris}
 
 trap appStop SIGINT SIGTERM
@@ -68,7 +69,7 @@ appStart () {
 
   LOGFILE=$(doas burpui mktemp)
   echo "Setting up burp & burp-ui:"
-  COMMAND="bui-manage -c $BURPUI_CONFIG setup-burp --burp-conf-cli $BURP_CLIENT_CONFIG --burp-conf-serv $BURP_SERVER_CONFIG --host $BURP_SERVER_ADDR --client $BURPUI_CLIENT_NAME --redis $REDIS_SERVER --database $DATABASE_URL --plugins $BURPUI_PLUGINS --monitor $BURPUI_MONITOR_CONFIG --concurrency $ASYNC_POOL_SIZE --backend $BURPUI_BACKEND"
+  COMMAND="bui-manage -c $BURPUI_CONFIG setup-burp --burp-conf-cli $BURP_CLIENT_CONFIG --burp-conf-serv $BURP_SERVER_CONFIG --host $BURP_SERVER_ADDR --client $BURPUI_CLIENT_NAME --redis $REDIS_SERVER --database $DATABASE_URL --plugins $BURPUI_PLUGINS --monitor $BURPUI_MONITOR_CONFIG --concurrency $ASYNC_CONCURRENCY --pool-size $ASYNC_POOL_SIZE --backend $BURPUI_BACKEND"
   echo $COMMAND
   doas burpui "$COMMAND 2>&1 | tee $LOGFILE"
   ret=$?

docker/docker-compose.yml

@@ -70,5 +70,6 @@ services:
   #    - GUNICORN_WORKER_CLASS=sync
   #    - REDIS_SERVER=redis:6379
   #    - BURP_SERVER_ADDR=burp-server
-  #    - ASYNC_POOL_SIZE=2
+  #    - ASYNC_POOL_SIZE=4
+  #    - ASYNC_CONCURRENCY=2
   #    - TIMEZONE=Europe/Paris

docs/docker.rst

@@ -13,38 +13,51 @@ provided. There are a few variables supported to setup your system:
 
 
  - **BURPUI_CONFIG** - Specify where the `Burp-UI`_ configuration file is
-   located. It defaults to "/etc/burp/burpui.cfg".
+   located. It defaults to ``/etc/burp/burpui.cfg``.
+ - **BURPUI_MONITOR_CONFIG** - Specify where the `bui-monitor`_ configuration
+   file is located. It defaults to ``/etc/burp/buimonitor.cfg``.
  - **BURPUI_VERBOSE** - Specify the log verbosity (between 0 and 4). It defaults
    to 0.
  - **BURPUI_CLIENT_NAME** - Specify the name of the burp client that will be
-   used by `Burp-UI`_. It defaults to "bui".
+   used by `Burp-UI`_. It defaults to ``bui``.
  - **BURPUI_UID** - uid of the *burpui* user you want to map in your host. It
-   defaults to 5337.
+   defaults to ``5337``.
  - **BURPUI_GID** - gid of the *burpui* group you want to map in your host. It
-   defaults to 5337.
- - **BURPUI_PLUGINS** - Directory where to look for plugins. It defaults to none
-   which means no plugins will be loaded.
+   defaults to ``5337``.
+ - **BURPUI_PLUGINS** - Directory where to look for plugins. It defaults to
+   ``none`` which means no plugins will be loaded.
  - **BURPUI_WS_WORKERS** - How many WebSocket servers to spawn. Defaults to the
    number of CPU cores/sockets/threads found.
  - **BURPUI_RP_SCHEME** - HTTP Scheme to set for the reverse-proxy. If you are
-   behind a reverse-proxy that provides SSL, you should set this to *https*
+   behind a reverse-proxy that provides SSL, you should set this to ``https``
    which is the default value.
  - **BURP_CLIENT_CONFIG** - Specify the path of the burp client configuration
-   file to use for the `Burp-UI`_ client. It defaults to "/tmp/burp.conf". It
+   file to use for the `Burp-UI`_ client. It defaults to ``/tmp/burp.conf``. It
    means you won't have access to it outside of the container. It is intended
-   to not override the */etc/burp/burp.conf* file if you already use it.
+   to not override the ``/etc/burp/burp.conf`` file if you already use it.
  - **BURP_SERVER_CONFIG** - Specify the path of the burp-server configuration
-   file. It defaults to "/etc/burp/burp-server.conf".
+   file. It defaults to ``/etc/burp/burp-server.conf``.
  - **DATABASE_URL** - Specify the URL of the database to connect to. It defaults
-   to "sqlite:////var/lib/burpui/store.db".
+   to ``postgresql://burpui:burpui@pgsql/burpuidb`` which is the default db
+   shipped with the *docker-compose.yml*.
  - **GUNICORN_WORKERS** - How many gunicorn workers to spawn. Defaults to the
    number of CPU cores/sockets/threads found.
+ - **GUNICORN_WORKER_CLASS** - What kind of gunicorn workers to use. Defaults to
+   ``sync`` which is the only worker supported if you are running the
+   ``parallel`` backend.
  - **REDIS_SERVER** - Specify the address of the redis server. It defaults to
-   "redis:6379".
+   ``redis:6379``.
  - **BURP_SERVER_ADDR** - Specify the address of the burp-server status port.
    If set to "auto", we will use the address of the docker host.
    Make sure your status port is listening on this interface.
-   Defaults to "burp-server" which is the burp container in the stack.
+   Defaults to ``burp-server`` which is the burp container shipped with the
+   *docker-compose.yml*.
+ - **ASYNC_POOL_SIZE** - Size of the `bui-monitor`_ pool size. Defaults to the
+   number of CPU cores/sockets/threads found.
+ - **ASYNC_CONCURRENCY** - Number of parallel requests to send to the
+   `bui-monitor`_ when using the ``parallel`` backend (defined in
+   **BURPUI_BACKEND**). The value should not exceed the **ASYNC_POOL_SIZE** and
+   defaults to ``2`` if **ASYNC_POOL_SIZE** > 2 else it defaults to ``1``.
  - **TIMEZONE** - Specify the timezone of your burp-server. It defaults to
    Europe/Paris.
 
@@ -150,5 +163,6 @@ running:
 
 
 .. _Burp-UI: https://git.ziirish.me/ziirish/burp-ui
+.. _bui-monitor: buimonitor.html
 .. _docker: https://docs.docker.com/engine/installation/linux/ubuntulinux/
 .. _docker-compose: https://docs.docker.com/compose/install/