rework tests/build system

.gitignore

@@ -4,6 +4,7 @@
 burpui-dev.cfg*
 burpui/RELEASE
 devel.sh
+clean.sh
 *.egg*
 .tox
 .reports
@@ -18,7 +19,5 @@ _build
 .tags
 celerybeat-schedule
 Pipfile*
-pkgs/burp-ui-sql/burpui_sql/VERSION
-pkgs/burp-ui-extra/burpui_extra/VERSION
 pkgs/burp-ui-agent/burpui_agent
 pkgs/burp-ui-monitor/burpui_monitor

.gitlab-ci.yml

@@ -3,6 +3,10 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
   DOCKER_DRIVER: overlay2
   DOCKER_HOST: tcp://localhost:2375
+  BURP_VERSION: 2.2.18
+  PG_VERSION: 10
+  FLAKE8_IGNORE: E501,E722,W605,W504
+  SRC_DIR: burpui
 
 image: docker:git
 
@@ -14,20 +18,43 @@ stages:
   - build
   - deploy
 
-test:lint:
+test:lint:3.6:
   stage: test
-  image: python:3.6
+  image: python:3.6-alpine
   script:
   - pip install flake8 pylint
-  - make flake8
+  - flake8 --ignore=$FLAKE8_IGNORE $SRC_DIR
   tags:
   - lint
   except:
   - tags
 
-test:py3.6:
+test:lint:3.7:
   stage: test
-  image: python:3.6
+  image: python:3.7-alpine
+  script:
+  - pip install flake8 pylint
+  - flake8 --ignore=$FLAKE8_IGNORE $SRC_DIR
+  tags:
+  - lint
+  except:
+  - tags
+
+test:lint:3.8:
+  stage: test
+  image: python:3.8-rc-alpine
+  script:
+  - pip install flake8 pylint
+  - flake8 --ignore=$FLAKE8_IGNORE $SRC_DIR
+  tags:
+  - lint
+  except:
+  - tags
+  allow_failure: true
+
+test:py:3.6:
+  stage: test
+  image: python:3.6-alpine
   script:
   - pip install tox
   - mkdir .reports
@@ -40,11 +67,42 @@ test:py3.6:
     reports:
       junit: .reports/burpui.junit.xml
 
+test:py:3.7:
+  stage: test
+  image: python:3.7-alpine
+  script:
+  - pip install tox
+  - mkdir .reports
+  - tox -e py37
+  tags:
+  - docker
+  except:
+  - tags
+  artifacts:
+    reports:
+      junit: .reports/burpui.junit.xml
+
+test:py:3.8:
+  stage: test
+  image: python:3.8-rc-alpine
+  script:
+  - pip install tox
+  - mkdir .reports
+  - tox -e py38
+  tags:
+  - docker
+  except:
+  - tags
+  artifacts:
+    reports:
+      junit: .reports/burpui.junit.xml
+  allow_failure: true
+
 build:py3:
   stage: build
-  image: python:3.6
+  image: python:3.6-alpine
   script:
-  - /bin/bash tests/run_build.sh
+  - tests/build.sh
   tags:
   - build
   only:
@@ -59,10 +117,10 @@ build:py3:
 
 build:doc:
   stage: build
-  image: python:3.6
+  image: python:3.6-alpine
   script:
   - pip install -U .[rtd]
-  - make doc
+  - cd doc && make html
   tags:
   - build
   only:
@@ -78,11 +136,11 @@ build:docker:latest:
   script:
   - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest -f docker/Dockerfile .
-  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 .)
-  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18 .)
+  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION .)
+  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION .)
   - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION
   tags:
   - registry
   only:
@@ -93,11 +151,11 @@ build:docker:release:
   script:
   - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG -f docker/Dockerfile .
-  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 .)
-  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18 .)
+  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION .)
+  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION .)
   - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION
   only:
   - tags
   tags:
@@ -108,11 +166,11 @@ build:docker:stable:
   script:
   - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:stable -f docker/Dockerfile .
-  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 .)
-  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18 .)
+  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION .)
+  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION .)
   - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:stable
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION
   only:
   - stable@ziirish/burp-ui
   tags:

requirements.txt

 trio==0.11.0
-Flask==1.0.2
+Flask==1.1.1
 Flask-Login==0.4.1
 Flask-Bower==1.3.0
 Flask-Babel==0.12.2
 Flask-WTF==0.14.2
 flask-restplus==0.12.1
-Flask-Caching==1.6.0
+Flask-Caching==1.7.2
 WTForms==2.1
-arrow==0.13.1
+arrow==0.14.2
 pluginbase==1.0.0
 tzlocal==1.5.1
 pyOpenSSL==19.0.0
 configobj==5.0.6
 async_generator
 Click==7.0
-werkzeug>=0.15.0

setup.py

@@ -301,6 +301,7 @@ setup(
         (os.path.join(contrib, 'gunicorn'), ['contrib/gunicorn/burpui_gunicorn.py']),
         (os.path.join(contrib, 'systemd'), ['contrib/systemd/bui-agent.service', 'contrib/systemd/bui-celery.service', 'contrib/systemd/bui-gunicorn.service']),
     ] + migrations,
+    python_requires=">=3.6",
     install_requires=requires,
     extras_require={
         'ldap_authentication': ['ldap3'],

tests/run_build.sh → tests/build.sh

-#!/bin/bash
+#!/bin/sh
 
-PYTHON=$(which python)
-
-echo "test requirements"
-
-[ -x "$PYTHON" ] && {
-    echo "python seems to be installed"
-} || {
-    echo "python is missing..."
-    exit 1
-}
-
-pip install -r requirements.txt
 pip install wheel
 
-
 echo "building dist"
-$PYTHON setup.py sdist bdist_wheel bdist_egg
+python setup.py sdist bdist_wheel bdist_egg
 mkdir meta
 cd pkgs
 for pkg in *
 do
     [ "$pkg" = "burp-ui-tpl" -o -f "$pkg" ] && continue
     cd $pkg
-    $PYTHON setup.py sdist bdist_wheel bdist_egg
-    find dist -exec cp "{}" ../../meta/ \;
+    python setup.py sdist bdist_wheel bdist_egg
+    find dist -type f -exec cp "{}" ../../meta/ \;
     cd ..
 done
 cd ..

tox.ini

 [tox]
-envlist = py{36}
+envlist = py{36,37,38}
 
 [testenv]
 description = Run burp-ui unit tests