variables:
TZ: "America/New_York"
# Configure postgres service (https://hub.docker.com/_/postgres/)
POSTGRES_DB: custom_db
POSTGRES_USER: custom_user
POSTGRES_PASSWORD: custom_pass
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
APT_CACHE_DIR: "$CI_PROJECT_DIR/apt-cache"
DEBIAN_FRONTEND: noninteractive
IMAGE_TAG: 'egrep -o "[0-9]\.([0-9]|[0-9][0-9])\.[0-9]" faraday/__init__.py'
## ENV_VARS LIST
# FULL_TEST = Test all jobs
# BUILD_TEST = Test default and build jobs
# DAILY_TEST = Run every night at 03 am
cache:
paths:
- "$CI_PROJECT_DIR/.cache/pip"
- "$CI_PROJECT_DIR/apt-cache"
before_script:
- mkdir -pv $APT_CACHE_DIR
stages:
- pre_testing
- testing
- post_testing
- build_faraday
- build
- distro_testing
services:
- postgres:latest
closure_compiler:
tags:
- faradaytests
image:
name: jborza/closure-compiler
entrypoint: ["/bin/sh", "-c"]
stage: pre_testing
script:
- /opt/cc.sh faraday/server/www/scripts
except:
variables:
- $BUILD_TEST
merge_conflict_check:
tags:
- faradaytests
image: python:3
stage: pre_testing
allow_failure: true
script:
- git config --global user.email "[email protected]"
- git config --global user.name "Mergerbot"
- python3 merge-conflict-detector.py
pylint:
tags:
- faradaytests
image: registry.gitlab.com/faradaysec/integrationrepo
stage: pre_testing
script:
- pip3 install virtualenv
- virtualenv -p python3 faraday_venv
- source faraday_venv/bin/activate
- pip install --upgrade -r requirements.txt
- python3 setup.py install
# pylint slow issue https://github.com/PyCQA/pylint/issues/2765
- pip install pylint isort==4.3.4
- pylint --rcfile=.pylintrc faraday --ignore=faraday_venv | tee pylint.txt
- score=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' pylint.txt)
#- anybadge --label pylint --value=$score --file pylint.svg 4=red 6=orange 8=yellow 10=green
artifacts:
paths:
- pylint.svg
- pylint3.svg
except:
variables:
- $BUILD_TEST
postgresql_test:
image: registry.gitlab.com/faradaysec/integrationrepo
tags:
- faradaytests
stage: testing
coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
script:
- export LC_ALL=C.UTF-8
- export LANG=C.UTF-8
- pip3 install virtualenv
- virtualenv -p python3 faraday_venv
- source faraday_venv/bin/activate
- pip install pip -U
- pip install --upgrade -r requirements.txt
- python3 setup.py install
- pip install --upgrade responses pytest-xdist pytest-cov
- pip install --upgrade -r requirements_dev.txt
- mkdir -p ~/.faraday/config
- cp tests/data/server.ini ~/.faraday/config
- mkdir run_from
- cd run_from && pytest ../tests -v --capture=sys --cov=../faraday/server --color=yes --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB
artifacts:
when: on_failure
paths:
- ~/.faraday/logs/faraday-server.log
except:
variables:
- $BUILD_TEST
sqlite_test:
image: registry.gitlab.com/faradaysec/integrationrepo
tags:
- faradaytests
stage: testing
coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
script:
- export LC_ALL=C.UTF-8
- export LANG=C.UTF-8
- pip3 install virtualenv
- virtualenv -p python3 faraday_venv
- source faraday_venv/bin/activate
- pip install pip -U
- pip install --upgrade -r requirements.txt
- python3 setup.py install
- pip install --upgrade responses pytest-xdist pytest-cov
- pip install --upgrade -r requirements_dev.txt
- mkdir -p ~/.faraday/config
- cp tests/data/server.ini ~/.faraday/config
- mkdir run_from
- cd run_from && pytest ../tests --capture=sys -v --cov=../faraday/server --color=yes --disable-warnings
artifacts:
when: on_failure
paths:
- dist/*
except:
variables:
- $BUILD_TEST
build_nix_python3:
image: nixorg/nix
stage: build_faraday
script:
- nix-channel --add https://nixos.org/channels/nixos-19.09 nixpkgs
- nix-channel --update
- nix-build buildpth.nix
- nix-build '<nixpkgs>' -A gnome3.vte -o result-vte
- nix-build '<nixpkgs>' -A gtk3 -o result-gtk
- tar cf /py3.tar $(nix-store --query --requisites $(readlink result) $(readlink result-vte)) $(readlink result-gtk)
- mkdir /opt
- nix-env -i findutils postgresql
- "nix-shell -p python37Packages.virtualenv --command 'virtualenv /opt/faraday'"
- source /opt/faraday/bin/activate
- "nix-shell -p libxml2 libxslt zlib cairo gobject-introspection glib pkgconfig --run 'SOURCE_DATE_EPOCH=$(date +%s) /opt/faraday/bin/pip install -r requirements.txt'"
- python setup.py install
- cp result /opt/faraday/lib/python3.7/site-packages/nix.pth
- mv /opt/faraday/bin/faraday-client /opt/faraday/bin/_faraday_client_novte
- 'nix-shell -p makeWrapper --command "makeWrapper /opt/faraday/bin/_faraday_client_novte /opt/faraday/bin/faraday-client --prefix GI_TYPELIB_PATH : $(find /nix/store -name "girepository-1.0" | tr "\n" ":")"'
- 'nix-shell -p makeWrapper --command "makeWrapper /opt/faraday/bin/python /opt/faraday/bin/test --prefix GI_TYPELIB_PATH : $(find /nix/store -name "girepository-1.0" | tr "\n" ":")"'
- /opt/faraday/bin/test -c "import gi;gi.require_version('Gtk', '3.0');gi.require_version('Vte', '2.91');from gi.repository import Gio, Gtk, GdkPixbuf, Vte, GLib, GObject, Gdk" # Test if GTK will work
- tar rvf /py3.tar /opt/faraday
- mv /py3.tar $CI_PROJECT_DIR
artifacts:
name: python3
paths:
- py3.tar
expire_in: 15 days # in the future we don't need to expire this.
only:
variables:
- $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
- $CI_COMMIT_TAG
- $BUILD_TEST
- $FULL_TEST
- $DAILY_TEST
generate_build_file:
image: registry.gitlab.com/faradaysec/integrationrepo
stage: build_faraday
script:
- "/bin/mkdir faraday_copy"
- "/usr/bin/rsync -aq --exclude 'faraday_copy' --exclude '.cache' . faraday_copy"
- "/bin/tar -zcf faraday.tar.gz faraday_copy"
only:
variables:
- $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
- $CI_COMMIT_TAG
- $BUILD_TEST
- $FULL_TEST
- $DAILY_TEST
artifacts:
name: 'faraday'
paths:
- "faraday.tar.gz"
expire_in: 15 days
generate_deb:
image: registry.gitlab.com/faradaysec/integrationrepo
stage: build
before_script:
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
- mv py3.tar /
- cd /; tar xf py3.tar; cd -
script:
- mkdir build_installer
- cp -a faraday.tar.gz build_installer/.
- cd build_installer
- /bin/tar zxf faraday.tar.gz
- cd faraday_copy
- cp -r /nix .
- mv ../../faraday-linux-installers-builder .
- cd faraday-linux-installers-builder
- git clone https://github.com/jordansissel/fpm.git
- cd fpm
- git checkout d7b466787d17581bc723e474ecf6e18f48226031
- git apply ../fpm-patchs/fpm.virtualenv.patch
- make gem
- gem install --no-ri --no-rdoc fpm-1.11.0.gem
- cd ../../
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server deb
- mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) client deb
- mv faraday-client_amd64.deb ../../faraday-client_amd64.deb
dependencies:
- generate_build_file
- build_nix_python3
artifacts:
name: 'faraday_$CI_COMMIT_REF_NAME.deb'
paths:
- "faraday-server_amd64.deb"
- "faraday-client_amd64.deb"
expire_in: 15 days
only:
variables:
- $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
- $CI_COMMIT_TAG
- $BUILD_TEST
- $FULL_TEST
- $DAILY_TEST
smoke_test_deb:
allow_failure: true
stage: distro_testing # TODO improve
image: ubuntu:18.04
dependencies:
- generate_deb
script:
- apt-get update -y
- apt install -y sudo curl
- apt-get install -y ./faraday-server_amd64.deb
- which faraday-manage
- faraday-manage show-urls
- export FARADAY_HOME=/home/faraday
- /opt/faraday/bin/faraday-server || true # create .faraday
- "echo '[database]' >>~faraday/.faraday/config/server.ini"
- echo "connection_string = postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB" >>~faraday/.faraday/config/server.ini
- cat ~faraday/.faraday/config/server.ini
- faraday-manage create-tables
- /opt/faraday/bin/faraday-server &
- sleep 5
- curl -v http://localhost:5985/_api/v2/info
- faraday-manage status-check
- kill $(cat ~faraday/.faraday/faraday-server-port-5985.pid)
- jobs
only:
variables:
- $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
- $CI_COMMIT_TAG
- $BUILD_TEST
- $FULL_TEST
- $DAILY_TEST
generate_rpm:
stage: build
image: centos:7
before_script:
- yum -y upgrade
- yum -y install which git
- yum -y install epel-release
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
- mv py3.tar /
- cd /; tar xf py3.tar; cd -
- yum -y install ruby ruby-devel rubygems gobject-introspection-devel curl zsh mailcap libffi-devel openssl-devel openldap-devel libxslt-devel libxml2-devel libxslt-devel freetype-devel libjpeg-devel gtk+-devel gtk3-devel gtk2-devel postgresql-devel
- yum groups -y install "Development Tools"
- yum -y install centos-release-scl
- yum -y install rh-python36
- source /opt/rh/rh-python36/enable
- pip install virtualenv
- pip install virtualenv-tools3
script:
- mkdir build_installer
- cp -a faraday.tar.gz build_installer/.
- cd build_installer
- /bin/tar zxf faraday.tar.gz
- cd faraday_copy
- cp -r /nix .
- mv ../../faraday-linux-installers-builder .
- cd faraday-linux-installers-builder
- git clone https://github.com/jordansissel/fpm.git
- cd fpm
- git checkout d7b466787d17581bc723e474ecf6e18f48226031
- git apply ../fpm-patchs/fpm.virtualenv.patch
- make gem
- gem install --no-ri --no-rdoc fpm-1.11.0.gem
- cd ../../
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
- mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) client rpm
- mv faraday-client_amd64.rpm ../../faraday-client_amd64.rpm
dependencies:
- generate_build_file
- build_nix_python3
artifacts:
name: 'faraday_$CI_COMMIT_REF_NAME.rpm'
paths:
- "faraday-server_amd64.rpm"
- "faraday-client_amd64.rpm"
expire_in: 15 days
only:
variables:
- $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
- $CI_COMMIT_TAG
- $BUILD_TEST
- $FULL_TEST
- $DAILY_TEST
macos_pkg:
tags:
- macos
stage: build
before_script:
- echo 'export PATH="/usr/local/opt/openssl/bin:$PATH"' >> ~/.bash_profile
- export LDFLAGS="-L/usr/local/opt/openssl/lib"
- export CPPFLAGS="-I/usr/local/opt/openssl/include"
- export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig"
script:
- pip3 install virtualenv
- pip3 install virtualenv-tools3
- export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/local/opt/libffi/lib/pkgconfig"
- mkdir build_installer
- cp -a faraday.tar.gz build_installer/.
- cd build_installer
- /usr/bin/tar zxf faraday.tar.gz
- cd faraday_copy
- git clone [email protected]:faradaysec/faraday-linux-installers-builder.git
- cd faraday-linux-installers-builder
- git clone https://github.com/jordansissel/fpm.git
- cd fpm
- git checkout d7b466787d17581bc723e474ecf6e18f48226031
- git apply ../fpm-patchs/fpm.virtualenv.patch
- make gem
- export GEM_HOME="$HOME/.gem"
- export GEM_PATH=$HOME/.gem
- export PATH=$PATH:$HOME/.gem/bin
- gem install fpm-1.11.0.gem
- cd ../../
- sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server osxpkg
- mv faraday-server_amd64.pkg ../../Faraday.pkg
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-binaries"
paths:
- Faraday.pkg
expire_in: 1 week
only:
variables:
- $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
- $CI_COMMIT_TAG
- $BUILD_TEST
- $FULL_TEST
- $DAILY_TEST
test_hypothesis:
tags:
- hypothesis
image: registry.gitlab.com/faradaysec/integrationrepo
stage: testing
allow_failure: true
script:
- pip3 install virtualenv
- virtualenv -p python3 faraday_venv
- source faraday_venv/bin/activate
- python3 setup.py install
- pip install --upgrade responses pytest-xdist pytest-cov
- pip install --upgrade -r requirements_dev.txt
- mkdir -p ~/.faraday/config
- cp tests/data/server.ini ~/.faraday/config
- pytest tests -v --cov=faraday/server/api --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB -m hypothesis
only:
variables:
- $FULL_TEST
- $HYPO_TEST
- $DAILY_TEST
agent_integration:
stage: post_testing
script:
- apt-get update -qy
- apt-get install curl -y
- "curl -X POST -F token=$INTEGRATION_TOKEN -F ref=master -F \"variables[FARADAY_REF]=$CI_COMMIT_REF_NAME\" https://gitlab.com/api/v4/projects/15311100/trigger/pipeline"
only:
variables:
- $CI_COMMIT_REF_NAME =~ /^.*\/(dev|master)$/
- $INTEGRATION