.gitlab-ci.yml - python-faraday (upstream/3.10.2)

Tree @upstream/3.10.2 (Download .tar.gz)

.gitlab-ci.yml @upstream/3.10.2 — raw · history · blame

variables:
    TZ: "America/New_York"
    # Configure postgres service (https://hub.docker.com/_/postgres/)
    POSTGRES_DB: custom_db
    POSTGRES_USER: custom_user
    POSTGRES_PASSWORD: custom_pass
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
    APT_CACHE_DIR: "$CI_PROJECT_DIR/apt-cache"
    DEBIAN_FRONTEND: noninteractive
    IMAGE_TAG: 'egrep -o "[0-9]\.([0-9]|[0-9][0-9])\.[0-9]" faraday/__init__.py'

    ## ENV_VARS LIST
    # FULL_TEST = Test all jobs
    # BUILD_TEST = Test default and build jobs
    # DAILY_TEST = Run every night at 03 am

cache:
  paths:
    - "$CI_PROJECT_DIR/.cache/pip"
    - "$CI_PROJECT_DIR/apt-cache"

before_script:
    - mkdir -pv $APT_CACHE_DIR

stages:
    - pre_testing
    - testing
    - post_testing
    - build_faraday
    - build
    - distro_testing

services:
    - postgres:latest

closure_compiler:
    tags:
        - faradaytests
    image:
        name: jborza/closure-compiler
        entrypoint: ["/bin/sh", "-c"]

    stage: pre_testing
    script:
        - /opt/cc.sh faraday/server/www/scripts
    except:
        variables:
            - $BUILD_TEST

merge_conflict_check:
    tags:
        - faradaytests
    image: python:3
    stage: pre_testing
    allow_failure: true
    script:
        - git config --global user.email "[email protected]"
        - git config --global user.name "Mergerbot"
        - python3 merge-conflict-detector.py

pylint:
    tags:
        - faradaytests
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: pre_testing
    script:
        - pip3 install virtualenv
        - virtualenv -p python3 faraday_venv
        - source faraday_venv/bin/activate
        - pip install --upgrade -r requirements.txt
        - python3 setup.py install
        # pylint slow issue https://github.com/PyCQA/pylint/issues/2765
        - pip install pylint isort==4.3.4
        - pylint --rcfile=.pylintrc faraday --ignore=faraday_venv | tee pylint.txt
        - score=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' pylint.txt)
          #- anybadge --label pylint --value=$score --file pylint.svg 4=red 6=orange 8=yellow 10=green
    artifacts:
        paths:
            - pylint.svg
            - pylint3.svg
    except:
        variables:
            - $BUILD_TEST

postgresql_test:
    image: registry.gitlab.com/faradaysec/integrationrepo
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - export LC_ALL=C.UTF-8
        - export LANG=C.UTF-8
        - pip3 install virtualenv
        - virtualenv -p python3 faraday_venv
        - source faraday_venv/bin/activate
        - pip install pip -U
        - pip install --upgrade -r requirements.txt
        - python3 setup.py install
        - pip install --upgrade responses pytest-xdist pytest-cov
        - pip install --upgrade -r requirements_dev.txt
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - cd run_from && pytest ../tests -v --capture=sys --cov=../faraday/server --color=yes --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB
    artifacts:
        when: on_failure
        paths:
            - ~/.faraday/logs/faraday-server.log
    except:
        variables:
            - $BUILD_TEST

sqlite_test:
    image: registry.gitlab.com/faradaysec/integrationrepo
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - export LC_ALL=C.UTF-8
        - export LANG=C.UTF-8
        - pip3 install virtualenv
        - virtualenv -p python3 faraday_venv
        - source faraday_venv/bin/activate
        - pip install pip -U
        - pip install --upgrade -r requirements.txt
        - python3 setup.py install
        - pip install --upgrade responses pytest-xdist pytest-cov
        - pip install --upgrade -r requirements_dev.txt
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - cd run_from && pytest ../tests --capture=sys -v --cov=../faraday/server --color=yes --disable-warnings
    artifacts:
        when: on_failure
        paths:
            - dist/*
    except:
        variables:
            - $BUILD_TEST


build_nix_python3:
  image: nixorg/nix
  stage: build_faraday
  script:
    - nix-channel --add https://nixos.org/channels/nixos-19.09 nixpkgs
    - nix-channel --update
    - nix-build buildpth.nix
    - nix-build '<nixpkgs>' -A gnome3.vte -o result-vte
    - nix-build '<nixpkgs>' -A gtk3 -o result-gtk
    - tar cf /py3.tar $(nix-store --query --requisites $(readlink result) $(readlink result-vte)) $(readlink result-gtk)
    - mkdir /opt
    - nix-env -i findutils
    - "nix-shell -p python37Packages.virtualenv --command 'virtualenv /opt/faraday'"
    # - cp result /opt/faraday/lib/python3.7/site-packages/nix.pth  # This doesn't include dependencies of dependencies
    - 'for dir in $(nix-store -qR result); do if [[ -d "$dir/lib/python3.7/site-packages" ]]; then echo "$dir/lib/python3.7/site-packages" >>recursive.pth; fi; done'
    - mv recursive.pth /opt/faraday/lib/python3.7/site-packages/nix.pth
    - source /opt/faraday/bin/activate
    # - "nix-shell -p libxml2 libxslt zlib cairo gobject-introspection glib pkgconfig --run 'SOURCE_DATE_EPOCH=$(date +%s) /opt/faraday/bin/pip install -r requirements.txt'"
    - python setup.py install
    - pip install -r requirements.txt  # This has to be below setup.py install
    - pip freeze
    - mv /opt/faraday/bin/faraday-client /opt/faraday/bin/_faraday_client_novte
    - 'nix-shell -p makeWrapper --command "makeWrapper /opt/faraday/bin/_faraday_client_novte /opt/faraday/bin/faraday-client --prefix GI_TYPELIB_PATH : $(find /nix/store -name "girepository-1.0" | tr "\n" ":")"'
    - 'nix-shell -p makeWrapper --command "makeWrapper /opt/faraday/bin/python /opt/faraday/bin/test --prefix GI_TYPELIB_PATH : $(find /nix/store -name "girepository-1.0" | tr "\n" ":")"'
    - nix-collect-garbage && /opt/faraday/bin/test -c "import gi;gi.require_version('Gtk', '3.0');gi.require_version('Vte', '2.91');from gi.repository import Gio, Gtk, GdkPixbuf, Vte, GLib, GObject, Gdk" # Test if GTK will work
    - tar rvf /py3.tar /opt/faraday
    - mv /py3.tar $CI_PROJECT_DIR
  artifacts:
    name: python3
    paths:
      - py3.tar
    expire_in: 15 days # in the future we don't need to expire this.
  only:
    variables:
      - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
      - $CI_COMMIT_TAG
      - $BUILD_TEST
      - $FULL_TEST
      - $DAILY_TEST

generate_build_file:
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: build_faraday
    script:
        - "/bin/mkdir faraday_copy"
        - "/usr/bin/rsync -aq --exclude 'faraday_copy' --exclude '.cache' . faraday_copy"
        - "/bin/tar -zcf faraday.tar.gz faraday_copy"
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST
    artifacts:
        name: 'faraday'
        paths:
            - "faraday.tar.gz"
        expire_in: 15 days


generate_deb:
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: build
    before_script:
        - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
        - mv py3.tar /
        - cd /; tar xf py3.tar; cd -

    script:
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /bin/tar zxf faraday.tar.gz
        - cd faraday_copy
        - cp -r /nix .
        - mv ../../faraday-linux-installers-builder .
        - cd faraday-linux-installers-builder
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git checkout d7b466787d17581bc723e474ecf6e18f48226031
        - git apply ../fpm-patchs/fpm.virtualenv.patch
        - make gem
        - gem install --no-ri --no-rdoc fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server deb
        - mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) client deb
        - mv faraday-client_amd64.deb ../../faraday-client_amd64.deb
    dependencies:
      - generate_build_file
      - build_nix_python3
    artifacts:
        name: 'faraday_$CI_COMMIT_REF_NAME.deb'
        paths:
            - "faraday-server_amd64.deb"
            - "faraday-client_amd64.deb"
        expire_in: 15 days
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST


smoke_test_deb:
    allow_failure: true
    stage: distro_testing  # TODO improve
    image: ubuntu:18.04
    dependencies:
        - generate_deb
    script:
        - apt-get update -y
        - apt install -y sudo curl
        - apt-get install -y ./faraday-server_amd64.deb
        - which faraday-manage
        - faraday-manage show-urls
        - export FARADAY_HOME=/home/faraday
        - /opt/faraday/bin/faraday-server || true  # create .faraday
        - "echo '[database]' >>~faraday/.faraday/config/server.ini"
        - echo "connection_string = postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB" >>~faraday/.faraday/config/server.ini
        - cat ~faraday/.faraday/config/server.ini
        - faraday-manage create-tables
        - /opt/faraday/bin/faraday-server &
        - sleep 5
        - curl -v http://localhost:5985/_api/v2/info
        - faraday-manage status-check
        - kill $(cat ~faraday/.faraday/faraday-server-port-5985.pid)
        - jobs
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

generate_rpm:
    stage: build
    image: centos:7
    before_script:
        - yum -y upgrade
        - yum -y install which git
        - yum -y install epel-release
        - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
        - mv py3.tar /
        - cd /; tar xf py3.tar; cd -
        - yum -y install ruby ruby-devel rubygems gobject-introspection-devel curl zsh mailcap libffi-devel openssl-devel openldap-devel libxslt-devel libxml2-devel libxslt-devel freetype-devel libjpeg-devel gtk+-devel gtk3-devel gtk2-devel postgresql-devel
        - yum groups -y install "Development Tools"
        - yum -y install centos-release-scl
        - yum -y install rh-python36
        - source /opt/rh/rh-python36/enable
        - pip install virtualenv
        - pip install virtualenv-tools3
    script:
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /bin/tar zxf faraday.tar.gz
        - cd faraday_copy
        - cp -r /nix .
        - mv ../../faraday-linux-installers-builder .
        - cd faraday-linux-installers-builder
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git checkout d7b466787d17581bc723e474ecf6e18f48226031
        - git apply ../fpm-patchs/fpm.virtualenv.patch
        - make gem
        - gem install --no-ri --no-rdoc fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
        - mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) client rpm
        - mv faraday-client_amd64.rpm ../../faraday-client_amd64.rpm
    dependencies:
        - generate_build_file
        - build_nix_python3
    artifacts:
        name: 'faraday_$CI_COMMIT_REF_NAME.rpm'
        paths:
            - "faraday-server_amd64.rpm"
            - "faraday-client_amd64.rpm"
        expire_in: 15 days
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

macos_pkg:
    tags:
        - macos
    stage: build
    allow_failure: true
    before_script:
      - echo 'export PATH="/usr/local/opt/openssl/bin:$PATH"' >> ~/.bash_profile
      - export LDFLAGS="-L/usr/local/opt/openssl/lib"
      - export CPPFLAGS="-I/usr/local/opt/openssl/include"
      - export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig"
    script:
        - pip3 install virtualenv
        - pip3 install virtualenv-tools3
        - export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/local/opt/libffi/lib/pkgconfig"
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /usr/bin/tar zxf faraday.tar.gz
        - cd faraday_copy
        - git clone [email protected]:faradaysec/faraday-linux-installers-builder.git
        - cd faraday-linux-installers-builder
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git checkout d7b466787d17581bc723e474ecf6e18f48226031
        - git apply ../fpm-patchs/fpm.osx.patch
        - make gem
        - export GEM_HOME="$HOME/.gem"
        - export GEM_PATH=$HOME/.gem
        - export PATH=$PATH:$HOME/.gem/bin
        - gem install fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build_osx.sh $(eval $IMAGE_TAG) server osxpkg
        - mv faraday-server_amd64.pkg ../../Faraday-server.pkg
        - sh faraday-linux-installers-builder/build_osx.sh $(eval $IMAGE_TAG) client osxpkg
        - mv faraday-client_amd64.pkg ../../Faraday-client.pkg
        - mv faraday-server_amd64.pkg ../../Faraday.pkg
    artifacts:
        name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-binaries"
        paths:
            - Faraday-server.pkg
            - Faraday-client.pkg
        expire_in: 1 week
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

test_hypothesis:
    tags:
        - hypothesis
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: testing
    allow_failure: true
    script:
        - pip3 install virtualenv
        - virtualenv -p python3 faraday_venv
        - source faraday_venv/bin/activate
        - python3 setup.py install
        - pip install --upgrade responses pytest-xdist pytest-cov
        - pip install --upgrade -r requirements_dev.txt
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - pytest tests -v --cov=faraday/server/api --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB -m hypothesis
    only:
        variables:
            - $FULL_TEST
            - $HYPO_TEST
            - $DAILY_TEST

agent_integration:
    stage: post_testing
    script:
        - apt-get update -qy
        - apt-get install curl -y
        - "curl -X POST -F token=$INTEGRATION_TOKEN -F ref=master -F \"variables[FARADAY_REF]=$CI_COMMIT_REF_NAME\" https://gitlab.com/api/v4/projects/15311100/trigger/pipeline"
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(dev|master)$/
            - $INTEGRATION