.gitlab-ci.yml - python-faraday (upstream/3.12)

.gitlab-ci.yml @upstream/3.12 — raw · history · blame

variables:
    TZ: "America/New_York"
    # Configure postgres service (https://hub.docker.com/_/postgres/)
    POSTGRES_DB: custom_db
    POSTGRES_USER: custom_user
    POSTGRES_PASSWORD: custom_pass
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
    APT_CACHE_DIR: "$CI_PROJECT_DIR/apt-cache"
    DEBIAN_FRONTEND: noninteractive
    IMAGE_TAG: 'egrep -o "[0-9]\.([0-9]|[0-9][0-9])(\.[0-9])?" faraday/__init__.py'

    ## ENV_VARS LIST
    # FULL_TEST = Test all jobs
    # BUILD_TEST = Test default and build jobs
    # DAILY_TEST = Run every night at 03 am
workflow:
  rules:
    - if: $CI_MERGE_REQUEST_ID
      when: never
    - when: always



cache:
  paths:
    - "$CI_PROJECT_DIR/.cache/pip"
    - "$CI_PROJECT_DIR/apt-cache"

before_script:
    - mkdir -pv $APT_CACHE_DIR

include:
  - local: .gitlab/ci/.set-tag-gitlab-ci.yml
  - local: .gitlab/ci/.storage-gitlab-ci.yml

stages:
    - pre_testing
    - testing
    - post_testing
    - build_faraday
    - build
    - distro_testing
    - deploy
    - post_deploy_testing
    - publish

services:
    - postgres:latest

merge_conflict_check:
    tags:
        - faradaytests
    image: python:3
    stage: pre_testing
    allow_failure: true
    script:
        - git config --global user.email "[email protected]"
        - git config --global user.name "Mergerbot"
        - python3 scripts/merge-conflict-detector.py
    rules:
        - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|dev)$/'
          when: on_success
        - when: never

sanity_check:
    tags:
        - faradaytests
    image: python:3
    stage: pre_testing
    script:
        - bash scripts/sanity_check_commit.sh
        - scripts/sanity_check_file.py --mode=ls
    rules:
        - when: on_success

bandit:
    tags:
        - faradaytests
    image: python:3
    stage: pre_testing
    script:
      - pip3 install virtualenv
      - virtualenv -p python3 faraday_venv
      - source faraday_venv/bin/activate
      - pip3 install bandit
      - "bandit -r ${CI_PROJECT_DIR}/faraday --format custom --skip B101 --msg-template \
          \"{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}\""

build_and_push_to_cachix:
    tags:
        - faradaytests
    image: nixorg/nix
    stage: pre_testing
    script:
        - nix-env -if pynixify/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - nix-build | cachix push faradaysec
    rules:
      - if: '$FULL_TEST || $DAILY_TEST'
        when: on_success
      - when: always

pylint:
    tags:
        - faradaytests
    image: nixorg/nix
    stage: testing  # This should be after build_and_push_to_cachix to improve performance
    script:
        - nix-env -if pynixify/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - nix-shell --command "pylint --rcfile=.pylintrc faraday" | tee pylint.txt
        - nix-env -if pynixify/nixpkgs.nix -A gnused
        - score=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' pylint.txt)
          #- anybadge --label pylint --value=$score --file pylint.svg 4=red 6=orange 8=yellow 10=green
    artifacts:
        paths:
            - pylint.svg
            - pylint3.svg
    rules:
        - if: $BUILD_TEST
          when: never
        - if: '$FULL_TEST || $DAILY_TEST'
          when: on_success
        - when: on_success

.postgresql_test_nix_base:
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - nix-env -if pynixify/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - "echo 'hosts: files dns' >/etc/nsswitch.conf"
        - export LC_ALL=C.UTF-8
        - export LANG=C.UTF-8
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - nix-shell --command "cd run_from && pytest ../tests -v --capture=sys --cov=../faraday/server --color=yes --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB"
    artifacts:
        when: on_failure
        paths:
            - ~/.faraday/logs/faraday-server.log
    rules:
      - if: $BUILD_TEST
        when: never
      - if: '$FULL_TEST || $DAILY_TEST'
        when: on_success
      - when: on_success

.sqlite_test_nix_base:
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - nix-env -if pynixify/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - "echo 'hosts: files dns' >/etc/nsswitch.conf"
        - export LC_ALL=C.UTF-8
        - export LANG=C.UTF-8
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - nix-shell --command "cd run_from && pytest ../tests --capture=sys -v --cov=../faraday/server --color=yes --disable-warnings"
    artifacts:
        when: on_failure
        paths:
            - dist/*
    rules:
        - if: $BUILD_TEST
          when: never
        - if: '$FULL_TEST || $DAILY_TEST'
          when: on_success
        - when: on_success

sqlite_test_nix:
    extends: .sqlite_test_nix_base
    image: nixorg/nix


postgresql_test_nix:
    extends: .postgresql_test_nix_base
    image: nixorg/nix


.unit_tests_base:
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - export LC_ALL=C.UTF-8
        - export LANG=C.UTF-8
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - source faraday_venv/bin/activate
        - cd run_from && pytest ../tests -v --capture=sys --cov=../faraday/server --color=yes --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB
    artifacts:
        when: on_failure
        paths:
            - ~/.faraday/logs/faraday-server.log

.latest_unit_test_base:
  extends: .unit_tests_base
  before_script:
      - pip install virtualenv
      - virtualenv faraday_venv
      - source faraday_venv/bin/activate
      - pip install .
      - pip install -r requirements_dev.txt
      - pip install pytest-cov
      - pip install pyyaml
  after_script:
      - source faraday_venv/bin/activate
      - pip freeze
  allow_failure: true
  rules:
      #- if: '$FULL_TEST || $DAILY_TEST || $ALPHA_TEST'
      - if: '$ALPHA_TEST' # FOR NOW, ASKED TO NOT CHARGE CI WORKER
        when: on_success
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|dev)$/'
        when: never # FOR NOW, ASKED TO NOT CHARGE CI WORKER
        #when: on_success
      - when: never


.alpha_unit_test_base:
  extends: .unit_tests_base
  before_script:
      - pip install virtualenv
      - virtualenv faraday_venv
      - source faraday_venv/bin/activate
      - pip install --pre .
      - pip install --pre -r requirements_dev.txt
      - pip install --pre pytest-cov
      - pip install --pre pyyaml
  after_script:
    - source faraday_venv/bin/activate
    - pip freeze
  allow_failure: true
  rules:
      #- if: '$FULL_TEST || $DAILY_TEST || $ALPHA_TEST'
      - if: '$ALPHA_TEST' # FOR NOW, ASKED TO FIX #6474 first
        when: on_success
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|dev)$/'
        when: never # FOR NOW, ASKED TO FIX #6474 first
        #when: on_success
      - when: never

unit_test 3.7:
    extends: .latest_unit_test_base
    image: python:3.7

unit_test 3.8:
    extends: .latest_unit_test_base
    image: python:3.8

unit_test 3.9:
    extends: .latest_unit_test_base
    image: python:3.9-rc

alpha_unit_test 3.7:
    extends: .alpha_unit_test_base
    image: python:3.7

alpha_unit_test 3.8:
    extends: .alpha_unit_test_base
    image: python:3.8
    rules:
      - if: '$FULL_TEST || $DAILY_TEST || $ALPHA_TEST'
        when: on_success
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|dev)$/'
        when: on_success
      - when: never

alpha_unit_test 3.9:
    extends: .alpha_unit_test_base
    image: python:3.9-rc

build_nix_python3:
  image: nixorg/nix
  stage: build_faraday
  script:
    - nix-env -if pynixify/nixpkgs.nix -A cachix
    - mkdir -p ~/.config/cachix
    - export USER=$(whoami)
    - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
    - cachix use faradaysec
    - nix-build
    - tar cf /py3.tar $(nix-store --query --requisites $(readlink result))
    - mkdir -p /opt/faraday
    - cp -r $(readlink result)/* /opt/faraday
    - tar rvf /py3.tar /opt/faraday
    - mv /py3.tar $CI_PROJECT_DIR
  artifacts:
    name: python3
    paths:
      - py3.tar
    expire_in: 15 days # in the future we don't need to expire this.
  rules:
    - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
      when: on_success
    - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
      when: on_success
    - when: never

generate_build_file:
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: build_faraday
    script:
        - "/bin/mkdir faraday_copy"
        - "/usr/bin/rsync -aq --exclude 'faraday_copy' --exclude '.cache' . faraday_copy"
        - "/bin/tar -zcf faraday.tar.gz faraday_copy"
    rules:
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
        when: on_success
      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
        when: on_success
      - when: never
    artifacts:
        name: 'faraday'
        paths:
            - "faraday.tar.gz"
        expire_in: 15 days


generate_deb:
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: build
    before_script:
        - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
        - mv py3.tar /
        - cd /; tar xf py3.tar; cd -

    script:
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /bin/tar zxf faraday.tar.gz
        - cd faraday_copy
        - cp -r /nix .
        - mv ../../faraday-linux-installers-builder .
        - cd faraday-linux-installers-builder
        - git rev-parse HEAD
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git checkout d7b466787d17581bc723e474ecf6e18f48226031
        - git apply ../fpm-patchs/fpm.virtualenv.patch
        - make gem
        - gem install --no-ri --no-rdoc fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server deb
        - mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
    dependencies:
      - generate_build_file
      - build_nix_python3
    artifacts:
        name: 'faraday_$CI_COMMIT_REF_NAME.deb'
        paths:
            - "faraday-server_amd64.deb"
        expire_in: 15 days
    rules:
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
        when: on_success
      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
        when: on_success
      - when: never


smoke_test_deb:
    allow_failure: false
    stage: distro_testing  # TODO improve
    image: ubuntu:18.04
    dependencies:
        - generate_deb
    script:
        - apt-get update -y
        - apt install -y sudo curl
        - apt-get install -y ./faraday-server_amd64.deb
        - which faraday-manage
        - faraday-manage show-urls
        - export FARADAY_HOME=/home/faraday
        - /opt/faraday/bin/faraday-server || true  # create .faraday
        - "echo '[database]' >>~faraday/.faraday/config/server.ini"
        - echo "connection_string = postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB" >>~faraday/.faraday/config/server.ini
        - cat ~faraday/.faraday/config/server.ini
        - faraday-manage create-tables
        - /opt/faraday/bin/faraday-server &
        - sleep 5
        - curl -v http://localhost:5985/_api/v2/info
        - faraday-manage status-check
        - kill $(cat ~faraday/.faraday/faraday-server-port-5985.pid)
        - jobs
    rules:
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
        when: on_success
      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
        when: on_success
      - when: never

generate_rpm:
    stage: build
    image: centos:7
    before_script:
      - yum -y upgrade
      - yum -y install which git epel-release centos-release-scl
      - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
      - mv py3.tar /
      - cd /; tar xf py3.tar; cd -
      - yum -y install curl zsh mailcap libffi-devel openssl-devel openldap-devel libjpeg-devel postgresql-devel
      - yum groups -y install "Development Tools"
      - yum -y install rh-python36 rh-ruby23 rh-ruby23-ruby-devel
      - source /opt/rh/rh-ruby23/enable
      - export X_SCLS="`scl enable rh-ruby23 'echo $X_SCLS'`"
      - source /opt/rh/rh-python36/enable
      - pip install virtualenv
      - pip install virtualenv-tools3
    script:
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /bin/tar zxf faraday.tar.gz
        - cd faraday_copy
        - cp -r /nix .
        - mv ../../faraday-linux-installers-builder .
        - cd faraday-linux-installers-builder
        - git rev-parse HEAD
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git checkout d7b466787d17581bc723e474ecf6e18f48226031
        - git apply ../fpm-patchs/fpm.virtualenv.patch
        - make gem
        - gem install --no-ri --no-rdoc fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
        - mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
    dependencies:
        - generate_build_file
        - build_nix_python3
    artifacts:
        name: 'faraday_$CI_COMMIT_REF_NAME.rpm'
        paths:
            - "faraday-server_amd64.rpm"
        expire_in: 15 days
    rules:
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
        when: on_success
      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
        when: on_success
      - when: never

test_hypothesis:
    tags:
        - hypothesis
    image: nixorg/nix
    stage: testing
    allow_failure: true
    script:
        - nix-env -if pynixify/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - "echo 'hosts: files dns' >/etc/nsswitch.conf"
        - export LC_ALL=C.UTF-8
        - export LANG=C.UTF-8
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - nix-shell --command "pytest tests -v --cov=faraday/server/api --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB -m hypothesis"
    rules:
      - if: '$HYPO_TEST || $FULL_TEST || $DAILY_TEST'
        when: on_success
      - when: never

agent_integration:
    stage: post_testing
    variables:
      FARADAY_REF: $CI_COMMIT_REF_NAME
    trigger:
      project: faradaysec/integrationrepo
      strategy: depend
    rules:
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
        when: on_success
      - if: '$INTEGRATION || $FULL_TEST || $DAILY_TEST'
        when: on_success
      - when: never

qa_integration:
    stage: post_deploy_testing
    variables:
      REMOTE_BRANCH: $CI_COMMIT_BRANCH
      MAIN_COMMIT_SHA: $CI_COMMIT_SHA
    trigger:
        project: faradaysec/qa/automation
        strategy: depend
        branch: develop
    rules:
      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
        when: on_success
      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
        when: on_success
Tree @upstream/3.12 (Download .tar.gz)

.gitlab-ci.yml @upstream/3.12 — raw · history · blame
