.gitlab-ci.yml - python-faraday (kali/3.11.1-0kali2)

.gitlab-ci.yml @kali/3.11.1-0kali2 — raw · history · blame

variables:
    TZ: "America/New_York"
    # Configure postgres service (https://hub.docker.com/_/postgres/)
    POSTGRES_DB: custom_db
    POSTGRES_USER: custom_user
    POSTGRES_PASSWORD: custom_pass
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
    APT_CACHE_DIR: "$CI_PROJECT_DIR/apt-cache"
    DEBIAN_FRONTEND: noninteractive
    IMAGE_TAG: 'egrep -o "[0-9]\.([0-9]|[0-9][0-9])(\.[0-9])?" faraday/__init__.py'

    ## ENV_VARS LIST
    # FULL_TEST = Test all jobs
    # BUILD_TEST = Test default and build jobs
    # DAILY_TEST = Run every night at 03 am

cache:
  paths:
    - "$CI_PROJECT_DIR/.cache/pip"
    - "$CI_PROJECT_DIR/apt-cache"

before_script:
    - mkdir -pv $APT_CACHE_DIR

stages:
    - pre_testing
    - testing
    - post_testing
    - build_faraday
    - build
    - distro_testing

services:
    - postgres:latest

closure_compiler:
    tags:
        - faradaytests
    image:
        name: jborza/closure-compiler
        entrypoint: ["/bin/sh", "-c"]

    stage: pre_testing
    script:
        - /opt/cc.sh faraday/server/www/scripts
    except:
        variables:
            - $BUILD_TEST

merge_conflict_check:
    tags:
        - faradaytests
    image: python:3
    stage: pre_testing
    allow_failure: true
    script:
        - git config --global user.email "[email protected]"
        - git config --global user.name "Mergerbot"
        - python3 scripts/merge-conflict-detector.py
    except:
        variables:
            - $BUILD_TEST

build_and_push_to_cachix:
    tags:
        - faradaytests
    image: nixorg/nix
    stage: pre_testing
    script:
        - nix-env -if pypi2nixpkgs/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - nix-build | cachix push faradaysec

pylint:
    tags:
        - faradaytests
    image: nixorg/nix
    stage: testing  # This should be after build_and_push_to_cachix to improve performance
    script:
        - nix-env -if pypi2nixpkgs/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - nix-shell --command "pylint --rcfile=.pylintrc faraday" | tee pylint.txt
        - nix-env -if pypi2nixpkgs/nixpkgs.nix -A gnused
        - score=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' pylint.txt)
          #- anybadge --label pylint --value=$score --file pylint.svg 4=red 6=orange 8=yellow 10=green
    artifacts:
        paths:
            - pylint.svg
            - pylint3.svg
    except:
        variables:
            - $BUILD_TEST

.postgresql_test_base:
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - nix-env -if pypi2nixpkgs/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - "echo 'hosts: files dns' >/etc/nsswitch.conf"
        - export LC_ALL=C.UTF-8
        - export LANG=C.UTF-8
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - nix-shell --command "cd run_from && pytest ../tests -v --capture=sys --cov=../faraday/server --color=yes --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB"
    artifacts:
        when: on_failure
        paths:
            - ~/.faraday/logs/faraday-server.log
    except:
        variables:
            - $BUILD_TEST

.sqlite_test_base:
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - nix-env -if pypi2nixpkgs/nixpkgs.nix -A cachix
        - mkdir -p ~/.config/cachix
        - export USER=$(whoami)
        - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
        - cachix use faradaysec
        - "echo 'hosts: files dns' >/etc/nsswitch.conf"
        - export LC_ALL=C.UTF-8
        - export LANG=C.UTF-8
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - nix-shell --command "cd run_from && pytest ../tests --capture=sys -v --cov=../faraday/server --color=yes --disable-warnings"
    artifacts:
        when: on_failure
        paths:
            - dist/*
    except:
        variables:
            - $BUILD_TEST

sqlite_test_3_7:
    extends: .sqlite_test_base
    image: nixorg/nix
    variables:
      PY_VERSION: python3.7


.sqlite_test_3_8:
    extends: .sqlite_test_base
    image: nixorg/nix
    variables:
      PY_VERSION: python3.8


postgresql_test_3_7:
    extends: .postgresql_test_base
    image: nixorg/nix
    variables:
      PY_VERSION: python3.7


.postgresql_test_3_8:
    extends: .postgresql_test_base
    image: nixorg/nix
    variables:
      PY_VERSION: python3.8


build_nix_python3:
  image: nixorg/nix
  stage: build_faraday
  script:
    - nix-env -if pypi2nixpkgs/nixpkgs.nix -A cachix
    - mkdir -p ~/.config/cachix
    - export USER=$(whoami)
    - echo "$CACHIX_CONFG" >~/.config/cachix/cachix.dhall
    - cachix use faradaysec
    - nix-build
    - tar cf /py3.tar $(nix-store --query --requisites $(readlink result))
    - mkdir -p /opt/faraday
    - cp -r $(readlink result)/* /opt/faraday
    - tar rvf /py3.tar /opt/faraday
    - mv /py3.tar $CI_PROJECT_DIR
  artifacts:
    name: python3
    paths:
      - py3.tar
    expire_in: 15 days # in the future we don't need to expire this.
  only:
    variables:
      - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
      - $CI_COMMIT_TAG
      - $BUILD_TEST
      - $FULL_TEST
      - $DAILY_TEST

generate_build_file:
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: build_faraday
    script:
        - "/bin/mkdir faraday_copy"
        - "/usr/bin/rsync -aq --exclude 'faraday_copy' --exclude '.cache' . faraday_copy"
        - "/bin/tar -zcf faraday.tar.gz faraday_copy"
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST
    artifacts:
        name: 'faraday'
        paths:
            - "faraday.tar.gz"
        expire_in: 15 days


generate_deb:
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: build
    before_script:
        - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
        - mv py3.tar /
        - cd /; tar xf py3.tar; cd -

    script:
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /bin/tar zxf faraday.tar.gz
        - cd faraday_copy
        - cp -r /nix .
        - mv ../../faraday-linux-installers-builder .
        - cd faraday-linux-installers-builder
        - git rev-parse HEAD
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git checkout d7b466787d17581bc723e474ecf6e18f48226031
        - git apply ../fpm-patchs/fpm.virtualenv.patch
        - make gem
        - gem install --no-ri --no-rdoc fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server deb
        - mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) client deb
    dependencies:
      - generate_build_file
      - build_nix_python3
    artifacts:
        name: 'faraday_$CI_COMMIT_REF_NAME.deb'
        paths:
            - "faraday-server_amd64.deb"
        expire_in: 15 days
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST


smoke_test_deb:
    allow_failure: false
    stage: distro_testing  # TODO improve
    image: ubuntu:18.04
    dependencies:
        - generate_deb
    script:
        - apt-get update -y
        - apt install -y sudo curl
        - apt-get install -y ./faraday-server_amd64.deb
        - which faraday-manage
        - faraday-manage show-urls
        - export FARADAY_HOME=/home/faraday
        - /opt/faraday/bin/faraday-server || true  # create .faraday
        - "echo '[database]' >>~faraday/.faraday/config/server.ini"
        - echo "connection_string = postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB" >>~faraday/.faraday/config/server.ini
        - cat ~faraday/.faraday/config/server.ini
        - faraday-manage create-tables
        - /opt/faraday/bin/faraday-server &
        - sleep 5
        - curl -v http://localhost:5985/_api/v2/info
        - faraday-manage status-check
        - kill $(cat ~faraday/.faraday/faraday-server-port-5985.pid)
        - jobs
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

generate_rpm:
    stage: build
    image: centos:7
    before_script:
      - yum -y upgrade
      - yum -y install which git epel-release centos-release-scl
      - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/faraday-linux-installers-builder.git
      - mv py3.tar /
      - cd /; tar xf py3.tar; cd -
      - yum -y install curl zsh mailcap libffi-devel openssl-devel openldap-devel libjpeg-devel postgresql-devel
      - yum groups -y install "Development Tools"
      - yum -y install rh-python36 rh-ruby23 rh-ruby23-ruby-devel
      - source /opt/rh/rh-ruby23/enable
      - export X_SCLS="`scl enable rh-ruby23 'echo $X_SCLS'`"
      - source /opt/rh/rh-python36/enable
      - pip install virtualenv
      - pip install virtualenv-tools3
    script:
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /bin/tar zxf faraday.tar.gz
        - cd faraday_copy
        - cp -r /nix .
        - mv ../../faraday-linux-installers-builder .
        - cd faraday-linux-installers-builder
        - git rev-parse HEAD
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git checkout d7b466787d17581bc723e474ecf6e18f48226031
        - git apply ../fpm-patchs/fpm.virtualenv.patch
        - make gem
        - gem install --no-ri --no-rdoc fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
        - mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) client rpm
    dependencies:
        - generate_build_file
        - build_nix_python3
    artifacts:
        name: 'faraday_$CI_COMMIT_REF_NAME.rpm'
        paths:
            - "faraday-server_amd64.rpm"
        expire_in: 15 days
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

test_hypothesis:
    tags:
        - hypothesis
    image: registry.gitlab.com/faradaysec/integrationrepo
    stage: testing
    allow_failure: true
    script:
        - pip3 install virtualenv
        - virtualenv -p python3 faraday_venv
        - source faraday_venv/bin/activate
        - python3 setup.py install
        - pip install --upgrade responses pytest-xdist pytest-cov
        - pip install --upgrade -r requirements_dev.txt
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - pytest tests -v --cov=faraday/server/api --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB -m hypothesis
    only:
        variables:
            - $FULL_TEST
            - $HYPO_TEST
            - $DAILY_TEST

agent_integration:
    stage: post_testing
    script:
        - apt-get update -qy
        - apt-get install curl -y
        - "curl -X POST -F token=$INTEGRATION_TOKEN -F ref=master -F \"variables[FARADAY_REF]=$CI_COMMIT_REF_NAME\" https://gitlab.com/api/v4/projects/15311100/trigger/pipeline"
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(dev|master)$/
            - $INTEGRATION
    except:
        variables:
            - $BUILD_TEST
Tree @kali/3.11.1-0kali2 (Download .tar.gz)

.gitlab-ci.yml @kali/3.11.1-0kali2 — raw · history · blame
