.gitlab-ci.yml - python-faraday (upstream/3.9.3)

Tree @upstream/3.9.3 (Download .tar.gz)

.gitlab-ci.yml @upstream/3.9.3 — raw · history · blame

variables:
    TZ: "America/New_York"
    # Configure postgres service (https://hub.docker.com/_/postgres/)
    POSTGRES_DB: custom_db
    POSTGRES_USER: custom_user
    POSTGRES_PASSWORD: custom_pass
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
    APT_CACHE_DIR: "$CI_PROJECT_DIR/apt-cache"
    DEBIAN_FRONTEND: noninteractive
    IMAGE_TAG: '/bin/egrep -o "[0-9]\.[0-9]\.[0-9]" faraday/__init__.py'
    ## ENV_VARS LIST
    # FULL_TEST = Test all jobs
    # BUILD_TEST = Test default and build jobs
    # DAILY_TEST = Run every night at 03 am

cache:
  paths:
    - "$CI_PROJECT_DIR/.cache/pip"
    - "$CI_PROJECT_DIR/apt-cache"

before_script:
    - mkdir -pv $APT_CACHE_DIR

stages:
    - pre_testing
    - testing
    - post_testing
    - build_faraday
    - build

services:
    - postgres:latest

closure_compiler:
    tags:
        - faradaytests
    image:
        name: jborza/closure-compiler
        entrypoint: ["/bin/sh", "-c"]

    stage: pre_testing
    script:
        - /opt/cc.sh faraday/server/www/scripts
    except:
        variables:
            - $BUILD_TEST

merge_conflict_check:
    tags:
        - faradaytests
    image: python:3
    stage: pre_testing
    allow_failure: true
    script:
        - git config --global user.email "[email protected]"
        - git config --global user.name "Mergerbot"
        - python3 merge-conflict-detector.py
    except:
        variables:
            - $BUILD_TEST

pylint:
    tags:
        - faradaytests
    image: registry.gitlab.com/faradaysec/faraday/faraday_testing_base  # I just need an image with python-dev and python-pip
    stage: pre_testing
    script:
        - apt-get update -qy
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" build-essential ipython python-setuptools python-pip python-dev libssl-dev libffi-dev pkg-config libxml2-dev libxslt1-dev libfreetype6-dev libpng-dev
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" libpq-dev python-pip python-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 python-gobject zsh curl
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" gobject-introspection libgirepository1.0-dev libcairo2-dev python-cairo libgdk-pixbuf2.0-dev
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" libcanberra-gtk-module libcanberra-gtk* libxml2-dev libxslt1-dev gir1.2-gtk-3.0 gir1.2-vte-2.91
        - pip install virtualenv
        - virtualenv -p python2 faraday_venv
        - source faraday_venv/bin/activate
        - pip install --upgrade -r requirements.txt
        - python setup.py install
        # pylint slow issue https://github.com/PyCQA/pylint/issues/2765
        - pip install pylint isort==4.3.4
        - pylint --rcfile=.pylintrc faraday --ignore=faraday_venv | tee pylint.txt
        - score=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' pylint.txt)
          #- anybadge --label pylint --value=$score --file pylint.svg 4=red 6=orange 8=yellow 10=green
    artifacts:
        paths:
            - pylint.svg
    except:
        variables:
            - $BUILD_TEST

py3checker:
    tags:
        - faradaytests
    image: python:3
    stage: testing
    script:
        - pip install pylint
        - python3 py3-checker.py -o py3_checker_result.txt
    artifacts:
        paths:
            - py3_checker_result.txt
        expire_in: 1 week
    except:
        variables:
            - $BUILD_TEST

postgresql_test:
    image: registry.gitlab.com/faradaysec/faraday/faraday_testing_base
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - apt-get update -qy
        - apt-get install -o dir::cache::archives="$APT_CACHE_DIR" -y gobject-introspection libgirepository1.0-dev libcairo2-dev python-cairo libpq-dev
        - pip install virtualenv
        - virtualenv -p python2 faraday_venv
        - source faraday_venv/bin/activate
        - pip install pip -U
        #- pip install --upgrade -r requirements_server.txt
        - pip install --upgrade -r requirements.txt
        - python setup.py install
        - pip install --upgrade responses pytest-xdist pytest-cov
        - pip install --upgrade -r requirements_dev.txt
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        #- sed -i 's/mapped_table/persist_selectable/' faraday_venv/lib/python2.7/site-packages/flask_sqlalchemy/__init__.py  # TODO remove when flask_sqlalchemy fixes the issue
        - mkdir run_from
        - cd run_from && pytest ../tests -v --capture=sys --cov=../faraday/server --color=yes --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB
    artifacts:
        when: on_failure
        paths:
            - ~/.faraday/logs/faraday-server.log
    except:
        variables:
            - $BUILD_TEST

sqlite_test:
    image: registry.gitlab.com/faradaysec/faraday/faraday_testing_base
    tags:
        - faradaytests
    stage: testing
    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
    script:
        - apt-get update -qy
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" gobject-introspection libgirepository1.0-dev libcairo2-dev python-cairo libpq-dev
        - pip install virtualenv
        - virtualenv -p python2 faraday_venv
        - source faraday_venv/bin/activate
        - pip install pip -U
        #- pip install --upgrade -r requirements_server.txt
        - pip install --upgrade -r requirements.txt
        - python setup.py install
        - pip install --upgrade responses pytest-xdist pytest-cov
        - pip install --upgrade -r requirements_dev.txt
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        - mkdir run_from
        - cd run_from && pytest ../tests --capture=sys -v --cov=../faraday/server --color=yes --disable-warnings
    artifacts:
        when: on_failure
        paths:
            - dist/*
    except:
        variables:
            - $BUILD_TEST

generate_release_file:
    image: python:3
    stage: post_testing
    allow_failure: true
    script:
        - apt-get update -qy
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" python-dev python-pip
        - pip install packaging
        - cd CHANGELOG && python3 changelog.py
    artifacts:
        paths:
            - CHANGELOG/RELEASE.md
    only:
        variables:
            - $FULL_DOC == "True"
            - $RELEASE_FILE == "True"
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST


generate_build_file:
    image: registry.gitlab.com/faradaysec/faraday/faraday_testing_pink
    stage: build_faraday
    script:
        - apt-get update -qy
        - apt-get install -y rsync
        - "/bin/mkdir faraday_copy"
        - "/usr/bin/rsync -aq --exclude 'faraday_copy' --exclude '.cache' . faraday_copy"
        - "/bin/tar -zcf faraday.tar.gz faraday_copy"
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST
    artifacts:
        name: 'faraday'
        paths:
            - "faraday.tar.gz"
        expire_in: 15 days


generate_deb:
    stage: build
    before_script:
        - 'which ssh-agent || ( apt-get -y update && apt-get -y install openssh-client )'
        - eval "$(ssh-agent -s)"
        - mkdir -p ~/.ssh
        - echo "$SSH_PRIVATE_KEY_OTHER_REPOS_GITLAB"  >> ~/.ssh/id_rsa && chmod 0600 ~/.ssh/id_rsa
        - chmod 700 ~/.ssh
        - ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
    script:
        - apt-get update -qy
        - apt-get -y install virtualenv git libcairo2-dev libgirepository1.0-dev gobject-introspection ruby ruby-dev rubygems build-essential build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev libfreetype6-dev libpng-dev libsasl2-dev libldap2-dev libkrb5-dev pandoc python2.7 python-dev pkg-config libpq-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 zsh curl pandoc
        - curl https://bootstrap.pypa.io/get-pip.py | python
        - pip install virtualenv-tools
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /bin/tar zxf faraday.tar.gz
        - ls
        - cd faraday_copy
        - git clone [email protected]:faradaysec/faraday-linux-installers-builder.git
        - cd faraday-linux-installers-builder
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git apply ../fpm-patchs/fpm.virtualenv_compile.patch
        - make gem
        - gem install --no-ri --no-rdoc fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server deb
        - mv faraday-server_amd64.deb ../../faraday-server_amd64.deb
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) client deb
        - mv faraday-client_amd64.deb ../../faraday-client_amd64.deb
    dependencies:
        - generate_build_file
    artifacts:
        name: 'faraday_$CI_COMMIT_REF_NAME.deb'
        paths:
            - "faraday-server_amd64.deb"
            - "faraday-client_amd64.deb"
        expire_in: 15 days
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

generate_rpm:
    stage: build
    image: centos:7
    before_script:
        - yum -y upgrade
        - yum -y install which
        - yum -y install epel-release
        - 'which ssh-agent || (yum -y update && yum -y install openssh-clients)'
        - eval "$(/usr/bin/ssh-agent -s)"
        - mkdir -p ~/.ssh
        - echo "$SSH_PRIVATE_KEY_OTHER_REPOS_GITLAB"  >> ~/.ssh/id_rsa && chmod 0600 ~/.ssh/id_rsa
        - chmod 700 ~/.ssh
        - /usr/bin/ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
    script:
        - yum -y install git ruby ruby-devel rubygems gobject-introspection-devel curl zsh mailcap libffi-devel python-devel openssl-devel openldap-devel libxslt-devel libxml2-devel libxslt-devel freetype-devel libjpeg-devel gtk+-devel gtk3-devel gtk2-devel pandoc
        - yum groups -y install "Development Tools"
        - curl https://bootstrap.pypa.io/get-pip.py | python
        - pip install virtualenv
        - pip install virtualenv-tools
        - mkdir build_installer
        - cp -a faraday.tar.gz build_installer/.
        - cd build_installer
        - /bin/tar zxf faraday.tar.gz
        - ls
        - cd faraday_copy
        - git clone [email protected]:faradaysec/faraday-linux-installers-builder.git
        - cd faraday-linux-installers-builder
        - git clone https://github.com/jordansissel/fpm.git
        - cd fpm
        - git apply ../fpm-patchs/fpm.virtualenv_compile.patch
        - make gem
        - gem install --no-ri --no-rdoc fpm-1.11.0.gem
        - cd ../../
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) server rpm
        - mv faraday-server_amd64.rpm ../../faraday-server_amd64.rpm
        - sh faraday-linux-installers-builder/build.sh $(eval $IMAGE_TAG) client rpm
        - mv faraday-client_amd64.rpm ../../faraday-client_amd64.rpm
    dependencies:
        - generate_build_file
    artifacts:
        name: 'faraday_$CI_COMMIT_REF_NAME.rpm'
        paths:
            - "faraday-server_amd64.rpm"
            - "faraday-client_amd64.rpm"
        expire_in: 15 days
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

binary_files:
    image: python:2.7.16
    stage: build
    script:
        - apt-get update -qy
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" build-essential ipython python-setuptools python-pip python-dev libssl-dev libffi-dev pkg-config libxml2-dev libxslt1-dev libfreetype6-dev libpng-dev
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" libpq-dev python-pip python-dev gir1.2-gtk-3.0 gir1.2-vte-2.91 python-gobject zsh curl
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" gobject-introspection libgirepository1.0-dev libcairo2-dev python-cairo libgdk-pixbuf2.0-dev
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" libcanberra-gtk-module libcanberra-gtk* libxml2-dev libxslt1-dev gir1.2-gtk-3.0 gir1.2-vte-2.91
        - pip install virtualenv
        - virtualenv -p python2 faraday_venv
        - source faraday_venv/bin/activate
        - pip install pyinstaller
        - pip install --upgrade -r requirements.txt
        - python setup.py install
        - pip install .[gtk]
        - pyinstaller -F -w --onefile --icon=faraday/faraday/server/www/favicon.ico start_server.spec
        - pyinstaller -F -w --onefile --icon=faraday/faraday/server/www/favicon.ico manage.spec
        - pyinstaller -F -w --onefile --icon=faraday/faraday/server/www/favicon.ico start_client.spec
        - pyinstaller -F -w --onefile --icon=faraday/faraday/server/www/favicon.ico fplugin.spec
        - pyinstaller -F -w --onefile --icon=faraday/faraday/server/www/favicon.ico searcher.spec
        - mkdir output
        - mv dist/start_client output/faraday-client
        - mv dist/start_server output/faraday-server
        - mv dist/manage output/faraday-manage
        - mv dist/fplugin output/faraday-fplugin
        - mv dist/searcher output/faraday-searcher
        - tar -zcvf linux-binaries.tar.gz output
    artifacts:
        name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-binaries"
        paths:
            - linux-binaries.tar.gz
        expire_in: 1 week
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

macos_dmg:
    tags:
        - macos
    stage: build
    allow_failure: true
    script:
        - pip install virtualenv
        - virtualenv -p python2 faraday_venv
        - source faraday_venv/bin/activate
        - export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/local/opt/libffi/lib/pkgconfig"
        - pip install pyinstaller
        - pip install --upgrade -r requirements.txt
        - python setup.py install
        - pip install .[gtk]
        - pyinstaller -F -w --onefile --onefile --icon=faraday/faraday/server/www/favicon.ico start_server.spec
        - pyinstaller -F -w --onefile --onefile --icon=faraday/faraday/server/www/favicon.ico manage.spec
        - pyinstaller -F -w --onefile --onefile --icon=faraday/faraday/server/www/favicon.ico start_client.spec
        - pyinstaller -F -w --onefile --icon=faraday/faraday/server/www/favicon.ico fplugin.spec
        - pyinstaller -F -w --onefile --icon=faraday/faraday/server/www/favicon.ico seacher.spec
        - mkdir output
        - mv dist/start_client output/faraday-client
        - mv dist/start_server output/faraday-server
        - mv dist/manage output/faraday-manage
        - mv dist/fplugin output/faraday-fplugin
        - mv dist/searcher output/faraday-searcher
        - hdiutil create /tmp/tmp.dmg -ov -volname "Faraday" -fs HFS+ -srcfolder "./output"
        - hdiutil convert /tmp/tmp.dmg -format UDZO -o Faraday.dmg
    artifacts:
        name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-binaries"
        paths:
            - Faraday.dmg
        expire_in: 1 week
    only:
        variables:
            - $CI_COMMIT_REF_NAME =~ /^.*\/(master)$/
            - $CI_COMMIT_TAG
            - $BUILD_TEST
            - $FULL_TEST
            - $DAILY_TEST

documentation:
    image: python:3
    stage: post_testing
    script:
        - apt-get update -qy
        - apt-get install -y -o dir::cache::archives="$APT_CACHE_DIR" python-pip python-dev
        - pip install sphinx
        - cd doc && make html
    artifacts:
        paths:
            - doc/_build/html
    only:
        variables:
            - $FULL_DOC
            - $DOC
            - $CI_COMMIT_REF_NAME =~ /^.*\/(dev|master)$/


test_hypothesis:
    tags:
        - hypothesis
    image: registry.gitlab.com/faradaysec/faraday/faraday_testing_base
    stage: testing
    allow_failure: true
    script:
        - apt-get update -qy
        - pip install virtualenv
        - virtualenv -p python2 faraday_venv
        - source faraday_venv/bin/activate
        #- pip install --upgrade -r requirements_server.txt
        - python setup.py install
        - pip install --upgrade responses pytest-xdist pytest-cov
        - pip install --upgrade -r requirements_dev.txt
        - mkdir -p ~/.faraday/config
        - cp tests/data/server.ini ~/.faraday/config
        #- sed -i 's/mapped_table/persist_selectable/' faraday_venv/lib/python2.7/site-packages/flask_sqlalchemy/__init__.py  # TODO remove when flask_sqlalchemy fixes the issue
        - pytest tests -v --cov=faraday/server/api --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB -m hypothesis
    only:
        variables:
            - $FULL_TEST
            - $HYPO_TEST
            - $DAILY_TEST