blockstream-satellite-api/.gitlab-ci.yml

variables:
  GIT_SUBMODULE_STRATEGY: none
  CI_DISPOSABLE_ENVIRONMENT: "true"
  DOCKER_DRIVER: overlay2
  DOCKER_HOST: tcp://localhost:2375
  DOCKER_TLS_CERTDIR: ""
  API_IMAGE: us.gcr.io/satellite-api/satellite-api
  SSE_IMAGE: us.gcr.io/satellite-api/satellite-api-sse

image: blockstream/gcloud-docker:1.0.5
stages:
  - test
  - build
  - plan
  - deploy

before_script:
  - TMPF=$(mktemp) || exit 1
  - echo $GCLOUD_KEY > $TMPF
  - export GOOGLE_APPLICATION_CREDENTIALS=$TMPF
  - gcloud auth activate-service-account --key-file=$TMPF
  - gcloud auth configure-docker
  - gcloud auth list
  - gcloud --version

# Run the unit tests
unit_tests:
  stage: test
  image: python:3.9
  tags:
    - k8s
  except:
    - cleanup_staging@satellite/satellite-api
    - /^staging_.*/
    - /^prod_.*/
    - /^testnet_prod_.*/
    - /^misc_.*/
  before_script:
    - pip install flake8 "yapf>=0.33" pytest pytest-cov
    - pip install -r server/requirements.txt
    - pip install -r server/test_requirements.txt
  script:
    - flake8 .
    - yapf --diff --recursive --verbose server/
    - cd server/ && python -m pytest --cov=.

# Test the Satellite API docker image build
docker_build:
  stage: build
  tags:
    - k8s-docker
  services:
    - docker:20.10.12-dind
  only:
    - branches@satellite/satellite-api
  except:
    - cleanup_staging@satellite/satellite-api
    - /^staging_.*/
    - /^prod_.*/
    - /^misc_.*/
  parallel:
    matrix:
      - IMAGE: $API_IMAGE
        DIR: server
      - IMAGE: $SSE_IMAGE
        DIR: sse
  script:
    - cd $DIR
    - docker pull $IMAGE:$CI_COMMIT_REF_SLUG || docker pull $IMAGE:latest || true
    - >
      docker build
      -f Dockerfile
      --network=host
      --cache-from $IMAGE:$CI_COMMIT_REF_SLUG
      --cache-from $IMAGE:latest
      --build-arg BUILDKIT_INLINE_CACHE=1
      .      

# Build and push the Satellite API docker image
docker_push:
  stage: build
  tags:
    - k8s-docker
  when: manual
  services:
    - docker:20.10.12-dind
  only:
    - branches@satellite/satellite-api
  parallel:
    matrix:
      - IMAGE: $API_IMAGE
        DIR: server
      - IMAGE: $SSE_IMAGE
        DIR: sse
  script:
    - cd $DIR
    - docker pull $IMAGE:$CI_COMMIT_REF_SLUG || docker pull $IMAGE:latest || true
    - >
      docker build
      -f Dockerfile
      --network=host
      --cache-from $IMAGE:$CI_COMMIT_REF_SLUG
      --cache-from $IMAGE:latest
      --build-arg BUILDKIT_INLINE_CACHE=1
      -t $IMAGE:$CI_COMMIT_SHA
      -t $IMAGE:$CI_COMMIT_REF_SLUG
      -t $IMAGE:latest
      .      
    - docker push $IMAGE:$CI_COMMIT_SHA
    - docker push $IMAGE:$CI_COMMIT_REF_SLUG
    - if [ $CI_COMMIT_BRANCH == "master" ]; then docker push $IMAGE:latest; fi

# Build and push the Satellite API docker image on deployments
tag_docker_push:
  stage: build
  tags:
    - k8s-docker
  services:
    - docker:20.10.12-dind
  only:
    - /^staging_.*/
    - /^prod_.*/
    - /^testnet_prod_.*/
  except:
    - branches@satellite/satellite-api
  parallel:
    matrix:
      - IMAGE: $API_IMAGE
        DIR: server
      - IMAGE: $SSE_IMAGE
        DIR: sse
  script:
    - cd $DIR
    - docker pull $IMAGE:$CI_COMMIT_REF_SLUG || docker pull $IMAGE:latest || true
    - >
      docker build
      -f Dockerfile
      --network=host
      --cache-from $IMAGE:$CI_COMMIT_REF_SLUG
      --cache-from $IMAGE:latest
      --build-arg BUILDKIT_INLINE_CACHE=1
      -t $IMAGE:$CI_COMMIT_SHA
      -t $IMAGE:$CI_COMMIT_REF_SLUG
      -t $IMAGE:latest
      .      
    - docker push $IMAGE:$CI_COMMIT_SHA
    - docker push $IMAGE:$CI_COMMIT_REF_SLUG
    - if [ $CI_COMMIT_BRANCH == "master" ]; then docker push $IMAGE:latest; fi

# Any newly-pushed, WIP branch will be ran through plan (keep in mind docker images are using the latest tag, which may be an older one since the current commit may not match an available image)
plan_staging:
  stage: plan
  tags:
    - k8s
  only:
    - branches@satellite/satellite-api
  except:
    - cleanup_staging@satellite/satellite-api
    - /^staging_.*/
    - /^prod_.*/
    - /^testnet_prod_.*/
    - /^misc_.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select staging
       && terraform init -input=false
       && terraform plan
         -var "sat_api_docker=$API_IMAGE:$CI_COMMIT_SHA"
         -var "sat_api_sse_docker=$SSE_IMAGE:$CI_COMMIT_SHA"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST_STAGING"
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
         -var "private_bucket=$PRIVATE_BUCKET"
         -var "letsencrypt_email=$LE_EMAIL"
         -var "charge_token=$CHARGE_TOKEN"
         -var "rpcpass=$RPCPASS"
         -var "k8s_autossh_lb=$GKE_LB"
         -var "station1=$STATION_1"
         -var "station2=$STATION_2"
         -input=false)

# Tag with staging_v.* to deploy mainnet + LB to staging (e.g. staging_v0.1.1)
deploy_staging:
  stage: deploy
  when: manual
  tags:
    - k8s
  only:
    - /^staging_v.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select staging
       && terraform init -input=false
       && terraform apply
         -var "sat_api_docker=$API_IMAGE:$CI_COMMIT_SHA"
         -var "sat_api_sse_docker=$SSE_IMAGE:$CI_COMMIT_SHA"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST_STAGING"
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
         -var "private_bucket=$PRIVATE_BUCKET"
         -var "letsencrypt_email=$LE_EMAIL"
         -var "charge_token=$CHARGE_TOKEN"
         -var "rpcpass=$RPCPASS"
         -var "k8s_autossh_lb=$GKE_LB"
         -var "station1=$STATION_1"
         -var "station2=$STATION_2"
         -input=false -auto-approve)

# Tag with prod_v.* to plan mainnet + LB to production (e.g. prod_v0.1.1)
plan_production:
  stage: plan
  tags:
    - k8s
  only:
    - /^prod_v.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select prod
       && terraform init -input=false
       && terraform plan
         -var "sat_api_docker=$API_IMAGE:$CI_COMMIT_SHA"
         -var "sat_api_sse_docker=$SSE_IMAGE:$CI_COMMIT_SHA"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST"
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
         -var "private_bucket=$PRIVATE_BUCKET"
         -var "letsencrypt_email=$LE_EMAIL"
         -var "charge_token=$CHARGE_TOKEN"
         -var "rpcpass=$RPCPASS"
         -var "k8s_autossh_lb=$GKE_LB"
         -var "station1=$STATION_1"
         -var "station2=$STATION_2"
         -input=false)

# Tag with prod_v.* to deploy mainnet + LB to production (e.g. prod_v0.1.1)
deploy_production:
  stage: deploy
  when: manual
  tags:
    - k8s
  only:
    - /^prod_v.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select prod
       && terraform init -input=false
       && terraform apply
         -var "sat_api_docker=$API_IMAGE:$CI_COMMIT_SHA"
         -var "sat_api_sse_docker=$SSE_IMAGE:$CI_COMMIT_SHA"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST"
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
         -var "private_bucket=$PRIVATE_BUCKET"
         -var "letsencrypt_email=$LE_EMAIL"
         -var "charge_token=$CHARGE_TOKEN"
         -var "rpcpass=$RPCPASS"
         -var "k8s_autossh_lb=$GKE_LB"
         -var "station1=$STATION_1"
         -var "station2=$STATION_2"
         -input=false -auto-approve)

# Tag with testnet_prod_v.* to plan testnet to production (e.g. testnet_prod_v0.1.1)
plan_production_testnet:
  stage: plan
  tags:
    - k8s
  only:
    - /^testnet_prod_v.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select testnet-prod
       && terraform init -input=false
       && terraform plan
         -var "sat_api_docker=$API_IMAGE:$CI_COMMIT_SHA"
         -var "sat_api_sse_docker=$SSE_IMAGE:$CI_COMMIT_SHA"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
         -var "private_bucket=$PRIVATE_BUCKET"
         -var "letsencrypt_email=$LE_EMAIL"
         -var "charge_token=$CHARGE_TOKEN"
         -var "rpcpass=$RPCPASS_TESTNET"
         -var "k8s_autossh_lb=$GKE_LB"
         -input=false)

# Tag with testnet_prod_v.* to deploy testnet to production (e.g. testnet_prod_v0.1.1)
deploy_production_testnet:
  stage: deploy
  when: manual
  tags:
    - k8s
  only:
    - /^testnet_prod_v.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select testnet-prod
       && terraform init -input=false
       && terraform apply
         -var "sat_api_docker=$API_IMAGE:$CI_COMMIT_SHA"
         -var "sat_api_sse_docker=$SSE_IMAGE:$CI_COMMIT_SHA"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
         -var "private_bucket=$PRIVATE_BUCKET"
         -var "letsencrypt_email=$LE_EMAIL"
         -var "charge_token=$CHARGE_TOKEN"
         -var "rpcpass=$RPCPASS_TESTNET"
         -var "k8s_autossh_lb=$GKE_LB"
         -input=false -auto-approve)

# This plan gets triggered only for miscellaneous branches/tags (i.e. tor, prometheus, etc), so make sure the branch/tag name starts with misc_
plan_misc:
  stage: plan
  tags:
    - k8s
  only:
    - /^misc_v.*/
  script:
    - (echo -n "$V3_PK" > terraform/modules/tor/v3.pk)
    - (echo -n "$V3_PUBK" > terraform/modules/tor/v3.pubk)
    - (cd terraform
      && terraform init -input=false
      && terraform workspace select misc
      && terraform init -input=false
      && terraform plan
        -var "region=$REGION"
        -var "zone=$ZONE"
        -var "instance_type=$INSTANCE_TYPE"
        -var "onion_host=$ONION_HOST"
        -var "prom_allowed_source_ip=$PROMETHEUS_ALLOWED_SOURCE_IP"
        -var "prom_service_acct=$PROM_SA"
        -var "satellite_lb=$SATELLITE_LB"
        -var "satellite_api_lb=$SATELLITE_API_LB"
        -var "satellite_api_lb_staging=$SATELLITE_API_LB_STAGING"
        -var "blocksat_monitoring=$BLOCKSAT_MONITORING_LB"
        -input=false)

# This deploys only tags/branches starting with misc_v.* (i.e. tor, prometheus, etc)
deploy_misc:
  stage: deploy
  when: manual
  tags:
    - k8s
  only:
    - /^misc_v.*/
  script:
    - (echo -n "$V3_PK" > terraform/modules/tor/v3.pk)
    - (echo -n "$V3_PUBK" > terraform/modules/tor/v3.pubk)
    - (cd terraform
      && terraform init -input=false
      && terraform workspace select misc
      && terraform init -input=false
      && terraform apply
        -var "region=$REGION"
        -var "zone=$ZONE"
        -var "instance_type=$INSTANCE_TYPE"
        -var "onion_host=$ONION_HOST"
        -var "prom_allowed_source_ip=$PROMETHEUS_ALLOWED_SOURCE_IP"
        -var "prom_service_acct=$PROM_SA"
        -var "satellite_lb=$SATELLITE_LB"
        -var "satellite_api_lb=$SATELLITE_API_LB"
        -var "satellite_api_lb_staging=$SATELLITE_API_LB_STAGING"
        -var "blocksat_monitoring=$BLOCKSAT_MONITORING_LB"
        -input=false -auto-approve)

# Pushing to this branch destroys the staging infrastructure
cleanup_staging:
  stage: deploy
  tags:
    - k8s
  only:
    - cleanup_staging@satellite/satellite-api
  script:
    - (cd terraform && terraform init -input=false &&
       terraform workspace select staging &&
       terraform destroy
          -target module.blc.google_compute_instance_group_manager.blc
          -target module.lb.google_compute_region_instance_group_manager.satapi-lb
          -auto-approve)
    - (cd terraform && terraform init -input=false &&
      terraform workspace select testnet-staging &&
       terraform destroy
          -target module.blc.google_compute_instance_group_manager.blc
          -auto-approve)