From ff12c7f9194538a97f1e63e63c801e75506e6045 Mon Sep 17 00:00:00 2001 From: nitram Date: Tue, 29 Jan 2019 12:17:51 -0800 Subject: [PATCH] add x-forwarded-for to nginx logs, rearrange local.vars, switch to mostly tags for deploys, some setup for tests, add Dockerfile.ci --- .gitlab-ci.yml | 46 +++++++++++++---------- Dockerfile.ci | 30 +++++++++++++++ terraform/modules/blc/cloud-init/blc.yaml | 46 +++++++++++++++-------- terraform/variables.tf | 4 +- 4 files changed, 89 insertions(+), 37 deletions(-) create mode 100644 Dockerfile.ci diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46bb18c..f4446d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,35 +16,41 @@ before_script: - gcloud auth list - gcloud --version +# Run tests and build docker images build: stage: build + image: blockstream/satapi-builder@sha256:4f96415f1216c5b6bdb266b1db48558328a497f02da38ce2991ab65b25b7590f only: - master@satellite/ionosphere + before_script: + - "" script: - - docker build -f Dockerfile -t us.gcr.io/blockstream-store/ionosphere:latest -t us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA . - - docker push us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA - - docker push us.gcr.io/blockstream-store/ionosphere:latest + - echo $DOCKERHUB_PW | docker login -u $DOCKERHUB_USER --password-stdin + - docker build -f Dockerfile -t blockstream/satellite-api:latest -t blockstream/satellite-api:$CI_COMMIT_SHA . + - docker push blockstream/satellite-api:latest + - docker push blockstream/satellite-api:$CI_COMMIT_SHA - echo "Building Ionosphere SSE image" - - docker build -f Dockerfile.sse -t us.gcr.io/blockstream-store/ionosphere-sse:latest -t us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA sse/ - - docker push us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA - - docker push us.gcr.io/blockstream-store/ionosphere-sse:latest + - docker build -f Dockerfile.sse -t blockstream/satellite-api-sse:latest -t blockstream/satellite-api-sse:$CI_COMMIT_SHA sse/ + - docker push blockstream/satellite-api-sse:$CI_COMMIT_SHA + - docker push blockstream/satellite-api-sse:latest +# Any newly-pushed, WIP branch will be ran through plan (keep in mind docker images are using the latest tag, which may be an older one since the current commit doesn't match an available image) plan_satapi: stage: plan only: - branches@satellite/ionosphere except: - - master@satellite/ionosphere - - production@satellite/ionosphere - cleanup_staging@satellite/ionosphere + - /^staging_.*/ + - /^prod_.*/ - /^misc_.*/ script: - (cd terraform && terraform init -input=false && terraform workspace select staging && terraform plan - -var "ionosphere_docker=us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA" - -var "ionosphere_sse_docker=us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA" + -var "ionosphere_docker=blockstream/satellite-api:latest" + -var "ionosphere_sse_docker=blockstream/satellite-api-sse:latest" -var "region=$REGION" -var "zone=$ZONE" -var "instance_type=$INSTANCE_TYPE" @@ -57,7 +63,7 @@ plan_satapi: -var "rpcpass=$RPCPASS" -input=false) -# this plan gets triggered only for miscellaneous branches/tags, so make sure the branch/tag name starts with misc_ +# This plan gets triggered only for miscellaneous branches/tags (i.e. tor and others), so make sure the branch/tag name starts with misc_ plan_misc: stage: plan only: @@ -77,17 +83,18 @@ plan_misc: -var "onion_host=$ONION_HOST" -input=false) +# Tag with staging_v.* to deploy staging (e.g. staging_v0.1.1) deploy_staging: stage: deploy only: - - master@satellite/ionosphere + - /^staging_v.*/ script: - (cd terraform && terraform init -input=false && terraform workspace select staging && terraform apply - -var "ionosphere_docker=us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA" - -var "ionosphere_sse_docker=us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA" + -var "ionosphere_docker=blockstream/satellite-api:$CI_COMMIT_SHA" + -var "ionosphere_sse_docker=blockstream/satellite-api-sse:$CI_COMMIT_SHA" -var "region=$REGION" -var "zone=$ZONE" -var "instance_type=$INSTANCE_TYPE" @@ -100,17 +107,18 @@ deploy_staging: -var "rpcpass=$RPCPASS" -input=false -auto-approve) +# Tag with production_v.* to deploy production (e.g. production_v0.1.1) deploy_production: stage: deploy only: - - production@satellite/ionosphere + - /^prod_v.*/ script: - (cd terraform && terraform init -input=false && terraform workspace select prod && terraform apply - -var "ionosphere_docker=us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA" - -var "ionosphere_sse_docker=us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA" + -var "ionosphere_docker=blockstream/satellite-api:$CI_COMMIT_SHA" + -var "ionosphere_sse_docker=blockstream/satellite-api-sse:$CI_COMMIT_SHA" -var "region=$REGION" -var "zone=$ZONE" -var "instance_type=$INSTANCE_TYPE" @@ -123,7 +131,7 @@ deploy_production: -var "rpcpass=$RPCPASS" -input=false -auto-approve) -# This deploys only tags/branches starting with misc_v +# This deploys only tags/branches starting with misc_v.* (i.e. tor and others) deploy_misc: stage: deploy only: @@ -141,7 +149,7 @@ deploy_misc: -var "onion_host=$ONION_HOST" -input=false -auto-approve) - +# Pushing to this branch destroys the staging infrastructure cleanup_staging: stage: deploy image: diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 0000000..85957a3 --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,30 @@ +FROM docker@sha256:f7211e1779c82e3a50d0d6f165e83f3e2be217a234b0181d5e8bee411d8fcc5f AS docker +FROM alpine:latest + +COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker + +RUN apk --no-cache add curl python py-crcmod bash libc6-compat openssh-client git gnupg + +ENV RUBY_PACKAGES ruby ruby-io-console ruby-irb ruby-rake ruby-bundler ruby-bigdecimal ruby-json +ENV RUBY_DEPS libstdc++ tzdata bash ca-certificates openssl sqlite sqlite-dev + +RUN apk update && \ + apk upgrade && \ + apk --update add $RUBY_PACKAGES $RUBY_DEPS && \ + echo 'gem: --no-document' > /etc/gemrc + +RUN mkdir /app && \ + mkdir -p /data/ionosphere + +COPY Gemfile /app +COPY Gemfile.lock /app +WORKDIR /app + +# install packages needed for building compiled gems; install gems; then delete build dependencies to keep Docker image small +ENV BUILD_PACKAGES sudo build-base ruby-dev libc-dev linux-headers openssl-dev +RUN apk --update add --virtual build_deps $BUILD_PACKAGES && \ + bundle install && \ + apk del build_deps && \ + rm -rf /var/cache/apk/* + +COPY . /app diff --git a/terraform/modules/blc/cloud-init/blc.yaml b/terraform/modules/blc/cloud-init/blc.yaml index 064dcf9..299322e 100644 --- a/terraform/modules/blc/cloud-init/blc.yaml +++ b/terraform/modules/blc/cloud-init/blc.yaml @@ -34,7 +34,7 @@ write_files: permissions: 0644 owner: root content: | - log_format withtime '$remote_addr - $remote_user [$time_local] ' + log_format withtime '$http_x_forwarded_for - $remote_addr - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent" ' 'rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; @@ -54,29 +54,23 @@ write_files: proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + location / { + root /usr/share/nginx/html/; + index index.html; + } + # Proxy to main ionosphere API location /api/ { - add_header 'Access-Control-Allow-Origin' null always; - add_header 'Access-Control-Allow-Headers' null always; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'X-Auth-Token,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' always; + add_header 'X-XSS-Protection' '1; mode=block' always; if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; - add_header 'Access-Control-Allow-Headers' 'X-Auth-Token,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'X-XSS-Protection' '1; mode=block' always; - return 200; } - if ($http_origin ~ '(preview.)?blockstream.com') - { - add_header 'Access-Control-Allow-Origin' '$http_origin' always; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE' always; - add_header 'Access-Control-Allow-Headers' 'X-Auth-Token,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' always; - add_header 'X-XSS-Protection' '1; mode=block' always; - } - proxy_pass http://0.0.0.0:9292/; } @@ -92,6 +86,25 @@ write_files: } } + - path: /home/bs/index.html + permissions: 0644 + owner: root + content: | + + + Blockstream Satellite API + +

This is our fancy default page. Here are some interesting places worth exploring:

+ +

Learn more: Satellite API Docs

+

Contribute: Satellite API Code

+ +

If you're trying to use Tor, make sure you're using the right path. For example:

+

http://sq65ekkwppmgka5pygi5vbp3l6jkpmxqnue5doiqhdobs55g7cyqsiyd.onion/api/orders/pending

+ + + + - path: /home/bs/check_containers.sh permissions: 0744 owner: root @@ -168,6 +181,7 @@ write_files: --pid=host \ --name=nginx \ -v /home/bs/default.conf:/etc/nginx/conf.d/default.conf:ro \ + -v /home/bs/index.html:/usr/share/nginx/html/index.html:ro \ "nginx:latest" ExecStop=/usr/bin/docker stop nginx ExecStopPost=/usr/bin/docker rm nginx diff --git a/terraform/variables.tf b/terraform/variables.tf index 9b41318..4b5fbeb 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -16,16 +16,16 @@ locals { "misc" = { env = "" - create_misc = 1 create_satapi = 0 + create_misc = 1 create_builders = 0 } "builders" = { env = "" - create_builders = 1 create_satapi = 0 create_misc = 0 + create_builders = 1 } }