add x-forwarded-for to nginx logs, rearrange local.vars, switch to mostly tags for deploys, some setup for tests, add Dockerfile.ci

2025-02-20 12:54:28 +01:00 · 2019-01-29 12:17:51 -08:00 · 2019-01-29 12:17:51 -08:00 · ff12c7f919
commit ff12c7f919
parent 414ede8602
4 changed files with 89 additions and 37 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -16,35 +16,41 @@ before_script:
  - gcloud auth list
  - gcloud --version

+# Run tests and build docker images
 build:
  stage: build
+  image: blockstream/satapi-builder@sha256:4f96415f1216c5b6bdb266b1db48558328a497f02da38ce2991ab65b25b7590f
  only:
    - master@satellite/ionosphere
+  before_script:
+    - ""
  script:
-    - docker build -f Dockerfile -t us.gcr.io/blockstream-store/ionosphere:latest -t us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA .
-    - docker push us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA
-    - docker push us.gcr.io/blockstream-store/ionosphere:latest
+    - echo $DOCKERHUB_PW | docker login -u $DOCKERHUB_USER --password-stdin 
+    - docker build -f Dockerfile -t blockstream/satellite-api:latest -t blockstream/satellite-api:$CI_COMMIT_SHA .
+    - docker push blockstream/satellite-api:latest
+    - docker push blockstream/satellite-api:$CI_COMMIT_SHA
    - echo "Building Ionosphere SSE image"
-    - docker build -f Dockerfile.sse -t us.gcr.io/blockstream-store/ionosphere-sse:latest -t us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA sse/
-    - docker push us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA
-    - docker push us.gcr.io/blockstream-store/ionosphere-sse:latest
+    - docker build -f Dockerfile.sse -t blockstream/satellite-api-sse:latest -t blockstream/satellite-api-sse:$CI_COMMIT_SHA sse/
+    - docker push blockstream/satellite-api-sse:$CI_COMMIT_SHA
+    - docker push blockstream/satellite-api-sse:latest

+# Any newly-pushed, WIP branch will be ran through plan (keep in mind docker images are using the latest tag, which may be an older one since the current commit doesn't match an available image)
 plan_satapi:
  stage: plan
  only:
    - branches@satellite/ionosphere
  except:
-    - master@satellite/ionosphere
-    - production@satellite/ionosphere
    - cleanup_staging@satellite/ionosphere
+    - /^staging_.*/
+    - /^prod_.*/
    - /^misc_.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select staging
       && terraform plan
-         -var "ionosphere_docker=us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA"
-         -var "ionosphere_sse_docker=us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA"
+         -var "ionosphere_docker=blockstream/satellite-api:latest"
+         -var "ionosphere_sse_docker=blockstream/satellite-api-sse:latest"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
@ -57,7 +63,7 @@ plan_satapi:
         -var "rpcpass=$RPCPASS"
         -input=false)

-# this plan gets triggered only for miscellaneous branches/tags, so make sure the branch/tag name starts with misc_
+# This plan gets triggered only for miscellaneous branches/tags (i.e. tor and others), so make sure the branch/tag name starts with misc_
 plan_misc:
  stage: plan
  only:
@ -77,17 +83,18 @@ plan_misc:
        -var "onion_host=$ONION_HOST"
        -input=false)

+# Tag with staging_v.* to deploy staging (e.g. staging_v0.1.1)
 deploy_staging:
  stage: deploy
  only:
-    - master@satellite/ionosphere
+    - /^staging_v.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select staging
       && terraform apply
-         -var "ionosphere_docker=us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA"
-         -var "ionosphere_sse_docker=us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA"
+         -var "ionosphere_docker=blockstream/satellite-api:$CI_COMMIT_SHA"
+         -var "ionosphere_sse_docker=blockstream/satellite-api-sse:$CI_COMMIT_SHA"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
@ -100,17 +107,18 @@ deploy_staging:
         -var "rpcpass=$RPCPASS"
         -input=false -auto-approve)

+# Tag with production_v.* to deploy production (e.g. production_v0.1.1)
 deploy_production:
  stage: deploy
  only:
-    - production@satellite/ionosphere
+    - /^prod_v.*/
  script:
    - (cd terraform
       && terraform init -input=false
       && terraform workspace select prod
       && terraform apply
-         -var "ionosphere_docker=us.gcr.io/blockstream-store/ionosphere:$CI_COMMIT_SHA"
-         -var "ionosphere_sse_docker=us.gcr.io/blockstream-store/ionosphere-sse:$CI_COMMIT_SHA"
+         -var "ionosphere_docker=blockstream/satellite-api:$CI_COMMIT_SHA"
+         -var "ionosphere_sse_docker=blockstream/satellite-api-sse:$CI_COMMIT_SHA"
         -var "region=$REGION"
         -var "zone=$ZONE"
         -var "instance_type=$INSTANCE_TYPE"
@ -123,7 +131,7 @@ deploy_production:
         -var "rpcpass=$RPCPASS"
         -input=false -auto-approve)

-# This deploys only tags/branches starting with misc_v
+# This deploys only tags/branches starting with misc_v.* (i.e. tor and others)
 deploy_misc:
  stage: deploy
  only:
@ -141,7 +149,7 @@ deploy_misc:
        -var "onion_host=$ONION_HOST"
         -input=false -auto-approve)

-
+# Pushing to this branch destroys the staging infrastructure
 cleanup_staging:
  stage: deploy
  image:
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@ -0,0 +1,30 @@
+FROM docker@sha256:f7211e1779c82e3a50d0d6f165e83f3e2be217a234b0181d5e8bee411d8fcc5f AS docker
+FROM alpine:latest
+
+COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker
+
+RUN apk --no-cache add curl python py-crcmod bash libc6-compat openssh-client git gnupg
+
+ENV RUBY_PACKAGES ruby ruby-io-console ruby-irb ruby-rake ruby-bundler ruby-bigdecimal ruby-json
+ENV RUBY_DEPS libstdc++ tzdata bash ca-certificates openssl sqlite sqlite-dev
+
+RUN apk update && \
+    apk upgrade && \
+    apk --update add $RUBY_PACKAGES $RUBY_DEPS && \
+    echo 'gem: --no-document' > /etc/gemrc
+
+RUN mkdir /app && \
+    mkdir -p /data/ionosphere
+
+COPY Gemfile /app
+COPY Gemfile.lock /app
+WORKDIR /app
+
+# install packages needed for building compiled gems; install gems; then delete build dependencies to keep Docker image small
+ENV BUILD_PACKAGES sudo build-base ruby-dev libc-dev linux-headers openssl-dev
+RUN apk --update add --virtual build_deps $BUILD_PACKAGES && \
+    bundle install && \
+    apk del build_deps && \
+    rm -rf /var/cache/apk/*
+
+COPY . /app
--- a/terraform/modules/blc/cloud-init/blc.yaml
+++ b/terraform/modules/blc/cloud-init/blc.yaml
@ -34,7 +34,7 @@ write_files:
    permissions: 0644
    owner: root
    content: |
-        log_format withtime '$remote_addr - $remote_user [$time_local] '
+        log_format withtime '$http_x_forwarded_for - $remote_addr - $remote_user [$time_local] '
                            '"$request" $status $body_bytes_sent '
                            '"$http_referer" "$http_user_agent" '
                            'rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
@ -54,29 +54,23 @@ write_files:
            proxy_set_header Host $host;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

+            location / {
+                root /usr/share/nginx/html/;
+                index index.html;
+            }
+
            # Proxy to main ionosphere API
            location /api/ {
-                add_header 'Access-Control-Allow-Origin' null always;
-                add_header 'Access-Control-Allow-Headers' null always;
+                add_header 'Access-Control-Allow-Origin' '*' always;
+                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE' always;
+                add_header 'Access-Control-Allow-Headers' 'X-Auth-Token,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' always;
+                add_header 'X-XSS-Protection' '1; mode=block' always;

                if ($request_method = 'OPTIONS')
                {
-                  add_header 'Access-Control-Allow-Origin' '*';
-                  add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE';
-                  add_header 'Access-Control-Allow-Headers' 'X-Auth-Token,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
-                  add_header 'X-XSS-Protection' '1; mode=block' always;
-
                  return 200;
                }

-                if ($http_origin ~ '(preview.)?blockstream.com')
-                {
-                  add_header 'Access-Control-Allow-Origin' '$http_origin' always;
-                  add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE' always;
-                  add_header 'Access-Control-Allow-Headers' 'X-Auth-Token,DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' always;
-                  add_header 'X-XSS-Protection' '1; mode=block' always;
-                }
-
                proxy_pass http://0.0.0.0:9292/;
            }

@ -92,6 +86,25 @@ write_files:
            }
        }

+  - path: /home/bs/index.html
+    permissions: 0644
+    owner: root
+    content: |
+        <html>
+        <body>
+        <title>Blockstream Satellite API</title>
+
+        <h3>This is our fancy default page. Here are some interesting places worth exploring:</h3>
+        <b>
+        <p>Learn more: <a href="https://blockstream.com/satellite-api-documentation/" target="_blank">Satellite API Docs</a></p>
+        <p>Contribute: <a href="https://github.com/blockstream/satellite-api" target="_blank">Satellite API Code</a></p>
+        </b>
+        <h4>If you're trying to use Tor, make sure you're using the right path. For example:</h4>
+        <p>http://sq65ekkwppmgka5pygi5vbp3l6jkpmxqnue5doiqhdobs55g7cyqsiyd.onion/api/orders/pending</p>
+
+        </body>
+        </html>
+
  - path: /home/bs/check_containers.sh
    permissions: 0744
    owner: root
@ -168,6 +181,7 @@ write_files:
            --pid=host \
            --name=nginx \
            -v /home/bs/default.conf:/etc/nginx/conf.d/default.conf:ro \
+            -v /home/bs/index.html:/usr/share/nginx/html/index.html:ro \
            "nginx:latest"
        ExecStop=/usr/bin/docker stop nginx
        ExecStopPost=/usr/bin/docker rm nginx
--- a/terraform/variables.tf
+++ b/terraform/variables.tf
@ -16,16 +16,16 @@ locals {

    "misc" = {
      env             = ""
-      create_misc     = 1
      create_satapi   = 0
+      create_misc     = 1
      create_builders = 0
    }

    "builders" = {
      env             = ""
-      create_builders = 1
      create_satapi   = 0
      create_misc     = 0
+      create_builders = 1
    }
  }