terminate SSL on instance; misc: add var.project, $request_uri, don't test_build on cleanup

2025-02-20 12:54:28 +01:00 · 2019-06-03 05:36:31 -07:00 · 2019-06-03 05:36:31 -07:00 · 6b361542a0
commit 6b361542a0
parent ce2d18b3ba
10 changed files with 268 additions and 42 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -24,6 +24,7 @@ test_build:
    - branches@satellite/ionosphere
  except:
    - master@satellite/ionosphere
+    - cleanup_staging@satellite/ionosphere
  before_script:
    - export CHARGE_ROOT=$CHARGE_ROOT_CI
    - export REDIS_URI=$REDIS_URI_CI
@ -77,6 +78,8 @@ plan_satapi:
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST_STAGING"
         -var "ssl_cert=$SSL_CERT_STAGING"
+         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
+         -var "letsencrypt_email"=$LE_EMAIL
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "opsgenie_key=$OPSGENIE_KEY"
@ -134,7 +137,6 @@ deploy_misc:
        -var "satellite_api_lb_staging=$SATELLITE_API_LB_STAGING"
         -input=false -auto-approve)

-
 # Tag with staging_v.* to deploy mainnet staging (e.g. staging_v0.1.1)
 deploy_staging:
  stage: deploy
@ -152,6 +154,8 @@ deploy_staging:
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST_STAGING"
         -var "ssl_cert=$SSL_CERT_STAGING"
+         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
+         -var "letsencrypt_email"=$LE_EMAIL
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "opsgenie_key=$OPSGENIE_KEY"
@ -176,6 +180,8 @@ deploy_production:
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST"
         -var "ssl_cert=$SSL_CERT"
+         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
+         -var "letsencrypt_email"=$LE_EMAIL
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "opsgenie_key=$OPSGENIE_KEY"
@ -183,7 +189,7 @@ deploy_production:
         -var "rpcpass=$RPCPASS"
         -input=false -auto-approve)

-# Tag with staging_v.* to deploy testnet staging (e.g. testnet_staging_v0.1.1)
+# Tag with testnet_staging_v.* to deploy testnet staging (e.g. testnet_staging_v0.1.1)
 deploy_staging_testnet:
  stage: deploy
  only:
@ -200,6 +206,8 @@ deploy_staging_testnet:
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST_STAGING"
         -var "ssl_cert=$SSL_CERT_STAGING"
+         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
+         -var "letsencrypt_email"=$LE_EMAIL
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "opsgenie_key=$OPSGENIE_KEY"
@ -224,6 +232,8 @@ deploy_production_testnet:
         -var "instance_type=$INSTANCE_TYPE"
         -var "host=$HOST"
         -var "ssl_cert=$SSL_CERT"
+         -var "public_bucket_url=$PUBLIC_BUCKET_URL"
+         -var "letsencrypt_email"=$LE_EMAIL
         -var "timeout=$TIMEOUT"
         -var "prom_service_acct=$PROM_SA"
         -var "opsgenie_key=$OPSGENIE_KEY"
--- a/terraform/main.tf
+++ b/terraform/main.tf
@ -27,6 +27,7 @@ module "blc-mainnet" {
  ionosphere_docker     = var.ionosphere_docker
  ionosphere_sse_docker = var.ionosphere_sse_docker
  node_exporter_docker  = var.node_exporter_docker
+  certbot_docker        = var.certbot_docker
  net                   = "mainnet"
  env                   = local.env

@ -44,6 +45,8 @@ module "blc-mainnet" {
  opsgenie_key      = var.opsgenie_key
  rpcuser           = var.rpcuser
  rpcpass           = var.rpcpass
+  letsencrypt_email = var.letsencrypt_email
+  public_bucket_url = var.public_bucket_url
 }

 module "blc-testnet" {
@ -58,6 +61,7 @@ module "blc-testnet" {
  ionosphere_docker     = var.ionosphere_docker
  ionosphere_sse_docker = var.ionosphere_sse_docker
  node_exporter_docker  = var.node_exporter_docker
+  certbot_docker        = var.certbot_docker
  net                   = "testnet"
  env                   = local.env

@ -75,6 +79,8 @@ module "blc-testnet" {
  opsgenie_key      = var.opsgenie_key
  rpcuser           = var.rpcuser
  rpcpass           = var.rpcpass
+  letsencrypt_email = var.letsencrypt_email
+  public_bucket_url = var.public_bucket_url
 }

 module "tor" {
--- a/terraform/modules/blc/cloud-init/blc.yaml
+++ b/terraform/modules/blc/cloud-init/blc.yaml
@ -1,4 +1,4 @@
-bootcmd:
+certbot_dockerbootcmd:
  - blkid /dev/disk/by-id/google-data || mkfs.ext4 -L data /dev/disk/by-id/google-data
  - mkdir -p /mnt/disks/data
 mounts:
@ -28,6 +28,37 @@ write_files:
        announce-addr=${announce_addr}
        bind-addr=0.0.0.0

+  - path: /home/bs/default.conf
+    permissions: 0644
+    owner: root
+    content: |
+        log_format withtimebs '$remote_addr - $remote_user [$time_local] '
+                            '"$request" $status $body_bytes_sent '
+                            '"$http_referer" "$http_user_agent" '
+                            'rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
+              
+        server {
+            index index.php index.html index.htm index.nginx-debian.html;
+
+            access_log /var/log/nginx/access-def.log withtime;
+            error_log /var/log/nginx/error-def.log;
+
+            server_name _;
+            listen 80;
+            server_tokens off;
+            proxy_set_header X-Forwarded-For 0.0.0.0;
+
+            location /.well-known {
+                auth_basic off;
+                allow all; # Allow all to see content
+                proxy_pass ${public_bucket_url}certs/.well-known;
+            }
+
+            location / {
+                return 301 https://$host$request_uri;
+            }
+        }
+
  - path: /home/bs/bscom.conf
    permissions: 0644
    owner: root
@ -40,10 +71,15 @@ write_files:
        server {
            access_log /var/log/nginx/access-bs.log withtimebs;
            error_log /var/log/nginx/error-bs.log;
-            root /usr/share/nginx/html/;

            server_name ${host};
-            listen 80;
+            listen 443 ssl;
+            ssl_certificate /etc/nginx/certs/live/${host}/fullchain.pem;
+            ssl_certificate_key /etc/nginx/certs/live/${host}/privkey.pem;
+            ssl_prefer_server_ciphers on;
+            ssl_dhparam /etc/nginx/certs/certs/dhparam.pem;
+
+            root /usr/share/nginx/html/;
            server_tokens off;
            proxy_set_header X-Forwarded-For 0.0.0.0;

@ -60,7 +96,7 @@ write_files:

                if ($request_uri ~* "^${url_path}(/api)?/?$")
                {
-                  return 301 https://$host;
+                  return 301 https://$host$request_uri;
                }
                
                if ($request_method = 'OPTIONS')
@ -95,10 +131,15 @@ write_files:
        server {
            access_log /var/log/nginx/access.log withtime;
            error_log /var/log/nginx/error.log;
-            root /usr/share/nginx/html/;

            server_name ${space_host};
-            listen 80 default_server;
+            listen 443 ssl default_server;
+            ssl_certificate /etc/nginx/certs/live/${space_host}/fullchain.pem;
+            ssl_certificate_key /etc/nginx/certs/live/${space_host}/privkey.pem;
+            ssl_prefer_server_ciphers on;
+            ssl_dhparam /etc/nginx/certs/certs/dhparam.pem;
+
+            root /usr/share/nginx/html/;
            server_tokens off;
            proxy_set_header X-Forwarded-For 0.0.0.0;
            
@ -115,7 +156,7 @@ write_files:

                if ($request_uri ~* "^${url_path}/?$")
                {
-                  return 301 https://$host;
+                  return 301 https://$host$request_uri;
                }

                if ($request_method = 'OPTIONS')
@ -138,7 +179,6 @@ write_files:
            }
        }

-
  - path: /home/bs/index.html
    permissions: 0644
    owner: root
@ -158,6 +198,127 @@ write_files:
        </body>
        </html>

+  - path: /etc/systemd/system/nginx.service
+    permissions: 0644
+    owner: root
+    content: |
+        [Unit]
+        Description=Nginx redirect
+        Wants=gcr-online.target
+        After=ionosphere.service
+
+        [Service]
+        Restart=always
+        RestartSec=3
+        Environment=HOME=/home/bs
+        ExecStartPre=/sbin/iptables -A INPUT -m tcp -p tcp --dport 80 -j ACCEPT
+        ExecStart=/usr/bin/docker run \
+            --network=host \
+            --pid=host \
+            --name=nginx \
+            --log-opt max-size=200m \
+            --log-opt max-file=3 \
+            -v /home/bs/default.conf:/etc/nginx/conf.d/default.conf:ro \
+            "nginx:latest"
+        ExecStop=/usr/bin/docker stop nginx
+        ExecStopPost=/usr/bin/docker rm nginx
+        ExecStopPost=/sbin/iptables -D INPUT -m tcp -p tcp --dport 80 -j ACCEPT
+        
+  - path: /etc/systemd/system/nginx-tls.service
+    permissions: 0644
+    owner: root
+    content: |
+        [Unit]
+        Description=Nginx TLS proxy
+        Wants=gcr-online.target
+        After=nginx.service
+
+        [Service]
+        Restart=always
+        RestartSec=3
+        Environment=HOME=/home/bs
+        ExecStartPre=/sbin/iptables -A INPUT -m tcp -p tcp --dport 443 -j ACCEPT
+        ExecStart=/usr/bin/docker run \
+            --network=host \
+            --pid=host \
+            --name=nginx-tls \
+            --log-opt max-size=200m \
+            --log-opt max-file=3 \
+            -v /home/bs/bscom.conf:/etc/nginx/conf.d/bscom.conf:ro \
+            -v /home/bs/space.conf:/etc/nginx/conf.d/space.conf:ro \
+            -v /home/bs/index.html:/usr/share/nginx/html/index.html:ro \
+            -v /home/bs/certs:/etc/nginx/certs:ro \
+            "nginx:latest"
+        ExecStop=/usr/bin/docker stop nginx-tls
+        ExecStopPost=/usr/bin/docker rm nginx-tls
+        ExecStopPost=/sbin/iptables -D INPUT -m tcp -p tcp --dport 443 -j ACCEPT
+
+
+  - path: /etc/systemd/system/cert-downloader.service
+    permissions: 0644
+    owner: root
+    content: |
+        [Unit]
+        Description=Run cert-downloader
+        Wants=gcr-online.target
+        After=nginx-tls.service
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        Environment=HOME=/home/bs
+        ExecStartPre=/usr/bin/docker-credential-gcr configure-docker
+        ExecStart=/usr/bin/docker run \
+            --name=cert-downloader \
+            --tmpfs /root \
+            --tmpfs /tmp \
+            --rm \
+            -v /home/bs/certs:/etc/letsencrypt:rw \
+            -e GCS_PUBLIC_BUCKET=${public_bucket} \
+            -e GCS_PRIVATE_BUCKET=${private_bucket} \
+            -e DOMAIN=${space_host} \
+            "${certbot_docker}" download
+
+  - path: /etc/systemd/system/cert-renewer.timer
+    permissions: 0644
+    owner: root
+    content: |
+        [Unit]
+        Description=Run full cert-renewer every 24 hours
+
+        [Timer]
+        OnUnitActiveSec=1d
+        Persistent=true
+
+        [Install]
+        WantedBy=timers.target
+
+  - path: /etc/systemd/system/cert-renewer.service
+    permissions: 0644
+    owner: root
+    content: |
+        [Unit]
+        Description=Run cert-renewer
+        Wants=gcr-online.target
+        After=gcr-online.target
+
+        [Service]
+        Type=oneshot
+        Environment=HOME=/home/bs
+        ExecStartPre=/usr/bin/docker-credential-gcr configure-docker
+        ExecStart=/usr/bin/docker run \
+            --name=cert-renewer \
+            --tmpfs /root \
+            --tmpfs /tmp \
+            --rm \
+            -v /home/bs/certs:/etc/letsencrypt:rw \
+            -e GCS_PUBLIC_BUCKET=${public_bucket} \
+            -e GCS_PRIVATE_BUCKET=${private_bucket} \
+            -e DOMAIN=${space_host} \
+            -e EMAIL=${letsencrypt_email} \
+            "${certbot_docker}" renew
+        ExecStartPost=-/usr/bin/systemctl restart nginx-tls
+
  - path: /home/bs/check_containers.sh
    permissions: 0744
    owner: root
@ -214,34 +375,6 @@ write_files:
        [Install]
        WantedBy=timers.target

-  - path: /etc/systemd/system/nginx.service
-    permissions: 0644
-    owner: root
-    content: |
-        [Unit]
-        Description=Nginx proxy
-        Wants=gcr-online.target
-        After=ionosphere.service
-
-        [Service]
-        Restart=always
-        RestartSec=3
-        Environment=HOME=/home/bs
-        ExecStartPre=/sbin/iptables -A INPUT -m tcp -p tcp --dport 80 -j ACCEPT
-        ExecStart=/usr/bin/docker run \
-            --network=host \
-            --pid=host \
-            --name=nginx \
-            --log-opt max-size=200m \
-            --log-opt max-file=3 \
-            -v /home/bs/bscom.conf:/etc/nginx/conf.d/bscom.conf:ro \
-            -v /home/bs/space.conf:/etc/nginx/conf.d/space.conf:ro \
-            -v /home/bs/index.html:/usr/share/nginx/html/index.html:ro \
-            "nginx:latest"
-        ExecStop=/usr/bin/docker stop nginx
-        ExecStopPost=/usr/bin/docker rm nginx
-        ExecStopPost=/sbin/iptables -D INPUT -m tcp -p tcp --dport 80 -j ACCEPT
-
  - path: /etc/systemd/system/node-exporter.service
    permissions: 0644
    owner: root
--- a/terraform/modules/blc/data.tf
+++ b/terraform/modules/blc/data.tf
@ -1,5 +1,5 @@
 data "google_compute_network" "blc" {
-  name = "default"
+  name    = "default"
  project = var.project
 }

@ -33,6 +33,9 @@ data "template_file" "blc" {
    opsgenie_key          = var.opsgenie_key
    host                  = var.host[0]
    space_host            = var.host[1]
+    public_bucket_url     = "${var.public_bucket_url}-${var.env}"
+    public_bucket         = replace(google_storage_bucket.blc-public[count.index].url, "gs://", "")
+    private_bucket        = replace(google_storage_bucket.blc-private[count.index].url, "gs://", "")
  }
 }

--- a/terraform/modules/blc/gcs.tf
+++ b/terraform/modules/blc/gcs.tf
@ -0,0 +1,49 @@
+# Public bucket (certbot acme-challenge)
+resource "google_storage_bucket" "blc-public" {
+  name          = "${var.name}-certbot-${var.env}"
+  location      = "US"
+  storage_class = "MULTI_REGIONAL"
+  count         = var.create_resources
+
+  lifecycle {
+    ignore_changes = ["name"]
+  }
+}
+
+resource "google_storage_bucket_acl" "blc-public-acl" {
+  bucket         = replace(google_storage_bucket.blc-public[count.index].url, "gs://", "")
+  predefined_acl = "publicread"
+  count          = var.create_resources
+}
+
+resource "google_storage_bucket_iam_binding" "blc-public-binding" {
+  bucket = replace(google_storage_bucket.blc-public[count.index].url, "gs://", "")
+  role   = "roles/storage.admin"
+  count  = var.create_resources
+
+  members = [
+    "serviceAccount:${google_service_account.blc[count.index].email}",
+  ]
+}
+
+# Private bucket (server certs)
+resource "google_storage_bucket" "blc-private" {
+  name          = "${var.name}-certs-${var.env}"
+  location      = "US"
+  storage_class = "MULTI_REGIONAL"
+  count         = var.create_resources
+
+  lifecycle {
+    ignore_changes = ["name"]
+  }
+}
+
+resource "google_storage_bucket_iam_binding" "blc-private-binding" {
+  bucket = replace(google_storage_bucket.blc-private[count.index].url, "gs://", "")
+  role   = "roles/storage.admin"
+  count  = var.create_resources
+
+  members = [
+    "serviceAccount:${google_service_account.blc[count.index].email}",
+  ]
+}
--- a/terraform/modules/blc/main.tf
+++ b/terraform/modules/blc/main.tf
@ -42,6 +42,7 @@ resource "google_compute_instance_template" "blc" {
  machine_type = var.instance_type
  region       = var.region
  count        = var.create_resources
+  project      = var.project

  labels = {
    type = "lightning-app"
@ -90,4 +91,3 @@ resource "google_compute_instance_template" "blc" {
    create_before_destroy = true
  }
 }
-
--- a/terraform/modules/blc/variables.tf
+++ b/terraform/modules/blc/variables.tf
@ -72,6 +72,14 @@ variable "prom_service_acct" {
  type = string
 }

+variable "public_bucket_url" {
+  type = string
+}
+
+variable "letsencrypt_email" {
+  type = string
+}
+
 variable "bitcoin_docker" {
  type = string
 }
@ -96,3 +104,6 @@ variable "node_exporter_docker" {
  type = string
 }

+variable "certbot_docker" {
+  type = string
+}
--- a/terraform/modules/prometheus/data.tf
+++ b/terraform/modules/prometheus/data.tf
@ -1,5 +1,5 @@
 data "google_compute_network" "default" {
-  name = "default"
+  name    = "default"
  project = var.project
 }

--- a/terraform/modules/tor/data.tf
+++ b/terraform/modules/tor/data.tf
@ -1,5 +1,5 @@
 data "google_compute_network" "default" {
-  name = "default"
+  name    = "default"
  project = var.project
 }

--- a/terraform/variables.tf
+++ b/terraform/variables.tf
@ -144,6 +144,16 @@ variable "ionosphere_sse_docker" {
  default = ""
 }

+variable "public_bucket_url" {
+  type    = string
+  default = ""
+}
+
+variable "letsencrypt_email" {
+  type    = string
+  default = ""
+}
+
 # Less frequently updated images
 variable "bitcoin_docker" {
  type    = string
@ -180,3 +190,7 @@ variable "gcloud_docker" {
  default = "google/cloud-sdk@sha256:78e68a98c5d6aa36eca45099bae38a1544a1688fd16b506fb914a29fdf6e4afa"
 }

+variable "certbot_docker" {
+  type    = string
+  default = "blockstream/certbot-gcs@sha256:"
+}