add autossh.service tunnel to on-prem k8s

.gitlab-ci.yml

@@ -80,13 +80,14 @@ plan_satapi:
          -var "timeout=$TIMEOUT"
          -var "prom_service_acct=$PROM_SA"
          -var "opsgenie_key=$OPSGENIE_KEY"
-         -var "rpcuser=$RPCUSER"
-         -var "rpcpass=$RPCPASS"
          -var "public_bucket_url=$PUBLIC_BUCKET_URL"
          -var "letsencrypt_email=$LE_EMAIL"
          -var "lb_svc_acct=$LB_SA_STAGING"
          -var "pguser=$PGUSER"
          -var "pgpass=$PGPASS"
+         -var "charge_token=$CHARGE_TOKEN"
+         -var "rpcpass=$RPCPASS_TESTNET"
+         -var "k8s_autossh_lb=$GKE_LB"
          -input=false)
          
 # This plan gets triggered only for miscellaneous branches/tags (i.e. tor, prometheus, etc), so make sure the branch/tag name starts with misc_
@@ -158,12 +159,13 @@ deploy_staging:
          -var "timeout=$TIMEOUT"
          -var "prom_service_acct=$PROM_SA"
          -var "opsgenie_key=$OPSGENIE_KEY"
-         -var "rpcuser=$RPCUSER"
-         -var "rpcpass=$RPCPASS"
          -var "public_bucket_url=$PUBLIC_BUCKET_URL"
          -var "letsencrypt_email=$LE_EMAIL"
          -var "pguser=$PGUSER"
          -var "pgpass=$PGPASS"
+         -var "charge_token=$CHARGE_TOKEN"
+         -var "rpcpass=$RPCPASS"
+         -var "k8s_autossh_lb=$GKE_LB"
          -input=false -auto-approve)
 
 # Tag with production_v.* to deploy mainnet production (e.g. prod_v0.1.1)
@@ -185,12 +187,13 @@ deploy_production:
          -var "timeout=$TIMEOUT"
          -var "prom_service_acct=$PROM_SA"
          -var "opsgenie_key=$OPSGENIE_KEY"
-         -var "rpcuser=$RPCUSER"
-         -var "rpcpass=$RPCPASS"
          -var "public_bucket_url=$PUBLIC_BUCKET_URL"
          -var "letsencrypt_email=$LE_EMAIL"
          -var "pguser=$PGUSER"
          -var "pgpass=$PGPASS"
+         -var "charge_token=$CHARGE_TOKEN"
+         -var "rpcpass=$RPCPASS"
+         -var "k8s_autossh_lb=$GKE_LB"
          -input=false -auto-approve)
 
 # Tag with testnet_staging_v.* to deploy testnet staging (e.g. testnet_staging_v0.1.1)
@@ -212,13 +215,14 @@ deploy_staging_testnet:
          -var "timeout=$TIMEOUT"
          -var "prom_service_acct=$PROM_SA"
          -var "opsgenie_key=$OPSGENIE_KEY"
-         -var "rpcuser=$RPCUSER"
-         -var "rpcpass=$RPCPASS"
          -var "public_bucket_url=$PUBLIC_BUCKET_URL"
          -var "letsencrypt_email=$LE_EMAIL"
          -var "lb_svc_acct=$LB_SA_STAGING"
          -var "pguser=$PGUSER"
          -var "pgpass=$PGPASS"
+         -var "charge_token=$CHARGE_TOKEN"
+         -var "rpcpass=$RPCPASS_TESTNET"
+         -var "k8s_autossh_lb=$GKE_LB"
          -input=false -auto-approve)
 
 # Tag with production_v.* to deploy testnet production (e.g. testnet_prod_v0.1.1)
@@ -240,13 +244,14 @@ deploy_production_testnet:
          -var "timeout=$TIMEOUT"
          -var "prom_service_acct=$PROM_SA"
          -var "opsgenie_key=$OPSGENIE_KEY"
-         -var "rpcuser=$RPCUSER"
-         -var "rpcpass=$RPCPASS"
          -var "public_bucket_url=$PUBLIC_BUCKET_URL"
          -var "letsencrypt_email=$LE_EMAIL"
          -var "lb_svc_acct=$LB_SA"
          -var "pguser=$PGUSER"
          -var "pgpass=$PGPASS"
+         -var "charge_token=$CHARGE_TOKEN"
+         -var "rpcpass=$RPCPASS_TESTNET"
+         -var "k8s_autossh_lb=$GKE_LB"
          -input=false -auto-approve)
 
 # Pushing to this branch destroys the staging infrastructure

terraform/main.tf

@@ -21,13 +21,14 @@ module "blc-mainnet" {
   project               = var.project
   name                  = "satellite-api"
   network               = "default"
-  bitcoin_docker        = var.bitcoin_docker
   lightning_docker      = var.lightning_docker
   charge_docker         = var.charge_docker
   ionosphere_docker     = var.ionosphere_docker
   ionosphere_sse_docker = var.ionosphere_sse_docker
   node_exporter_docker  = var.node_exporter_docker
   postgres_docker       = var.postgres_docker
+  autossh_docker        = var.autossh_docker
+  certbot_docker        = var.certbot_docker
   net                   = "mainnet"
   env                   = local.env
   lb_svc_acct           = module.lb.lb_svc_acct
@@ -42,10 +43,11 @@ module "blc-mainnet" {
   timeout           = var.timeout
   prom_service_acct = var.prom_service_acct
   opsgenie_key      = var.opsgenie_key
-  rpcuser           = var.rpcuser
   rpcpass           = var.rpcpass
   pguser            = var.pguser
   pgpass            = var.pgpass
+  charge_token      = var.charge_token
+  k8s_autossh_lb    = var.k8s_autossh_lb
 }
 
 module "blc-testnet" {
@@ -54,13 +56,14 @@ module "blc-testnet" {
   project               = var.project
   name                  = "satellite-api"
   network               = "default"
-  bitcoin_docker        = var.bitcoin_docker
   lightning_docker      = var.lightning_docker
   charge_docker         = var.charge_docker
   ionosphere_docker     = var.ionosphere_docker
   ionosphere_sse_docker = var.ionosphere_sse_docker
   node_exporter_docker  = var.node_exporter_docker
   postgres_docker       = var.postgres_docker
+  autossh_docker        = var.autossh_docker
+  certbot_docker        = var.certbot_docker
   net                   = "testnet"
   env                   = local.env
   cert_bucket           = "" #data.terraform_remote_state.blc-mainnet.outputs.lb_cert_bucket
@@ -74,11 +77,12 @@ module "blc-testnet" {
   timeout           = var.timeout
   prom_service_acct = var.prom_service_acct
   opsgenie_key      = var.opsgenie_key
-  rpcuser           = var.rpcuser
   rpcpass           = var.rpcpass
   lb_svc_acct       = var.lb_svc_acct
   pguser            = var.pguser
   pgpass            = var.pgpass
+  charge_token      = var.charge_token
+  k8s_autossh_lb    = var.k8s_autossh_lb
 }
 
 module "lb" {

terraform/modules/blc/cloud-init/blc.yaml

@@ -121,47 +121,61 @@ write_files:
         ExecStopPost=/usr/bin/docker rm postgres
         ExecStopPost=/sbin/iptables -D INPUT -p tcp -s localhost --dport 5432 -j ACCEPT
 
-  - path: /home/bs/bitcoin.conf
-    permissions: 0644
-    owner: root
-    content: |
-        rpcuser=${rpcuser}
-        rpcpassword=${rpcpass}
-        txindex=1
-        dbcache=4000
-
-  - path: /etc/systemd/system/bitcoin.service
+  - path: /etc/systemd/system/autossh-key-downloader.service
     permissions: 0644
     owner: root
     content: |
         [Unit]
-        Description=Bitcoin node
+        Description=Download SSH privkey from GCS
         Wants=gcr-online.target
-        After=gcr-online.service
+        After=gcr-online.target
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        Environment=HOME=/home/bs
+        ExecStart=/usr/bin/docker run \
+            --name=autosssh-key \
+            --tmpfs /root \
+            --tmpfs /tmp \
+            --rm \
+            -v /home/bs:/mnt/bs:rw \
+            "${certbot_docker}" /google-cloud-sdk/bin/gsutil -m cp -r gs://${cert_bucket}/k8s_keys/* /mnt/bs/
+        ExecStartPost=-/bin/chmod 0600 /home/bs/k8s_autossh.key
+        ExecStopPost=-/bin/chmod 0600 /home/bs/k8s_autossh.key
+
+  - path: /etc/systemd/system/k8s-autossh.service
+    permissions: 0644
+    owner: root
+    content: |
+        [Unit]
+        Description=SSH tunnel to on-prem K8s node
+        Wants=gcr-online.target
+        After=autossh-key-downloader.service
 
         [Service]
         Restart=always
-        RestartSec=3
+        RestartSec=5
         Environment=HOME=/home/bs
-        ExecStartPre=/usr/bin/docker pull ${bitcoin_docker}
+        ExecStartPre=/sbin/iptables -A INPUT -p tcp -s localhost --dport ${k8s_autossh_btc_port} -j ACCEPT
         ExecStart=/usr/bin/docker run \
             --network=host \
-            --pid=host \
-            --name=bitcoin \
-            --log-opt max-size=1g \
-            -v /home/bs/bitcoin.conf:/root/.bitcoin/bitcoin.conf:ro \
-            -v /mnt/disks/data/${net}:/root/.bitcoin:rw \
-            "${bitcoin_docker}" ${bitcoin_cmd}
-        ExecStop=/usr/bin/docker exec bitcoin bitcoin-cli stop
-        ExecStopPost=/usr/bin/sleep 3
-        ExecStopPost=/usr/bin/docker rm -f bitcoin
+            --name=k8s-autossh \
+            -e AUTOSSH_GATETIME=0 \
+            -v /home/bs/k8s_autossh.key:/root/.ssh/id_ed25519:ro \
+            ${autossh_docker} ${k8s_autossh_btc_port}:localhost:${k8s_autossh_btc_port} -p ${k8s_autossh_ssh_port} root@${k8s_autossh_lb}
+        ExecStop=/usr/bin/docker stop k8s-autossh
+        ExecStopPost=/usr/bin/docker rm k8s-autossh
+        ExecStopPost=/sbin/iptables -D INPUT -p tcp -s localhost --dport ${k8s_autossh_btc_port} -j ACCEPT
 
   - path: /home/bs/lightning.conf
     permissions: 0644
     owner: root
     content: |
+        log-level=debug
+        plugin-dir=/usr/local/bin/plugins
         alias=ionosphere-${net}
-        bitcoin-rpcuser=${rpcuser}
+        bitcoin-rpcuser=${net}-def
         bitcoin-rpcpassword=${rpcpass}
         announce-addr=${announce_addr}
         bind-addr=0.0.0.0
@@ -173,7 +187,7 @@ write_files:
         [Unit]
         Description=Lightning node
         Wants=gcr-online.target
-        After=bitcoin.service
+        After=k8s-autossh.service
 
         [Service]
         Restart=always
@@ -186,6 +200,8 @@ write_files:
             --network=host \
             --pid=host \
             --name=lightning \
+            --cap-add=SYS_PTRACE \
+            --memory=2g \
             --log-opt max-size=1g \
             -v /home/bs/lightning.conf:/root/.lightning/lightning.conf:ro \
             -v /mnt/disks/data/lightning:/root/.lightning:rw \
@@ -251,7 +267,7 @@ write_files:
             --log-opt max-file=3 \
             -v /mnt/disks/data/ionosphere:/data \
             -e "RACK_ENV=production" \
-            -e "CHARGE_ROOT=http://api-token:${rpcpass}@localhost:9112" \
+            -e "CHARGE_ROOT=http://api-token:${charge_token}@localhost:9112" \
             -e "CALLBACK_URI_ROOT=http://localhost:9292" \
             "${ionosphere_docker}"
         ExecStop=/usr/bin/docker stop ionosphere
@@ -327,7 +343,7 @@ write_files:
             --name=charge \
             -v /mnt/disks/data/lightning:/root/.lightning:ro \
             -v /mnt/disks/data/charge:/data:rw \
-            -e "API_TOKEN=${rpcpass}" \
+            -e "API_TOKEN=${charge_token}" \
             "${charge_docker}" ${charge_cmd}
         ExecStop=/usr/bin/docker stop charge
         ExecStopPost=/usr/bin/docker rm charge
@@ -335,8 +351,10 @@ write_files:
 
 runcmd:
   - systemctl daemon-reload
-  - systemctl start bitcoin.service
-  - systemctl enable bitcoin.service
+  - systemctl start autossh-key-downloader.service
+  - systemctl enable autossh-key-downloader.service
+  - systemctl start k8s-autossh.service
+  - systemctl enable k8s-autossh.service
   - systemctl start lightning.service
   - systemctl enable lightning.service
   - systemctl start postgres.service

terraform/modules/blc/data.tf

@@ -14,15 +14,12 @@ data "template_file" "blc" {
   count    = var.create_resources
 
   vars = {
-    rpcuser               = var.rpcuser
-    rpcpass               = var.rpcpass
+    charge_token          = var.charge_token
     net                   = var.net
-    bitcoin_cmd           = "bitcoind ${var.net == "testnet" ? "-testnet" : ""} -printtoconsole"
     lightning_cmd         = "lightningd ${var.net == "testnet" ? "--testnet" : "--mainnet"} --conf=/root/.lightning/lightning.conf --plugin-dir=/usr/local/bin/plugins"
     charge_cmd            = "charged -d /data/charge.db -l /root/.lightning"
     announce_addr         = google_compute_address.blc[0].address
     lightning_port        = 9735
-    bitcoin_docker        = var.bitcoin_docker
     lightning_docker      = var.lightning_docker
     charge_docker         = var.charge_docker
     redis_port            = 6379
@@ -30,9 +27,16 @@ data "template_file" "blc" {
     ionosphere_sse_docker = var.ionosphere_sse_docker
     node_exporter_docker  = var.node_exporter_docker
     postgres_docker       = var.postgres_docker
+    autossh_docker        = var.autossh_docker
+    certbot_docker        = var.certbot_docker
     pguser                = var.pguser
     pgpass                = var.pgpass
     opsgenie_key          = var.opsgenie_key
+    k8s_autossh_lb        = var.k8s_autossh_lb
+    rpcpass               = var.rpcpass
+    k8s_autossh_ssh_port  = "${var.net == "testnet" ? "2222" : "2223"}"
+    k8s_autossh_btc_port  = "${var.net == "testnet" ? "18332" : "8332"}"
+    cert_bucket           = var.cert_bucket
   }
 }
 

terraform/modules/blc/variables.tf

@@ -12,11 +12,7 @@ variable "create_resources" {
   type = string
 }
 
-variable "rpcuser" {
-  type = string
-}
-
-variable "rpcpass" {
+variable "charge_token" {
   type = string
 }
 
@@ -72,12 +68,16 @@ variable "pgpass" {
   type = string
 }
 
-variable "cert_bucket" {
-  type = string
+variable "k8s_autossh_lb" {
+  type    = string
 }
 
-variable "bitcoin_docker" {
-  type = string
+variable "rpcpass" {
+  type    = string
+}
+
+variable "cert_bucket" {
+  type    = string
 }
 
 variable "charge_docker" {
@@ -102,4 +102,12 @@ variable "node_exporter_docker" {
 
 variable "postgres_docker" {
   type = string
+}
+
+variable "autossh_docker" {
+  type = string
+}
+
+variable "certbot_docker" {
+  type = string
 }

terraform/modules/lb/gcs.tf

@@ -17,7 +17,7 @@ resource "google_storage_bucket_acl" "satapi-lb-public-acl" {
   count          = var.create_resources
 }
 
-# Private bucket (server certs)
+# Private bucket (server certs, ssh keys)
 resource "google_storage_bucket" "satapi-lb-private" {
   name          = "${var.name}-certs-${var.env}"
   location      = "US"

terraform/variables.tf

@@ -58,12 +58,7 @@ variable "target_pool" {
   default = ""
 }
 
-variable "rpcuser" {
-  type    = string
-  default = ""
-}
-
-variable "rpcpass" {
+variable "charge_token" {
   type    = string
   default = ""
 }
@@ -158,6 +153,16 @@ variable "pgpass" {
   default = ""
 }
 
+variable "k8s_autossh_lb" {
+  type    = string
+  default = ""
+}
+
+variable "rpcpass" {
+  type    = string
+  default = ""
+}
+
 # Overwritten by CI
 variable "public_bucket_url" {
   type    = string
@@ -180,11 +185,6 @@ variable "ionosphere_sse_docker" {
 }
 
 # Less frequently updated images
-variable "bitcoin_docker" {
-  type    = string
-  default = "blockstream/bitcoind@sha256:70f5ed2674975cf353b3ff07e85e23bb6e3dd6082dc3de91ce5fd06b6f16395a"
-}
-
 variable "lightning_docker" {
   type    = string
   default = "blockstream/lightningd@sha256:3aab864ba0ee4bf1191c6243bf4bc00f99d29590f5d7bce4340c5b5a9f2b4c98"
@@ -217,10 +217,15 @@ variable "gcloud_docker" {
 
 variable "certbot_docker" {
   type    = string
-  default = "blockstream/certbot-gcs@sha256:516ba43a03f558c73cd3807dc2b31a3ad123205dd53682a5da70396b75b53881"
+  default = "blockstream/certbot-gcs@sha256:fc5d7cb31bcf04169f37cbebd74c3bde49651f79e54e1ff3c3eaf6ec47b9f6d0"
 }
 
 variable "postgres_docker" {
   type    = string
   default = "postgres@sha256:077793cc0ed31fd0568ce468d85d0843b8dea37c9ef74eb81b4ccf0fe9539e2e"
 }
+
+variable "autossh_docker" {
+  type    = "string"
+  default = "blockstream/autossh@sha256:5e30a60d6ef17aeafdde63bb859238e132fadef174af4092a435bc7325430ebd"
+}