fix postgresql 13 recovery (#4521)

* postgres: add comments, backup and remove unused databases
* bats: remove output, use --verbose-run locally
* always drop empty pg 15 cluster
* check cluster correctly, fix port after upgrade
* always restart postgresql.service to fix port

home.admin/config.scripts/bonus.postgresql.sh

@@ -36,8 +36,7 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
   sudo systemctl stop postgresql@$PG_VERSION-main
 
   if [ ! -d /mnt/hdd/app-data/postgresql ]; then
-    # there is no old data
+    echo "# There is no old pg data"
-
     # symlink conf dir
     sudo mkdir -p /mnt/hdd/app-data/postgresql-conf/postgresql
     sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql-conf # fix ownership
@@ -52,7 +51,7 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
     sudo rm -rf $postgres_datadir                                # not a symlink.. delete it silently
     sudo ln -s /mnt/hdd/app-data/postgresql /var/lib/            # create symlink
 
-    echo "# Create PostgreSQL data"
+    echo "# Create PostgreSQL $PG_VERSION data"
     sudo mkdir -p $postgres_datadir/$PG_VERSION/main
     sudo chown -R postgres:postgres $postgres_datadir
 
@@ -61,6 +60,7 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
     sudo pg_ctlcluster $PG_VERSION main start
 
   elif [ -d /mnt/hdd/app-data/postgresql/$PG_VERSION/main ]; then
+    echo "# There is old data for $PG_VERSION, restoring ..."
     if [ -d /mnt/hdd/app-data/postgresql-conf ]; then
       # symlink conf dir
       sudo mkdir -p /mnt/hdd/app-data/postgresql-conf/postgresql
@@ -70,7 +70,7 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
       sudo ln -s /mnt/hdd/app-data/postgresql-conf/postgresql /etc/     # create symlink
     else
       # generate new cluster and use default config
-      echo "# Create PostgreSQL data"
+      echo "# Create $PG_VERSION config"
       sudo mkdir -p $postgres_datadir/$PG_VERSION/main
       sudo chown -R postgres:postgres $postgres_datadir
       sudo pg_createcluster $PG_VERSION main
@@ -106,7 +106,7 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
     sudo pg_ctlcluster $PG_VERSION main start
 
   elif [ -d /mnt/hdd/app-data/postgresql/13/main ]; then
-    # if there is old data for pg 13 start and upgrade cluster
+    echo "# There is old data for pg 13, start and upgrade cluster ..."
     sudo apt install -y postgresql-13 || exit 1
     sudo systemctl stop postgresql
     sudo systemctl stop postgresql@13-main
@@ -119,7 +119,7 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
       sudo ln -s /mnt/hdd/app-data/postgresql-conf/postgresql /etc/     # create symlink
     else
       # generate new cluster and use default config
-      echo "# Create PostgreSQL data"
+      echo "# Create pg 13 config"
       sudo mkdir -p $postgres_datadir/13/main
       sudo chown -R postgres:postgres $postgres_datadir
       # start cluster temporarily
@@ -150,10 +150,31 @@ if [ "$command" = "1" ] || [ "$command" = "on" ]; then
     sudo systemctl start postgresql@13-main
     sudo pg_createcluster 13 main
     sudo pg_ctlcluster 13 main start
+
+    if [ -d /mnt/hdd/app-data/postgresql/$PG_VERSION ] || pg_lsclusters | grep -q "$PG_VERSION  main"; then
+      echo "# backup /mnt/hdd/app-data/postgresql/$PG_VERSION"
+      now=$(date +"%Y_%m_%d_%H%M%S")
+      sudo mv /mnt/hdd/app-data/postgresql/$PG_VERSION /mnt/hdd/app-data/postgresql/$PG_VERSION-backup-$now
+      echo "# Drop empty pg 15 cluster"
+      sudo pg_dropcluster $PG_VERSION main
+    fi
+
     # /usr/bin/pg_upgradecluster [OPTIONS] <old version> <cluster name> [<new data directory>]
     sudo pg_upgradecluster 13 main $postgres_datadir/$PG_VERSION/main || exit 1
+    sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql/$PG_VERSION
+    echo "# backup /mnt/hdd/app-data/postgresql/13"
+    now=$(date +"%Y_%m_%d_%H%M%S")
+    sudo mv /mnt/hdd/app-data/postgresql/13 /mnt/hdd/app-data/postgresql/13-backup-$now
+    sudo pg_dropcluster 13 main
     sudo systemctl disable --now postgresql@13-main
     sudo apt remove -y postgresql-13
+
+    if sudo cat /etc/postgresql/$PG_VERSION/main/postgresql.conf | grep 5433; then
+      echo "# Switch port back to 5432"
+      sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/$PG_VERSION/main/postgresql.conf
+      echo "# Restart posgresql.service"
+      sudo systemctl restart postgresql
+    fi
   fi
 
   # start cluster

test/README.md

@@ -7,6 +7,6 @@ sudo apt install bats
 
 ## Run tests manually
 ```
-bats ./bonus.postgresql-13.bats
+bats --verbose-run ./bonus.postgresql-13.bats
-bats ./bonus.postgresql-15.bats
+bats --verbose-run ./bonus.postgresql-15.bats
 ```

test/bonus.postgresql-13.bats

@@ -23,9 +23,7 @@
   sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE testdb13 TO testuser13;"
   run pg_lsclusters
   [ "$status" -eq 0 ]
-  echo "$output"
   run sudo -u postgres psql -l
-  echo "$output"
   echo "$output" | grep -q "testdb13"
   [ "$?" -eq 0 ]
   echo "$output" | grep -q "testuser13"
@@ -34,16 +32,15 @@
 
 @test "Switch cluster 13 off and move" {
   sudo apt-get remove -y postgresql-13
+  sudo apt-get remove -y postgresql-15
+  sudo apt-get remove -y postgresql
   run ../home.admin/config.scripts/bonus.postgresql.sh off
-  echo "$output"
   [ "$status" -eq 0 ]
   sudo mkdir -p /mnt/hdd/app-data/
   sudo mv /var/lib/postgresql /mnt/hdd/app-data/
-  sudo rm -rf /mnt/hdd/app-data/postgresql/15
   run sudo ls /mnt/hdd/app-data/postgresql/13
   [ "$status" -eq 0 ]
   sudo mv /mnt/hdd/app-data/postgresql /mnt/hdd/app-data/postgresql.bak
-  sudo pg_dropcluster 15 main --stop || true
   run sudo pg_dropcluster 13 main --stop
   [ "$status" -eq 0 ]
 }
@@ -54,10 +51,14 @@
   sudo rm -rf /mnt/hdd/app-data/postgresql-conf.bak
   # run the script
   run ../home.admin/config.scripts/bonus.postgresql.sh on
-  echo "$output"
   [ "$status" -eq 0 ]
+  run pg_lsclusters
+  # check that no 13 cluster is present
+  [ $(echo "$output" | grep -c "13  main") -eq 0 ]
+  # check that no 13 cluster is present
+  echo "$output" | grep -q "15  main"
+  [ "$?" -eq 0 ]
   run sudo -u postgres psql -l
-  echo "$output"
   echo "$output" | grep -q "testdb13"
   [ "$?" -eq 0 ]
   echo "$output" | grep -q "testuser13"
@@ -65,17 +66,15 @@
 }
 
 @test "Create test database (2)" {
-  sudo -u postgres psql -c "CREATE DATABASE testdb TEMPLATE template0 LC_CTYPE 'C' LC_COLLATE 'C' ENCODING 'UTF8';"
+  sudo -u postgres psql -c "CREATE DATABASE testdb15 TEMPLATE template0 LC_CTYPE 'C' LC_COLLATE 'C' ENCODING 'UTF8';"
-  sudo -u postgres psql -c "CREATE USER testuser WITH ENCRYPTED PASSWORD 'raspiblitz';"
+  sudo -u postgres psql -c "CREATE USER testuser15 WITH ENCRYPTED PASSWORD 'raspiblitz';"
-  sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE testdb TO testuser;"
+  sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE testdb15 TO testuser15;"
   run pg_lsclusters
   [ "$status" -eq 0 ]
-  echo "$output"
   run sudo -u postgres psql -l
-  echo "$output"
+  echo "$output" | grep -q "testdb15"
-  echo "$output" | grep -q "testdb"
   [ "$?" -eq 0 ]
-  echo "$output" | grep -q "testuser"
+  echo "$output" | grep -q "testuser15"
   [ "$?" -eq 0 ]
 }
 
@@ -84,8 +83,10 @@
   [ "$status" -eq 0 ]
   run pg_lsclusters
   [ "$status" -eq 0 ]
-  echo "$output"
   sudo pg_dropcluster 15 main --stop || true
   sudo pg_dropcluster 13 main --stop || true
   sudo rm -rf /mnt/hdd/app-data/postgresql*
+  sudo apt-get remove -y postgresql-13
+  sudo apt-get remove -y postgresql-15
+  sudo apt-get remove -y postgresql
 }

test/bonus.postgresql-15.bats

@@ -3,12 +3,10 @@
 @test "Start PostgreSQL cluster" {
   # run the script
   run ../home.admin/config.scripts/bonus.postgresql.sh on
-  echo "$output"
   [ "$status" -eq 0 ]
   # check if PostgreSQL cluster is running
   run pg_lsclusters
   [ "$status" -eq 0 ]
-  echo "$output"
 }
 
 @test "Create test database" {
@@ -18,9 +16,7 @@
   # check if PostgreSQL cluster is running
   run pg_lsclusters
   [ "$status" -eq 0 ]
-  echo "$output"
   run sudo -u postgres psql -l
-  echo "$output"
   echo "$output" | grep -q "testdb"
   [ "$?" -eq 0 ]
   echo "$output" | grep -q "testuser"
@@ -32,7 +28,6 @@
   run ../home.admin/config.scripts/bonus.postgresql.sh off
   # check if PostgreSQL cluster is running
   run pg_lsclusters
-  echo "$output"
   [ "$status" -eq 0 ]
   sudo mv /mnt/hdd/app-data/postgresql /mnt/hdd/app-data/postgresql.bak
   sudo mv /mnt/hdd/app-data/postgresql-conf /mnt/hdd/app-data/postgresql-conf.bak
@@ -47,15 +42,11 @@
   sudo mv /mnt/hdd/app-data/postgresql-conf.bak /mnt/hdd/app-data/postgresql-conf
   # run the script
   run ../home.admin/config.scripts/bonus.postgresql.sh on
-  echo "$output"
   [ "$status" -eq 0 ]
   # check the database
   run pg_lsclusters
-  echo "$output"
   [ "$status" -eq 0 ]
-  echo "$output"
   run sudo -u postgres psql -l
-  echo "$output"
   echo "$output" | grep -q "testdb"
   [ "$?" -eq 0 ]
   echo "$output" | grep -q "testuser"
@@ -64,10 +55,8 @@
 
 @test "Switch cluster off and move (2)" {
   run ../home.admin/config.scripts/bonus.postgresql.sh off
-  echo "$output"
   [ "$status" -eq 0 ]
   run pg_lsclusters
-  echo "$output"
   [ "$status" -eq 0 ]
   sudo mv /mnt/hdd/app-data/postgresql /mnt/hdd/app-data/postgresql.bak
   sudo mv /mnt/hdd/app-data/postgresql-conf /mnt/hdd/app-data/postgresql-conf.bak
@@ -83,13 +72,10 @@
   sudo rm -rf /etc/postgresql
   sudo rm -rf /mnt/hdd/app-data/postgresql-conf.bak
   run ../home.admin/config.scripts/bonus.postgresql.sh on
-  echo "$output"
   [ "$status" -eq 0 ]
   run pg_lsclusters
   [ "$status" -eq 0 ]
-  echo "$output"
   run sudo -u postgres psql -l
-  echo "$output"
   echo "$output" | grep -q "testdb"
   [ "$?" -eq 0 ]
   echo "$output" | grep -q "testuser"
@@ -98,11 +84,9 @@
 
 @test "Cleanup" {
   run ../home.admin/config.scripts/bonus.postgresql.sh off
-  echo "$output"
   [ "$status" -eq 0 ]
   run pg_lsclusters
   [ "$status" -eq 0 ]
-  echo "$output"
   sudo pg_dropcluster 15 main --stop || true
   sudo pg_dropcluster 13 main --stop || true
   sudo rm -rf /mnt/hdd/app-data/postgresql*