From 655f5cb7b08d090fd5e5a54d5ee7f300bb975aab Mon Sep 17 00:00:00 2001 From: openoms <43343391+openoms@users.noreply.github.com> Date: Tue, 2 Apr 2024 12:03:02 +0200 Subject: [PATCH] Postgres migration fix and test (#4510) --- .github/workflows/test-bats.yml | 38 ++++ home.admin/config.scripts/bonus.postgresql.sh | 182 +++++++++++++----- test/README.md | 12 ++ test/bonus.postgresql-13.bats | 91 +++++++++ test/bonus.postgresql-15.bats | 109 +++++++++++ typos.toml | 3 +- 6 files changed, 386 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/test-bats.yml mode change 100644 => 100755 home.admin/config.scripts/bonus.postgresql.sh create mode 100644 test/README.md create mode 100644 test/bonus.postgresql-13.bats create mode 100644 test/bonus.postgresql-15.bats diff --git a/.github/workflows/test-bats.yml b/.github/workflows/test-bats.yml new file mode 100644 index 000000000..b4a64a78a --- /dev/null +++ b/.github/workflows/test-bats.yml @@ -0,0 +1,38 @@ +name: Test bats + +concurrency: + group: test-bats-${{ github.head_ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: ["dev"] + paths: + - "home.admin/config.scripts/bonus.postgresql.sh" + pull_request: + branches: ["dev"] + paths: + - "home.admin/config.scripts/bonus.postgresql.sh" + +jobs: + run-bats-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install bats + run: | + sudo apt update &>/dev/null + sudo apt install -y bats + + - name: Run the bats tests with postgresql 15 + run: | + cd test + sudo bats ./bonus.postgresql-15.bats + + - name: Run the bats tests with postgresql 13 + run: | + cd test + sudo bats ./bonus.postgresql-13.bats diff --git a/home.admin/config.scripts/bonus.postgresql.sh b/home.admin/config.scripts/bonus.postgresql.sh old mode 100644 new mode 100755 index 334c2365f..d3cbb8619 --- a/home.admin/config.scripts/bonus.postgresql.sh +++ b/home.admin/config.scripts/bonus.postgresql.sh @@ -16,66 +16,155 @@ db_user=$3 db_user_pw=$4 db_backupfile=$5 +PG_VERSION=15 +echo "# Using the default PostgreSQL version: $PG_VERSION" + # switch on if [ "$command" = "1" ] || [ "$command" = "on" ]; then - - # https://github.com/rootzoll/raspiblitz/issues/3218 echo "# Install PostgreSQL" + if [ ! -f /etc/apt/trusted.gpg.d/postgresql.gpg ]; then + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg + echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt update + fi + sudo apt install -y postgresql-$PG_VERSION - sudo apt install -y postgresql postgres_datadir="/var/lib/postgresql" # default data dir + postgres_confdir="/etc/postgresql" # default conf dir - # sudo -u postgres psql -c "show data_directory" - if [ ! -d $postgres_datadir ]; then - # Get the default or highest version of PostgreSQL installed - PG_VERSION=$(psql -V | awk '{print $3}' | cut -d'.' -f1) - echo "Detected PostgreSQL version: $PG_VERSION" + sudo systemctl stop postgresql + sudo systemctl stop postgresql@$PG_VERSION-main + + if [ ! -d /mnt/hdd/app-data/postgresql ]; then + # there is no old data + + # symlink conf dir + sudo mkdir -p /mnt/hdd/app-data/postgresql-conf/postgresql + sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql-conf # fix ownership + sudo mv $postgres_confdir /etc/postgresql.bak # backup new empty dir + sudo rm -rf $postgres_confdir # not a symlink.. delete it silently + sudo ln -s /mnt/hdd/app-data/postgresql-conf/postgresql /etc/ # create symlink + + # symlink data dir + sudo mkdir -p /mnt/hdd/app-data/postgresql + sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql # fix ownership + sudo mv $postgres_datadir /var/lib/postgresql.bak # backup new empty dir + sudo rm -rf $postgres_datadir # not a symlink.. delete it silently + sudo ln -s /mnt/hdd/app-data/postgresql /var/lib/ # create symlink echo "# Create PostgreSQL data" sudo mkdir -p $postgres_datadir/$PG_VERSION/main sudo chown -R postgres:postgres $postgres_datadir - # create cluster - sudo pg_createcluster $PG_VERSION main --start - fi + echo "# Create cluster" + sudo pg_createcluster $PG_VERSION main + sudo pg_ctlcluster $PG_VERSION main start - fix_postgres=0 - if [ -L $postgres_datadir ]; then - if [ -e $postgres_datadir ]; then - echo "# Good link in $postgres_datadir" + elif [ -d /mnt/hdd/app-data/postgresql/$PG_VERSION/main ]; then + if [ -d /mnt/hdd/app-data/postgresql-conf ]; then + # symlink conf dir + sudo mkdir -p /mnt/hdd/app-data/postgresql-conf/postgresql + sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql-conf # fix ownership + sudo mv $postgres_confdir /etc/postgresql.bak # backup new empty dir + sudo rm -rf $postgres_confdir # not a symlink.. delete it silently + sudo ln -s /mnt/hdd/app-data/postgresql-conf/postgresql /etc/ # create symlink else - echo "# Broken link in $postgres_datadir" - fix_postgres=1 + # generate new cluster and use default config + echo "# Create PostgreSQL data" + sudo mkdir -p $postgres_datadir/$PG_VERSION/main + sudo chown -R postgres:postgres $postgres_datadir + sudo pg_createcluster $PG_VERSION main + sudo pg_ctlcluster $PG_VERSION main start + echo "Setting default password for postgres user" + # start cluster temporarily + sudo systemctl start postgresql + sudo pg_createcluster $PG_VERSION main + sudo pg_ctlcluster $PG_VERSION main start + echo "Setting default password for postgres user" + sudo -u postgres psql -c "ALTER USER postgres WITH PASSWORD 'postgres';" + sudo systemctl stop postgresql + sudo systemctl stop postgresql@$PG_VERSION-main + # move and symlink conf dir + sudo mkdir -p /mnt/hdd/app-data/postgresql-conf + sudo mv /etc/postgresql /mnt/hdd/app-data/postgresql-conf/ + sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql-conf + sudo ln -s /mnt/hdd/app-data/postgresql-conf/postgresql /etc/ # create symlink + sudo chown -R postgres:postgres $postgres_confdir fi - elif [ -e $postgres_datadir ]; then - echo "# Not a link in $postgres_datadir" - fix_postgres=1 - else - echo "# Missing Link in $postgres_datadir" - fix_postgres=1 + + # symlink data dir + sudo mkdir -p /mnt/hdd/app-data/postgresql + sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql # fix ownership + sudo mv $postgres_datadir /var/lib/postgresql.bak # backup new empty dir + sudo rm -rf $postgres_datadir # not a symlink.. delete it silently + sudo ln -s /mnt/hdd/app-data/postgresql /var/lib/ # create symlink + + sudo chown -R postgres:postgres $postgres_datadir + sudo systemctl start postgresql + sudo systemctl start postgresql@13-main + sudo pg_createcluster $PG_VERSION main + sudo pg_ctlcluster $PG_VERSION main start + + elif [ -d /mnt/hdd/app-data/postgresql/13/main ]; then + # if there is old data for pg 13 start and upgrade cluster + sudo apt install -y postgresql-13 || exit 1 + sudo systemctl stop postgresql + sudo systemctl stop postgresql@13-main + if [ -d /mnt/hdd/app-data/postgresql-conf ]; then + # symlink conf dir + sudo mkdir -p /mnt/hdd/app-data/postgresql-conf/postgresql + sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql-conf # fix ownership + sudo mv $postgres_confdir /etc/postgresql.bak # backup new empty dir + sudo rm -rf $postgres_confdir # not a symlink.. delete it silently + sudo ln -s /mnt/hdd/app-data/postgresql-conf/postgresql /etc/ # create symlink + else + # generate new cluster and use default config + echo "# Create PostgreSQL data" + sudo mkdir -p $postgres_datadir/13/main + sudo chown -R postgres:postgres $postgres_datadir + # start cluster temporarily + sudo systemctl start postgresql + sudo pg_createcluster 13 main + sudo pg_ctlcluster 13 main start + echo "Setting default password for postgres user" + sudo -u postgres psql -c "ALTER USER postgres WITH PASSWORD 'postgres';" + sudo systemctl stop postgresql + sudo systemctl stop postgresql@13-main + # move and symlink conf dir + sudo mkdir -p /mnt/hdd/app-data/postgresql-conf + sudo mv /etc/postgresql /mnt/hdd/app-data/postgresql-conf/ + sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql-conf + sudo ln -s /mnt/hdd/app-data/postgresql-conf/postgresql /etc/ # create symlink + sudo chown -R postgres:postgres $postgres_confdir + fi + + # symlink data dir + sudo mkdir -p /mnt/hdd/app-data/postgresql + sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql # fix ownership + sudo mv $postgres_datadir /var/lib/postgresql.bak # backup new empty dir + sudo rm -rf $postgres_datadir # not a symlink.. delete it silently + sudo ln -s /mnt/hdd/app-data/postgresql /var/lib/ # create symlink + + sudo chown -R postgres:postgres $postgres_datadir + sudo systemctl start postgresql + sudo systemctl start postgresql@13-main + sudo pg_createcluster 13 main + sudo pg_ctlcluster 13 main start + # /usr/bin/pg_upgradecluster [OPTIONS] [] + sudo pg_upgradecluster 13 main $postgres_datadir/$PG_VERSION/main || exit 1 + sudo systemctl disable --now postgresql@13-main + sudo apt remove -y postgresql-13 fi - if [ ${fix_postgres} = 1 ] || [ ! -d /mnt/hdd/app-data/postgresql ]; then - echo "# Move the PostgreSQL data to /mnt/hdd/app-data/postgresql" - sudo systemctl stop postgresql 2>/dev/null - sudo rsync -av $postgres_datadir /mnt/hdd/app-data - sudo mv $postgres_datadir /var/lib/postgresql.bak - sudo rm -rf $postgres_datadir # not a symlink.. delete it silently - sudo ln -s /mnt/hdd/app-data/postgresql /var/lib/ - fi - - # always fix ownership - sudo chown -R postgres:postgres /mnt/hdd/app-data/postgresql - - sudo systemctl enable postgresql - sudo systemctl start postgresql + # start cluster + sudo systemctl enable --now postgresql # check if PostgreSQL was installed if psql --version; then echo "# wait for the postgresql server to start" count=0 count_max=30 - while ! nc -zv 127.0.0.1 5432 2>/dev/null; do + while ! nc -zv '127.0.0.1' 5432 2>/dev/null; do count=$((count + 1)) echo "sleep $count/$count_max" sleep 1 @@ -101,13 +190,16 @@ fi # switch off if [ "$command" = "0" ] || [ "$command" = "off" ]; then echo "*** REMOVING POSTGRESQL ***" - sudo systemctl stop postgresql - sudo systemctl disable postgresql + sudo systemctl disable --now postgresql + sudo systemctl disable --now postgresql@$PG_VERSION-main + sudo systemctl disable --now postgresql@13-main sudo apt remove -y postgresql + if dpkg -l | grep -q "postgresql-13"; then + sudo apt remove -y postgresql-13 + fi echo "# remove symlink /var/lib/postgresql" sudo rm /var/lib/postgresql - # would delete all pg data: 'sudo pg_dropcluster $PG_VERSION main' - echo "# OK PostgreSQL is removed." + sudo rm /etc/postgresql exit 0 fi @@ -120,7 +212,6 @@ fi # https://www.postgresql.org/docs/current/backup-dump.html if [ "$command" = "backup" ] && [ "$db_name" != "" ]; then - echo "*** BACKUP POSTGRESQL $db_name ***" sudo -u postgres pg_dump $db_name >$backup_target/${backup_file}.sql || exit 1 # Delete old backups (keep last 3 backups) @@ -128,12 +219,10 @@ if [ "$command" = "backup" ] && [ "$db_name" != "" ]; then ls -tp $backup_target/*.sql | grep -v '/$' | tail -n +4 | tr '\n' '\0' | xargs -0 rm -- 2>/dev/null echo "OK - backup finished, file saved as $backup_target/${backup_file}.sql" exit 0 - fi # restore if [ "$command" = "restore" ] && [ "$db_name" != "" ] && [ "$db_user" != "" ] && [ "$db_user_pw" != "" ]; then - echo "*** RESTORE POSTGRESQL $db_name ***" # find recent backup if [ "$db_backupfile" != "" ]; then @@ -168,11 +257,9 @@ if [ "$command" = "restore" ] && [ "$db_name" != "" ] && [ "$db_user" != "" ] && echo "$backup_target/sql_import.log written" echo "OK - database $db_name restored from ${backup_file}" exit 0 - fi if [ "$command" = "info" ]; then - check=$(sudo -u postgres psql -c "show data_directory;" | grep data_directory) if [ "$check" = "" ]; then echo "show data_directory failed, PostgreSQL not installed?!" @@ -182,7 +269,6 @@ if [ "$command" = "info" ]; then sudo -u postgres psql -c "SELECT datname FROM pg_database;" fi exit 0 - fi echo "FAIL - Unknown Parameter $command" diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..8ce1710e8 --- /dev/null +++ b/test/README.md @@ -0,0 +1,12 @@ +# Running tests + +## Install bats +``` +sudo apt install bats +``` + +## Run tests manually +``` +bats ./bonus.postgresql-13.bats +bats ./bonus.postgresql-15.bats +``` diff --git a/test/bonus.postgresql-13.bats b/test/bonus.postgresql-13.bats new file mode 100644 index 000000000..7fa3d6f65 --- /dev/null +++ b/test/bonus.postgresql-13.bats @@ -0,0 +1,91 @@ +#!/usr/bin/env bats + +@test "Create PostgreSQL 13 cluster" { + postgres_datadir="/var/lib/postgresql" # default data dir + postgres_confdir="/etc/postgresql" # default conf dir + if [ ! -f /etc/apt/trusted.gpg.d/postgresql.gpg ]; then + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg + echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt-get update + fi + sudo apt-get install -y postgresql-13 + + # avoid failure in github action + sudo pg_createcluster 13 main || true + sudo pg_ctlcluster 13 main start + + sudo -u postgres psql -c "ALTER USER postgres WITH PASSWORD 'postgres';" +} + +@test "Create test database in 13" { + sudo -u postgres psql -c "CREATE DATABASE testdb13 TEMPLATE template0 LC_CTYPE 'C' LC_COLLATE 'C' ENCODING 'UTF8';" + sudo -u postgres psql -c "CREATE USER testuser13 WITH ENCRYPTED PASSWORD 'raspiblitz';" + sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE testdb13 TO testuser13;" + run pg_lsclusters + [ "$status" -eq 0 ] + echo "$output" + run sudo -u postgres psql -l + echo "$output" + echo "$output" | grep -q "testdb13" + [ "$?" -eq 0 ] + echo "$output" | grep -q "testuser13" + [ "$?" -eq 0 ] +} + +@test "Switch cluster 13 off and move" { + sudo apt-get remove -y postgresql-13 + run ../home.admin/config.scripts/bonus.postgresql.sh off + echo "$output" + [ "$status" -eq 0 ] + sudo mkdir -p /mnt/hdd/app-data/ + sudo mv /var/lib/postgresql /mnt/hdd/app-data/ + sudo rm -rf /mnt/hdd/app-data/postgresql/15 + run sudo ls /mnt/hdd/app-data/postgresql/13 + [ "$status" -eq 0 ] + sudo mv /mnt/hdd/app-data/postgresql /mnt/hdd/app-data/postgresql.bak + sudo pg_dropcluster 15 main --stop || true + run sudo pg_dropcluster 13 main --stop + [ "$status" -eq 0 ] +} + +@test "Recover cluster from 13 without config" { + sudo mv /mnt/hdd/app-data/postgresql.bak /mnt/hdd/app-data/postgresql + sudo rm -rf /etc/postgresql + sudo rm -rf /mnt/hdd/app-data/postgresql-conf.bak + # run the script + run ../home.admin/config.scripts/bonus.postgresql.sh on + echo "$output" + [ "$status" -eq 0 ] + run sudo -u postgres psql -l + echo "$output" + echo "$output" | grep -q "testdb13" + [ "$?" -eq 0 ] + echo "$output" | grep -q "testuser13" + [ "$?" -eq 0 ] +} + +@test "Create test database (2)" { + sudo -u postgres psql -c "CREATE DATABASE testdb TEMPLATE template0 LC_CTYPE 'C' LC_COLLATE 'C' ENCODING 'UTF8';" + sudo -u postgres psql -c "CREATE USER testuser WITH ENCRYPTED PASSWORD 'raspiblitz';" + sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE testdb TO testuser;" + run pg_lsclusters + [ "$status" -eq 0 ] + echo "$output" + run sudo -u postgres psql -l + echo "$output" + echo "$output" | grep -q "testdb" + [ "$?" -eq 0 ] + echo "$output" | grep -q "testuser" + [ "$?" -eq 0 ] +} + +@test "Final cleanup" { + run ../home.admin/config.scripts/bonus.postgresql.sh off + [ "$status" -eq 0 ] + run pg_lsclusters + [ "$status" -eq 0 ] + echo "$output" + sudo pg_dropcluster 15 main --stop || true + sudo pg_dropcluster 13 main --stop || true + sudo rm -rf /mnt/hdd/app-data/postgresql* +} diff --git a/test/bonus.postgresql-15.bats b/test/bonus.postgresql-15.bats new file mode 100644 index 000000000..a5b4f2140 --- /dev/null +++ b/test/bonus.postgresql-15.bats @@ -0,0 +1,109 @@ +#!/usr/bin/env bats + +@test "Start PostgreSQL cluster" { + # run the script + run ../home.admin/config.scripts/bonus.postgresql.sh on + echo "$output" + [ "$status" -eq 0 ] + # check if PostgreSQL cluster is running + run pg_lsclusters + [ "$status" -eq 0 ] + echo "$output" +} + +@test "Create test database" { + sudo -u postgres psql -c "CREATE DATABASE testdb TEMPLATE template0 LC_CTYPE 'C' LC_COLLATE 'C' ENCODING 'UTF8';" + sudo -u postgres psql -c "CREATE USER testuser WITH ENCRYPTED PASSWORD 'raspiblitz';" + sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE testdb TO testuser;" + # check if PostgreSQL cluster is running + run pg_lsclusters + [ "$status" -eq 0 ] + echo "$output" + run sudo -u postgres psql -l + echo "$output" + echo "$output" | grep -q "testdb" + [ "$?" -eq 0 ] + echo "$output" | grep -q "testuser" + [ "$?" -eq 0 ] +} + +@test "Switch cluster off and move" { + # run the script + run ../home.admin/config.scripts/bonus.postgresql.sh off + # check if PostgreSQL cluster is running + run pg_lsclusters + echo "$output" + [ "$status" -eq 0 ] + sudo mv /mnt/hdd/app-data/postgresql /mnt/hdd/app-data/postgresql.bak + sudo mv /mnt/hdd/app-data/postgresql-conf /mnt/hdd/app-data/postgresql-conf.bak + if echo "${output}" | grep "15 main"; then + run sudo pg_dropcluster 15 main --stop + [ "$status" -eq 0 ] + fi +} + +@test "Restore pg cluster" { + sudo mv /mnt/hdd/app-data/postgresql.bak /mnt/hdd/app-data/postgresql + sudo mv /mnt/hdd/app-data/postgresql-conf.bak /mnt/hdd/app-data/postgresql-conf + # run the script + run ../home.admin/config.scripts/bonus.postgresql.sh on + echo "$output" + [ "$status" -eq 0 ] + # check the database + run pg_lsclusters + echo "$output" + [ "$status" -eq 0 ] + echo "$output" + run sudo -u postgres psql -l + echo "$output" + echo "$output" | grep -q "testdb" + [ "$?" -eq 0 ] + echo "$output" | grep -q "testuser" + [ "$?" -eq 0 ] +} + +@test "Switch cluster off and move (2)" { + run ../home.admin/config.scripts/bonus.postgresql.sh off + echo "$output" + [ "$status" -eq 0 ] + run pg_lsclusters + echo "$output" + [ "$status" -eq 0 ] + sudo mv /mnt/hdd/app-data/postgresql /mnt/hdd/app-data/postgresql.bak + sudo mv /mnt/hdd/app-data/postgresql-conf /mnt/hdd/app-data/postgresql-conf.bak + if echo "${output}" | grep "15 main"; then + run sudo pg_dropcluster 15 main --stop + [ "$status" -eq 0 ] + fi + +} + +@test "Restore cluster without config dir" { + sudo mv /mnt/hdd/app-data/postgresql.bak /mnt/hdd/app-data/postgresql + sudo rm -rf /etc/postgresql + sudo rm -rf /mnt/hdd/app-data/postgresql-conf.bak + run ../home.admin/config.scripts/bonus.postgresql.sh on + echo "$output" + [ "$status" -eq 0 ] + run pg_lsclusters + [ "$status" -eq 0 ] + echo "$output" + run sudo -u postgres psql -l + echo "$output" + echo "$output" | grep -q "testdb" + [ "$?" -eq 0 ] + echo "$output" | grep -q "testuser" + [ "$?" -eq 0 ] +} + +@test "Cleanup" { + run ../home.admin/config.scripts/bonus.postgresql.sh off + echo "$output" + [ "$status" -eq 0 ] + run pg_lsclusters + [ "$status" -eq 0 ] + echo "$output" + sudo pg_dropcluster 15 main --stop || true + sudo pg_dropcluster 13 main --stop || true + sudo rm -rf /mnt/hdd/app-data/postgresql* +} diff --git a/typos.toml b/typos.toml index b26145103..3a7032045 100644 --- a/typos.toml +++ b/typos.toml @@ -6,10 +6,11 @@ [files] # skip these files -extend-exclude = ["*.nodes","*.torrent","*.sh","*.py","*.proto","*.json"] +extend-exclude = ["*.nodes","*.torrent","*.sh","*.py","*.proto","*.json","*.conf","*.toml"] [default.extend-words] # don't correct these false positives tpos = "tpos" ba = "ba" tne = "tne" +HD = "HD"