diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 8e72cf624..90006d2a8 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -9,9 +9,9 @@ jobs: postgres: image: postgres:latest env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres + POSTGRES_USER: lnbits + POSTGRES_PASSWORD: lnbits + POSTGRES_DB: migration ports: # maps tcp port 5432 on service container to the host - 5432:5432 @@ -36,11 +36,4 @@ jobs: sudo apt install unzip - name: Run migrations run: | - rm -rf ./data - mkdir -p ./data - export LNBITS_DATA_FOLDER="./data" - unzip tests/data/mock_data.zip -d ./data - timeout 5s poetry run lnbits --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi - export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres" - timeout 5s poetry run lnbits --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi - poetry run python tools/conv.py + make test-migration diff --git a/Makefile b/Makefile index d91d0421d..20fe65ce8 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: format check requirements.txt format: prettier isort black -check: mypy checkprettier checkisort checkblack +check: mypy checkprettier checkisort checkblack prettier: $(shell find lnbits -name "*.js" -name ".html") ./node_modules/.bin/prettier --write lnbits/static/js/*.js lnbits/core/static/js/*.js lnbits/extensions/*/templates/*/*.html ./lnbits/core/templates/core/*.html lnbits/templates/*.html lnbits/extensions/*/static/js/*.js lnbits/extensions/*/static/components/*/*.js lnbits/extensions/*/static/components/*/*.html @@ -46,5 +46,24 @@ test-venv: PYTHONUNBUFFERED=1 \ ./venv/bin/pytest --durations=1 -s --cov=lnbits --cov-report=xml tests +test-migration: + rm -rf ./migration-data + mkdir -p ./migration-data + unzip tests/data/mock_data.zip -d ./migration-data + HOST=0.0.0.0 \ + PORT=5002 \ + LNBITS_DATA_FOLDER="./migration-data" \ + timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + HOST=0.0.0.0 \ + PORT=5002 \ + LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \ + timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + LNBITS_DATA_FOLDER="./migration-data" \ + LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \ + poetry run python tools/conv.py + +migration: + poetry run python tools/conv.py + bak: # LNBITS_DATABASE_URL=postgres://postgres:postgres@0.0.0.0:5432/postgres diff --git a/docs/devs/extensions.md b/docs/devs/extensions.md index 0ceb9cb3f..cd81a0210 100644 --- a/docs/devs/extensions.md +++ b/docs/devs/extensions.md @@ -48,4 +48,25 @@ LNbits currently supports SQLite and PostgreSQL databases. There is a migration ### Adding mock data to `mock_data.zip` -`mock_data.zip` contains a few lines of sample SQLite data and is used in automated GitHub test to see whether your migration in `conv.py` works. Run your extension and save a few lines of data into a SQLite `your_extension.sqlite3` file. Unzip `tests/data/mock_data.zip`, add `your_extension.sqlite3` and zip it again. Add the updated `mock_data.zip` to your PR. \ No newline at end of file +`mock_data.zip` contains a few lines of sample SQLite data and is used in automated GitHub test to see whether your migration in `conv.py` works. Run your extension and save a few lines of data into a SQLite `your_extension.sqlite3` file. Unzip `tests/data/mock_data.zip`, add `your_extension.sqlite3`, updated `database.sqlite3` and zip it again. Add the updated `mock_data.zip` to your PR. + +### running migration locally +you will need a running postgres database + +#### create lnbits user for migration database +```console +sudo su - postgres -c "psql -c 'CREATE ROLE lnbits LOGIN PASSWORD 'lnbits';'" +``` +#### create migration database +```console +sudo su - postgres -c "psql -c 'CREATE DATABASE migration;'" +``` +#### run the migration +```console +make test-migration +``` +sudo su - postgres -c "psql -c 'CREATE ROLE lnbits LOGIN PASSWORD 'lnbits';'" +#### clean migration database afterwards, fails if you try again +```console +sudo su - postgres -c "psql -c 'DROP DATABASE IF EXISTS migration;'" +``` diff --git a/docs/guide/installation.md b/docs/guide/installation.md index cc5b96a98..6de893adb 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -170,8 +170,9 @@ LNBITS_DATABASE_URL="postgres://postgres:postgres@localhost/lnbits" # START LNbits # STOP LNbits -# on the LNBits folder, locate and edit 'tools/conv.py' with the relevant credentials -python3 tools/conv.py +poetry run python tools/conv.py +# or +make migration ``` Hopefully, everything works and get migrated... Launch LNbits again and check if everything is working properly. diff --git a/tests/data/mock_data.zip b/tests/data/mock_data.zip index 3a7938917..8704eb6d4 100644 Binary files a/tests/data/mock_data.zip and b/tests/data/mock_data.zip differ diff --git a/tools/conv.py b/tools/conv.py index 10e8c9cdd..5084660ff 100644 --- a/tools/conv.py +++ b/tools/conv.py @@ -19,16 +19,12 @@ env.read_env() # Change these values as needed -sqfolder = "data/" +sqfolder = env.str("LNBITS_DATA_FOLDER", default=None) LNBITS_DATABASE_URL = env.str("LNBITS_DATABASE_URL", default=None) if LNBITS_DATABASE_URL is None: - pgdb = "lnbits" - pguser = "lnbits" - pgpswd = "postgres" - pghost = "localhost" - pgport = "5432" - pgschema = "" + print("missing LNBITS_DATABASE_URL") + sys.exit(1) else: # parse postgres://lnbits:postgres@localhost:5432/lnbits pgdb = LNBITS_DATABASE_URL.split("/")[-1] @@ -129,7 +125,7 @@ def migrate_db(file: str, schema: str, exclude_tables: List[str] = []): sq = get_sqlite_cursor(file) tables = sq.execute( """ - SELECT name FROM sqlite_master + SELECT name FROM sqlite_master WHERE type='table' AND name not like 'sqlite?_%' escape '?' """ ).fetchall()