diff --git a/Dockerfile b/Dockerfile index a5ef573f0..feb95af15 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,7 +68,8 @@ RUN apt-get update -qq && \ libev-dev \ libevent-dev \ qemu-user-static \ - wget + wget \ + jq RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz \ && tar xvf zlib-1.2.13.tar.gz \ diff --git a/contrib/docker/Dockerfile.builder b/contrib/docker/Dockerfile.builder index 2c717a907..6b5e7d7ee 100644 --- a/contrib/docker/Dockerfile.builder +++ b/contrib/docker/Dockerfile.builder @@ -34,6 +34,7 @@ RUN apt-get -qq update && \ libxml2-utils \ lowdown \ wget \ + jq \ gettext \ xsltproc \ zlib1g-dev && \ diff --git a/contrib/docker/Dockerfile.builder.fedora b/contrib/docker/Dockerfile.builder.fedora index 22cb8c22b..2c3b1f553 100644 --- a/contrib/docker/Dockerfile.builder.fedora +++ b/contrib/docker/Dockerfile.builder.fedora @@ -19,6 +19,7 @@ RUN dnf update -y && \ net-tools \ valgrind \ wget \ + jq \ xz \ zlib-devel && \ dnf clean all diff --git a/contrib/docker/Dockerfile.tester b/contrib/docker/Dockerfile.tester index 14b61e7b1..d79c3efe5 100644 --- a/contrib/docker/Dockerfile.tester +++ b/contrib/docker/Dockerfile.tester @@ -50,6 +50,7 @@ RUN apt-get -qq update && \ unzip \ valgrind \ wget \ + jq \ xsltproc \ zlib1g-dev && \ rm -rf /var/lib/apt/lists/* diff --git a/contrib/docker/Dockerfile.ubuntu b/contrib/docker/Dockerfile.ubuntu index 026c549a6..401a60d1c 100644 --- a/contrib/docker/Dockerfile.ubuntu +++ b/contrib/docker/Dockerfile.ubuntu @@ -24,7 +24,7 @@ RUN locale-gen en_US.UTF-8 && dpkg-reconfigure --frontend noninteractive tzdata RUN apt-get -qq update && \ apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ - build-essential libssl-dev libffi-dev \ + jq build-essential libssl-dev libffi-dev \ python3-dev cargo python3-pip COPY . . diff --git a/contrib/docker/scripts/setup.sh b/contrib/docker/scripts/setup.sh index 9af49030e..94c59361d 100755 --- a/contrib/docker/scripts/setup.sh +++ b/contrib/docker/scripts/setup.sh @@ -47,6 +47,7 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ unzip \ valgrind \ wget \ + jq \ xsltproc \ zlib1g-dev diff --git a/contrib/reprobuild/Dockerfile.focal b/contrib/reprobuild/Dockerfile.focal index a02e552ce..cbc0f45a2 100644 --- a/contrib/reprobuild/Dockerfile.focal +++ b/contrib/reprobuild/Dockerfile.focal @@ -24,6 +24,7 @@ RUN apt-get update \ sudo \ unzip \ wget \ + jq \ zip # install Python3.8 (more reproducible than relying on python3-setuptools) diff --git a/contrib/reprobuild/Dockerfile.jammy b/contrib/reprobuild/Dockerfile.jammy index 6c97db546..632799414 100644 --- a/contrib/reprobuild/Dockerfile.jammy +++ b/contrib/reprobuild/Dockerfile.jammy @@ -25,6 +25,7 @@ RUN apt-get update \ sudo \ unzip \ wget \ + jq \ zip # Install Python3.10 (more reproducible than relying on python3-setuptools) diff --git a/doc/contribute-to-core-lightning/contributor-workflow.md b/doc/contribute-to-core-lightning/contributor-workflow.md index 489b2c99f..30a610b83 100644 --- a/doc/contribute-to-core-lightning/contributor-workflow.md +++ b/doc/contribute-to-core-lightning/contributor-workflow.md @@ -12,7 +12,7 @@ Install the following dependencies for best results: ```shell sudo apt update -sudo apt install valgrind cppcheck shellcheck libsecp256k1-dev libpq-dev +sudo apt install jq valgrind cppcheck shellcheck libsecp256k1-dev libpq-dev ``` Re-run `configure` and build using `make`: diff --git a/doc/getting-started/getting-started/installation.md b/doc/getting-started/getting-started/installation.md index 9297f806e..7477785f0 100644 --- a/doc/getting-started/getting-started/installation.md +++ b/doc/getting-started/getting-started/installation.md @@ -82,7 +82,7 @@ Get dependencies: ```shell sudo apt-get update sudo apt-get install -y \ - autoconf automake build-essential git libtool libsqlite3-dev \ + jq autoconf automake build-essential git libtool libsqlite3-dev \ python3 python3-pip net-tools zlib1g-dev libsodium-dev gettext pip3 install --upgrade pip pip3 install --user poetry @@ -115,7 +115,7 @@ For development or running tests, get additional dependencies: ```shell sudo apt-get install -y valgrind libpq-dev shellcheck cppcheck \ - libsecp256k1-dev jq lowdown + libsecp256k1-dev lowdown ``` If you can't install `lowdown`, a version will be built in-tree. @@ -193,6 +193,7 @@ sudo dnf update -y && \ net-tools \ valgrind \ wget \ + jq \ zlib-devel \ libsodium-devel && \ sudo dnf clean all diff --git a/doc/lightning-sql.7.md b/doc/lightning-sql.7.md index 4932bee18..c1dc06de4 100644 --- a/doc/lightning-sql.7.md +++ b/doc/lightning-sql.7.md @@ -249,6 +249,7 @@ The following tables are currently supported: - `peerchannels` indexed by `peer_id` (see lightning-listpeerchannels(7)) - `peer_id` (type `pubkey`, sqltype `BLOB`) - `peer_connected` (type `boolean`, sqltype `INTEGER`) + - `reestablished` (type `boolean`, sqltype `INTEGER`) - `state` (type `string`, sqltype `TEXT`) - `scratch_txid` (type `txid`, sqltype `BLOB`) - related table `peerchannels_channel_type_bits`, from JSON object `channel_type` @@ -357,7 +358,6 @@ The following tables are currently supported: - `local_trimmed` (type `boolean`, sqltype `INTEGER`) - `status` (type `string`, sqltype `TEXT`) - `state` (type `string`, sqltype `TEXT`) - - `reestablished` (type `boolean`, sqltype `INTEGER`) - `close_to_addr` (type `string`, sqltype `TEXT`) - `last_tx_fee_msat` (type `msat`, sqltype `INTEGER`) - `direction` (type `u32`, sqltype `INTEGER`) @@ -389,6 +389,7 @@ The following tables are currently supported: - `bolt11` (type `string`, sqltype `TEXT`) - `description` (type `string`, sqltype `TEXT`) - `bolt12` (type `string`, sqltype `TEXT`) + - `completed_at` (type `u64`, sqltype `INTEGER`) - `payment_preimage` (type `secret`, sqltype `BLOB`) - `erroronion` (type `hex`, sqltype `BLOB`) @@ -540,4 +541,4 @@ RESOURCES --------- Main web site: -[comment]: # ( SHA256STAMP:24eeff191907cb804299716293ef0733ef25cf8e74eb1808749aebbfd9e9fa9f) +[comment]: # ( SHA256STAMP:5c2e41cab6704299c65a8537761f3864e6b3327c1bf9163b00afe723d1f84eb6) diff --git a/plugins/Makefile b/plugins/Makefile index 0da2e5738..7467610db 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -225,16 +225,27 @@ plugins/recover: common/gossmap.o common/fp16.o $(PLUGIN_RECOVER_OBJS) $(PLUGIN # This covers all the low-level list RPCs which return simple arrays SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listpeerchannels listclosedchannels listtransactions listsendpays bkpr-listaccountevents bkpr-listincome -SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/$l.schema.json) -# We squeeze: -# descriptions (we don't need) -# fields with no members (we don't need) -# whitespace -# We can't simply *remove* fields, since the extra comma left over can -# make invalid JSON. Grr! -# But these simple removals drop us from 100k to 29k. +SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/lightning-$l.json) + +# RES_JSON is generated by: +# 1: deleting description array (used for field details) but keeping description keys +# 2: deleting additionalProperties, required, enum, maxLength, minLength, pre_return_value_notes, post_return_value_notes, anyOf, oneOf, hidden, untyped, default, comment, added, maximum, minimum, pattern, format +# (We only need [field].type, [field].name, [field].deprecated, [field].allOf, [field].items.allOf and allOf from response.properties) +# 3: deleting empty objects from .response.properties +# But these simple removals drop us from 100k to 18k. plugins/sql-schema_gen.h: plugins/Makefile $(SQL_LISTRPCS_SCHEMAS) - @$(call VERBOSE,GEN $@, (SEP=""; echo '"{'; for f in $(SQL_LISTRPCS); do echo "$$SEP\\\"$$f\\\":"; sed -e s/\"description\":\ *\".\*\"/\"\":\"\"/ -e s/\".*\":\ *{}/\"\":{}/ -e s/\"/\\\\\"/g < doc/schemas/$$f.schema.json; SEP=","; done; echo '}"') | tr -d ' \n' > $@) + @$(call VERBOSE,GEN $@, \ + ( \ + SEP=""; \ + echo '"{'; \ + for f in $(SQL_LISTRPCS); do \ + RES_JSON=$$(echo "$$(jq 'walk(if type == "object" then if has("description") and (.description | type) == "array" then del(.description) else . end else . end ) | walk(if type == "object" then del(.additionalProperties, .required, .enum, .maxLength, .minLength, .pre_return_value_notes, .post_return_value_notes, .anyOf, .oneOf, .hidden, .untyped, .default, .comment, .added, .maximum, .minimum, .pattern, .format) else . end) | walk(if type == "object" then with_entries(select(.value != {})) else . end) | .response.properties' < doc/schemas/lightning-$$f.json)" | sed 's/"/\\"/g'); \ + echo "$$SEP\\\"$$f\\\":{\\\"properties\\\": $$RES_JSON}"; \ + SEP=","; \ + done; \ + echo '}"' \ + ) | tr -d ' \n' > $@ \ + ) plugins/sql.o: plugins/sql-schema_gen.h plugins/sql: $(PLUGIN_SQL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossip_store.o gossipd/gossip_store_wiregen.o