script: Updated sql-schema_gen and documented it's jq dependency.

Dockerfile

@@ -68,7 +68,8 @@ RUN apt-get update -qq && \
         libev-dev \
         libevent-dev \
         qemu-user-static \
-        wget
+        wget \
+        jq
 
 RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz \
     && tar xvf zlib-1.2.13.tar.gz \

contrib/docker/Dockerfile.builder

@@ -34,6 +34,7 @@ RUN apt-get -qq update && \
 	libxml2-utils \
 	lowdown \
 	wget \
+	jq \
 	gettext \
 	xsltproc \
 	zlib1g-dev && \

contrib/docker/Dockerfile.builder.fedora

@@ -19,6 +19,7 @@ RUN dnf update -y && \
 		net-tools \
 		valgrind \
 		wget \
+		jq \
 		xz \
 		zlib-devel && \
 	dnf clean all

contrib/docker/Dockerfile.tester

@@ -50,6 +50,7 @@ RUN apt-get -qq update && \
 	unzip \
 	valgrind \
 	wget \
+	jq \
 	xsltproc \
 	zlib1g-dev && \
 	rm -rf /var/lib/apt/lists/*

contrib/docker/Dockerfile.ubuntu

@@ -24,7 +24,7 @@ RUN locale-gen en_US.UTF-8 && dpkg-reconfigure --frontend noninteractive tzdata
 
 RUN apt-get -qq update && \
     apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
-    build-essential libssl-dev libffi-dev \
+    jq build-essential libssl-dev libffi-dev \
     python3-dev cargo python3-pip
 
 COPY . .

contrib/docker/scripts/setup.sh

@@ -47,6 +47,7 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      unzip \
      valgrind \
      wget \
+     jq \
      xsltproc \
      zlib1g-dev
 

contrib/reprobuild/Dockerfile.focal

@@ -24,6 +24,7 @@ RUN apt-get update \
 	sudo \
 	unzip \
 	wget \
+    jq \
 	zip
 
 # install Python3.8 (more reproducible than relying on python3-setuptools)

contrib/reprobuild/Dockerfile.jammy

@@ -25,6 +25,7 @@ RUN apt-get update \
 	sudo \
 	unzip \
 	wget \
+    jq \
 	zip
 
 # Install Python3.10 (more reproducible than relying on python3-setuptools)

doc/contribute-to-core-lightning/contributor-workflow.md

@@ -12,7 +12,7 @@ Install the following dependencies for best results:
 
 ```shell
 sudo apt update
-sudo apt install valgrind cppcheck shellcheck libsecp256k1-dev libpq-dev
+sudo apt install jq valgrind cppcheck shellcheck libsecp256k1-dev libpq-dev
 ```
 
 Re-run `configure` and build using `make`:

doc/getting-started/getting-started/installation.md

@@ -82,7 +82,7 @@ Get dependencies:
 ```shell
 sudo apt-get update
 sudo apt-get install -y \
-  autoconf automake build-essential git libtool libsqlite3-dev \
+  jq autoconf automake build-essential git libtool libsqlite3-dev \
   python3 python3-pip net-tools zlib1g-dev libsodium-dev gettext
 pip3 install --upgrade pip
 pip3 install --user poetry
@@ -115,7 +115,7 @@ For development or running tests, get additional dependencies:
 
 ```shell
 sudo apt-get install -y valgrind libpq-dev shellcheck cppcheck \
-  libsecp256k1-dev jq lowdown
+  libsecp256k1-dev lowdown
 ```
 
 If you can't install `lowdown`, a version will be built in-tree.
@@ -193,6 +193,7 @@ sudo dnf update -y && \
                 net-tools \
                 valgrind \
                 wget \
+                jq \
                 zlib-devel \
 				libsodium-devel && \
         sudo dnf clean all

doc/lightning-sql.7.md

@@ -249,6 +249,7 @@ The following tables are currently supported:
 - `peerchannels` indexed by `peer_id` (see lightning-listpeerchannels(7))
   - `peer_id` (type `pubkey`, sqltype `BLOB`)
   - `peer_connected` (type `boolean`, sqltype `INTEGER`)
+  - `reestablished` (type `boolean`, sqltype `INTEGER`)
   - `state` (type `string`, sqltype `TEXT`)
   - `scratch_txid` (type `txid`, sqltype `BLOB`)
   - related table `peerchannels_channel_type_bits`, from JSON object `channel_type`
@@ -357,7 +358,6 @@ The following tables are currently supported:
     - `local_trimmed` (type `boolean`, sqltype `INTEGER`)
     - `status` (type `string`, sqltype `TEXT`)
     - `state` (type `string`, sqltype `TEXT`)
-  - `reestablished` (type `boolean`, sqltype `INTEGER`)
   - `close_to_addr` (type `string`, sqltype `TEXT`)
   - `last_tx_fee_msat` (type `msat`, sqltype `INTEGER`)
   - `direction` (type `u32`, sqltype `INTEGER`)
@@ -389,6 +389,7 @@ The following tables are currently supported:
   - `bolt11` (type `string`, sqltype `TEXT`)
   - `description` (type `string`, sqltype `TEXT`)
   - `bolt12` (type `string`, sqltype `TEXT`)
+  - `completed_at` (type `u64`, sqltype `INTEGER`)
   - `payment_preimage` (type `secret`, sqltype `BLOB`)
   - `erroronion` (type `hex`, sqltype `BLOB`)
 
@@ -540,4 +541,4 @@ RESOURCES
 ---------
 
 Main web site: <https://github.com/ElementsProject/lightning>
-[comment]: # ( SHA256STAMP:24eeff191907cb804299716293ef0733ef25cf8e74eb1808749aebbfd9e9fa9f)
+[comment]: # ( SHA256STAMP:5c2e41cab6704299c65a8537761f3864e6b3327c1bf9163b00afe723d1f84eb6)

plugins/Makefile

@@ -225,16 +225,27 @@ plugins/recover:  common/gossmap.o common/fp16.o $(PLUGIN_RECOVER_OBJS) $(PLUGIN
 
 # This covers all the low-level list RPCs which return simple arrays
 SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listpeerchannels listclosedchannels listtransactions listsendpays bkpr-listaccountevents bkpr-listincome
-SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/$l.schema.json)
-# We squeeze:
-#  descriptions (we don't need)
-#  fields with no members (we don't need)
-#  whitespace
-# We can't simply *remove* fields, since the extra comma left over can
-# make invalid JSON. Grr!
-# But these simple removals drop us from 100k to 29k.
+SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/lightning-$l.json)
+
+# RES_JSON is generated by:
+# 1: deleting description array (used for field details) but keeping description keys
+# 2: deleting additionalProperties, required, enum, maxLength, minLength, pre_return_value_notes, post_return_value_notes, anyOf, oneOf, hidden, untyped, default, comment, added, maximum, minimum, pattern, format
+# (We only need [field].type, [field].name, [field].deprecated, [field].allOf, [field].items.allOf and allOf from response.properties)
+# 3: deleting empty objects from .response.properties
+# But these simple removals drop us from 100k to 18k.
 plugins/sql-schema_gen.h: plugins/Makefile $(SQL_LISTRPCS_SCHEMAS)
-	@$(call VERBOSE,GEN $@, (SEP=""; echo '"{'; for f in $(SQL_LISTRPCS); do echo "$$SEP\\\"$$f\\\":"; sed -e s/\"description\":\ *\".\*\"/\"\":\"\"/ -e s/\".*\":\ *{}/\"\":{}/ -e s/\"/\\\\\"/g < doc/schemas/$$f.schema.json; SEP=","; done; echo '}"') | tr -d ' \n' > $@)
+	@$(call VERBOSE,GEN $@, \
+		( \
+			SEP=""; \
+			echo '"{'; \
+			for f in $(SQL_LISTRPCS); do \
+				RES_JSON=$$(echo "$$(jq 'walk(if type == "object" then if has("description") and (.description | type) == "array" then del(.description) else . end else . end ) | walk(if type == "object" then del(.additionalProperties, .required, .enum, .maxLength, .minLength, .pre_return_value_notes, .post_return_value_notes, .anyOf, .oneOf, .hidden, .untyped, .default, .comment, .added, .maximum, .minimum, .pattern, .format) else . end) | walk(if type == "object" then with_entries(select(.value != {})) else . end) | .response.properties' < doc/schemas/lightning-$$f.json)" | sed 's/"/\\"/g'); \
+				echo "$$SEP\\\"$$f\\\":{\\\"properties\\\": $$RES_JSON}"; \
+				SEP=","; \
+			done; \
+			echo '}"' \
+		) | tr -d ' \n' > $@ \
+	)
 
 plugins/sql.o: plugins/sql-schema_gen.h
 plugins/sql: $(PLUGIN_SQL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossip_store.o gossipd/gossip_store_wiregen.o