renepay: add const probability cost

The probability for a success forward of x amount on a channel is now
computed as:

	P_success(x) = b * P_rene(x)

where P_rene(x) is the probability distribution proposed by Rene
Pickhardt with a uniform distribution on the known liquidity of the
channel and 'b' is a new parameter we add in this PR, by the name of
"base probability". The "base probability" represents the probability
for a channel in the network choosen at random to be able to forward at
least 1msat, ie. of being alive, non-depleted and with at least one HTLC
slot. We don't know the value of 'b', but for the moment we assume that
it is 0.98 and use that as default.

As a consequence the probability cost becomes non-linear and non-convex
because of the additional constant term:

	Cost(x) = - log P_success(x)
	        = - log b  -  - log P_rene(x)
		= - log b  +  Cost_rene(x)

We currently don't handle well base fees and neither this additional
"base probability" but we can as a first approximation linearize the
cost function in this way:

	Cost_linear(x) = (- log b)*x  +  Cost_rene_linear(x)

Changelog-Added: renepay: Add a dev parameter to renepay that represents
a constant probability of availability for all channels in the network.

Signed-off-by: Lagrang3 <lagrang3@protonmail.com>

plugins/renepay/flow.c

@@ -179,6 +179,15 @@ bool flowset_delivers(struct amount_msat *delivers, struct flow **flows)
 	return true;
 }
 
+/* FIXME: pass a pointer to const here */
+size_t flowset_size(struct flow **flows)
+{
+	size_t size = 0;
+	for (size_t i = 0; i < tal_count(flows); i++)
+		size += tal_count(flows[i]->path);
+	return size;
+}
+
 /* Checks if the flows satisfy the liquidity bounds imposed by the known maximum
  * liquidity and pending HTLCs.
  *

plugins/renepay/flow.h

@@ -55,6 +55,9 @@ bool flowset_fee(struct amount_msat *fee, struct flow **flows);
 
 bool flowset_delivers(struct amount_msat *delivers, struct flow **flows);
 
+/* how many channels are being used */
+size_t flowset_size(struct flow **flows);
+
 static inline struct amount_msat flow_delivers(const struct flow *flow)
 {
 	return flow->amount;

plugins/renepay/main.c

@@ -184,6 +184,11 @@ static struct command_result *json_pay(struct command *cmd, const char *buf,
 	u64 *riskfactor_millionths; // delay to proportional proportional fee
 	u64 *min_prob_success_millionths; // target probability
 
+	/* base probability of success, probability for a randomly picked
+	 * channel to be able to forward a payment request of amount greater
+	 * than zero. */
+	u64 *base_prob_success_millionths;
+
 	if (!param(cmd, buf, params,
 		   p_req("invstring", param_invstring, &invstr),
 		   p_opt("amount_msat", param_msat, &msat),
@@ -217,10 +222,19 @@ static struct command_result *json_pay(struct command *cmd, const char *buf,
 			     &riskfactor_millionths, 1), // default is 1e-6
 		   p_opt_dev("dev_min_prob_success", param_millionths,
 			     &min_prob_success_millionths,
-			     900000), // default is 0.9
+			     800000), // default is 0.8
+		   p_opt_dev("dev_base_prob_success", param_millionths,
+			     &base_prob_success_millionths,
+			     980000), // default is 0.98
 		   NULL))
 		return command_param_failed();
 
+	if (*base_prob_success_millionths == 0 ||
+	    *base_prob_success_millionths > 1000000)
+		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+				    "Base probability should be a number "
+				    "greater than zero and no greater than 1.");
+
 	/* === Parse invoice === */
 
 	// FIXME: add support for bolt12 invoices
@@ -315,6 +329,7 @@ static struct command_result *json_pay(struct command *cmd, const char *buf,
 			*prob_cost_factor_millionths,
 			*riskfactor_millionths,
 			*min_prob_success_millionths,
+			*base_prob_success_millionths,
 			use_shadow,
 			cast_const2(const struct route_exclusion**, exclusions));
 
@@ -352,6 +367,7 @@ static struct command_result *json_pay(struct command *cmd, const char *buf,
 				    *prob_cost_factor_millionths,
 				    *riskfactor_millionths,
 				    *min_prob_success_millionths,
+				    *base_prob_success_millionths,
 				    use_shadow,
 				    cast_const2(const struct route_exclusion**, exclusions)))
 			return command_fail(

plugins/renepay/mcf.c

@@ -331,6 +331,7 @@ struct pay_parameters {
 
 	struct amount_msat max_fee;
 	double min_probability;
+	double base_probability;
 	double delay_feefactor;
 	double base_fee_penalty;
 	u32 prob_cost_factor;
@@ -501,15 +502,19 @@ static bool linearize_channel(const struct pay_parameters *params,
 
 	capacity[0]=a;
 	cost[0]=0;
+	assert(params->base_probability > 5e-7);
+	assert(params->base_probability <= 1.0);
+	const double base_prob_factor = -log(params->base_probability);
+
 	for(size_t i=1;i<CHANNEL_PARTS;++i)
 	{
 		capacity[i] = MIN(params->cap_fraction[i]*(b-a), cap_on_capacity);
 		cap_on_capacity -= capacity[i];
 		assert(cap_on_capacity>=0);
 
-		cost[i] = params->cost_fraction[i]
+		cost[i] = (params->cost_fraction[i]*1.0/(b-a) + base_prob_factor)
 		          *params->amount.millisatoshis /* Raw: linearize_channel */
-		          *params->prob_cost_factor*1.0/(b-a);
+		          *params->prob_cost_factor;
 	}
 	return true;
 }
@@ -1264,6 +1269,7 @@ get_flow_paths(const tal_t *ctx, const struct gossmap *gossmap,
 	       // how many msats in excess we paid for not having msat accuracy
 	       // in the MCF solver
 	       struct amount_msat excess,
+	       const double base_probability,
 
 	       // error message
 	       char **fail)
@@ -1437,8 +1443,10 @@ get_flow_paths(const tal_t *ctx, const struct gossmap *gossmap,
 			excess = amount_msat(0);
 			fp->amount = delivered;
 
+
 			fp->success_prob =
-			    flow_probability(fp, gossmap, chan_extra_map);
+			    flow_probability(fp, gossmap, chan_extra_map)
+			    * pow(base_probability, tal_count(fp->path));
 			if (fp->success_prob < 0) {
 				if (fail)
 					*fail =
@@ -1603,6 +1611,7 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 		      struct chan_extra_map *chan_extra_map,
 		      const bitmap *disabled, struct amount_msat amount,
 		      struct amount_msat max_fee, double min_probability,
+		      double base_probability,
 		      double delay_feefactor, double base_fee_penalty,
 		      u32 prob_cost_factor, char **fail)
 {
@@ -1647,6 +1656,7 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 	params->delay_feefactor = delay_feefactor;
 	params->base_fee_penalty = base_fee_penalty;
 	params->prob_cost_factor = prob_cost_factor;
+	params->base_probability = base_probability;
 
 	// build the uncertainty network with linearization and residual arcs
 	struct linear_network *linear_network= init_linear_network(this_ctx, params, &errmsg);
@@ -1709,7 +1719,8 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 	// first flow found
 	best_flow_paths = get_flow_paths(
 	    this_ctx, params->gossmap, params->disabled, params->chan_extra_map,
-	    linear_network, residual_network, excess, &errmsg);
+	    linear_network, residual_network, excess, params->base_probability,
+	    &errmsg);
 	if (!best_flow_paths) {
 		if (fail)
 		*fail =
@@ -1720,7 +1731,8 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 
 	best_prob_success =
 	    flowset_probability(this_ctx, best_flow_paths, params->gossmap,
-				params->chan_extra_map, &errmsg);
+				params->chan_extra_map, &errmsg)
+	    * pow(params->base_probability, flowset_size(best_flow_paths));
 	if (best_prob_success < 0) {
 		if (fail)
 		*fail =
@@ -1764,7 +1776,8 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 		flow_paths =
 		    get_flow_paths(this_ctx, params->gossmap, params->disabled,
 				   params->chan_extra_map, linear_network,
-				   residual_network, excess, &errmsg);
+				   residual_network, excess, params->base_probability,
+				   &errmsg);
 		if(!flow_paths)
 		{
 			// get_flow_paths doesn't fail unless there is a bug.
@@ -1776,7 +1789,8 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 
 		double prob_success =
 		    flowset_probability(this_ctx, flow_paths, params->gossmap,
-					params->chan_extra_map, &errmsg);
+					params->chan_extra_map, &errmsg)
+		    * pow(params->base_probability, flowset_size(flow_paths));
 		if (prob_success < 0) {
 			// flowset_probability doesn't fail unless there is a bug.
 			if (fail)

plugins/renepay/mcf.h

@@ -61,6 +61,7 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 		      struct chan_extra_map *chan_extra_map,
 		      const bitmap *disabled, struct amount_msat amount,
 		      struct amount_msat max_fee, double min_probability,
+		      double base_probability,
 		      double delay_feefactor, double base_fee_penalty,
 		      u32 prob_cost_factor, char **fail);
 #endif /* LIGHTNING_PLUGINS_RENEPAY_MCF_H */

plugins/renepay/payment.c

@@ -32,6 +32,7 @@ struct payment *payment_new(
 	    u64 prob_cost_factor_millionths,
 	    u64 riskfactor_millionths,
 	    u64 min_prob_success_millionths,
+	    u64 base_prob_success_millionths,
 	    bool use_shadow,
 	    const struct route_exclusion **exclusions)
 {
@@ -81,6 +82,7 @@ struct payment *payment_new(
 	pinfo->prob_cost_factor = prob_cost_factor_millionths / 1e6;
 	pinfo->delay_feefactor = riskfactor_millionths / 1e6;
 	pinfo->min_prob_success = min_prob_success_millionths / 1e6;
+	pinfo->base_prob_success = base_prob_success_millionths / 1e6;
 	pinfo->use_shadow = use_shadow;
 
 
@@ -146,6 +148,7 @@ bool payment_update(
 		u64 prob_cost_factor_millionths,
 		u64 riskfactor_millionths,
 		u64 min_prob_success_millionths,
+		u64 base_prob_success_millionths,
 		bool use_shadow,
 		const struct route_exclusion **exclusions)
 {
@@ -170,6 +173,7 @@ bool payment_update(
 	pinfo->prob_cost_factor = prob_cost_factor_millionths / 1e6;
 	pinfo->delay_feefactor = riskfactor_millionths / 1e6;
 	pinfo->min_prob_success = min_prob_success_millionths / 1e6;
+	pinfo->base_prob_success = base_prob_success_millionths / 1e6;
 	pinfo->use_shadow = use_shadow;
 
 

plugins/renepay/payment.h

@@ -117,6 +117,7 @@ struct payment *payment_new(
 	u64 prob_cost_factor_millionths,
 	u64 riskfactor_millionths,
 	u64 min_prob_success_millionths,
+	u64 base_prob_success_millionths,
 	bool use_shadow,
 	const struct route_exclusion **exclusions);
 
@@ -131,6 +132,7 @@ bool payment_update(
 	u64 prob_cost_factor_millionths,
 	u64 riskfactor_millionths,
 	u64 min_prob_success_millionths,
+	u64 base_prob_success_millionths,
 	bool use_shadow,
 	const struct route_exclusion **exclusions);
 

plugins/renepay/payment_info.h

@@ -61,6 +61,10 @@ struct payment_info {
 	double prob_cost_factor;
 	/* prob. cost = - prob_cost_factor * log prob. */
 
+	/* The probability for a channel to be able to forward an amount
+	 * greater than zero. */
+	double base_prob_success;
+
 	/* Penalty for CLTV delays */
 	double delay_feefactor;
 

plugins/renepay/routebuilder.c

@@ -151,6 +151,7 @@ struct route **get_routes(const tal_t *ctx,
 	struct route **routes = tal_arr(ctx, struct route *, 0);
 
 	double probability_budget = payment_info->min_prob_success;
+	const double base_probability = payment_info->base_prob_success;
 	double delay_feefactor = payment_info->delay_feefactor;
 	const double base_fee_penalty = payment_info->base_fee_penalty;
 	const double prob_cost_factor = payment_info->prob_cost_factor;
@@ -215,8 +216,11 @@ struct route **get_routes(const tal_t *ctx,
 		    minflow(this_ctx, gossmap, src, dst,
 			    uncertainty_get_chan_extra_map(uncertainty),
 			    disabled_bitmap, amount_to_deliver, feebudget,
-			    probability_budget, delay_feefactor,
+			    probability_budget,
-			    base_fee_penalty, prob_cost_factor, &errmsg);
+			    base_probability,
+			    delay_feefactor,
+			    base_fee_penalty,
+			    prob_cost_factor, &errmsg);
 		delay_feefactor_updated = false;
 
 		if (!flows) {

plugins/renepay/test/run-bottleneck.c

@@ -202,6 +202,7 @@ int main(int argc, char *argv[])
  		    AMOUNT_MSAT(100 * 1000 * 1000),
  		    /* max_fee = */ AMOUNT_MSAT(20 * 1000 * 1000),
  		    /* min probability = */ 0.9,
+		    /* base probability = */ 1.0,
  		    /* delay fee factor = */ 1e-6,
  		    /* base fee penalty */ 10,
  		    /* prob cost factor = */ 10, &errmsg);
@@ -239,6 +240,7 @@ int main(int argc, char *argv[])
 	pinfo.prob_cost_factor = 1e-5;
 	pinfo.delay_feefactor = 1e-6;
 	pinfo.min_prob_success = 0.9;
+	pinfo.base_prob_success = 1.0;
 	pinfo.use_shadow = false;
 
 	randombytes_buf(&preimage, sizeof(preimage));

plugins/renepay/test/run-mcf-diamond.c

@@ -194,6 +194,7 @@ int main(int argc, char *argv[])
 			AMOUNT_MSAT(1000000), // 1000 sats
 			 /* max_fee = */ AMOUNT_MSAT(10000), // 10 sats
 			 /* min probability = */ 0.8, // 80%
+			 /* base probability = */ 1.0,
 			 /* delay fee factor = */ 0,
 			 /* base fee penalty */ 0,
 			 /* prob cost factor = */ 1,

plugins/renepay/test/run-mcf.c

@@ -399,6 +399,7 @@ int main(int argc, char *argv[])
 			AMOUNT_MSAT(500000000),
 			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
 			 /* min probability = */ 0.1,
+			 /* base probability = */ 1.0,
 			 /* delay fee factor = */ 1,
 			 /* base fee penalty */ 1,
 			 /* prob cost factor = */ 10,
@@ -564,6 +565,7 @@ int main(int argc, char *argv[])
 			 /* amount = */ AMOUNT_MSAT(500000000), //500k sats
 			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
 			 /* min probability = */ 0.1, // 10%
+			 /* base probability = */ 1.0,
 			 /* delay fee factor = */ 1,
 			 /* base fee penalty */ 1,
 			 /* prob cost factor = */ 10,

plugins/renepay/test/run-missingcapacity.c

@@ -133,6 +133,7 @@ int main(int argc, char *argv[])
 	pinfo.prob_cost_factor = 1e-5;
 	pinfo.delay_feefactor = 1e-6;
 	pinfo.min_prob_success = 0.9;
+	pinfo.base_prob_success = 1.0;
 	pinfo.use_shadow = false;
 
 	randombytes_buf(&preimage, sizeof(preimage));