mirror of
https://github.com/Blockstream/satellite-api.git
synced 2025-03-13 11:35:20 +01:00
5882fddc49
This change implements a mechanism to retransmit orders if some of the order's selected regions do not confirm transmission in due time. It adds a worker to repeatedly check the orders and determine if they need retransmission. Such orders will be added to a new table named tx_retries. The tx_start function now first checks if there are regular new paid orders to transmit. If not, it will check the retransmission table and retransmit an order from there if one is available. This patch also introduces a new order state called "retranmission". The order enters this state while waiting for retransmission.
362 lines
16 KiB
Python
362 lines
16 KiB
Python
import json
|
|
import pytest
|
|
from datetime import datetime, timedelta
|
|
from unittest.mock import patch
|
|
|
|
from common import generate_test_order, pay_invoice, confirm_tx
|
|
from constants import InvoiceStatus, OrderStatus
|
|
import transmitter
|
|
from database import db
|
|
from schemas import order_schema
|
|
from models import Order, TxRetry, TxConfirmation
|
|
from regions import Regions, all_region_numbers, region_number_list_to_code
|
|
from order_helpers import refresh_retransmission_table, \
|
|
get_next_retransmission, sent_or_received_criteria_met, assert_order_state
|
|
import constants
|
|
import server
|
|
|
|
|
|
@pytest.fixture
|
|
def app(mockredis):
|
|
app = server.create_app(from_test=True)
|
|
app.app_context().push()
|
|
yield app
|
|
server.teardown_app(app)
|
|
|
|
|
|
@pytest.fixture
|
|
def client(app):
|
|
with app.test_client() as client:
|
|
yield client
|
|
|
|
|
|
def assert_redis_call(mockredis, order):
|
|
msg = order_schema.dump(order)
|
|
msg = json.dumps(msg)
|
|
mockredis.publish.assert_called_with(channel='transmissions', message=msg)
|
|
|
|
|
|
@patch('orders.new_invoice')
|
|
def test_tx_start(mock_new_invoice, client, mockredis):
|
|
# Create two orders and pay only for the first.
|
|
first_order_uuid = generate_test_order(mock_new_invoice, client)['uuid']
|
|
second_order_uuid = generate_test_order(mock_new_invoice,
|
|
client,
|
|
order_id=2)['uuid']
|
|
|
|
first_db_order = Order.query.filter_by(uuid=first_order_uuid).first()
|
|
|
|
# The transmission should start immediately upon payment
|
|
pay_invoice(first_db_order.invoices[0], client)
|
|
assert_redis_call(mockredis, first_db_order)
|
|
|
|
# The expectation is that the first order gets transmitted and the second
|
|
# stays untouched. The invoice callback handler should call tx_start.
|
|
assert_order_state(first_order_uuid, 'transmitting')
|
|
assert first_db_order.tx_seq_num is not None
|
|
assert_order_state(second_order_uuid, 'pending')
|
|
second_db_order = Order.query.filter_by(uuid=second_order_uuid).first()
|
|
assert second_db_order.tx_seq_num is None
|
|
|
|
# Calling tx_start explicitly won't change anything since the second order
|
|
# is still unpaid
|
|
transmitter.tx_start()
|
|
second_db_order = Order.query.filter_by(uuid=second_order_uuid).first()
|
|
assert second_db_order.status == OrderStatus.pending.value
|
|
assert second_db_order.tx_seq_num is None
|
|
|
|
|
|
@patch('orders.new_invoice')
|
|
def test_tx_end(mock_new_invoice, client, mockredis):
|
|
# Create two orders and pay for both.
|
|
first_order_uuid = generate_test_order(mock_new_invoice, client,
|
|
bid=2000)['uuid']
|
|
second_order_uuid = generate_test_order(mock_new_invoice,
|
|
client,
|
|
bid=1000,
|
|
order_id=2)['uuid']
|
|
|
|
first_db_order = Order.query.filter_by(uuid=first_order_uuid).first()
|
|
second_db_order = Order.query.filter_by(uuid=second_order_uuid).first()
|
|
|
|
# As soon as the first order is paid, its transmission should start
|
|
# immediately.
|
|
pay_invoice(first_db_order.invoices[0], client)
|
|
assert_redis_call(mockredis, first_db_order)
|
|
|
|
# Meanwhile, if the second order is paid, its transmission cannot start
|
|
# immediately because the Tx line is still blocked by the first order.
|
|
pay_invoice(second_db_order.invoices[0], client)
|
|
|
|
# The expectation is that the first order gets transmitted and the second
|
|
# stays in paid state. The invoice callback handler should call tx_start.
|
|
assert_order_state(first_order_uuid, 'transmitting')
|
|
assert first_db_order.tx_seq_num is not None
|
|
assert_order_state(second_order_uuid, 'paid')
|
|
assert second_db_order.tx_seq_num is None
|
|
|
|
# The second order should start transmitting immediately after ending the
|
|
# first. The only prerequisite is that the first order is in sent state
|
|
# (after Tx confirmations) when ended.
|
|
confirm_tx(first_db_order.tx_seq_num, all_region_numbers, client)
|
|
if sent_or_received_criteria_met(first_db_order):
|
|
transmitter.tx_end(first_db_order)
|
|
|
|
assert_order_state(first_order_uuid, 'sent')
|
|
assert first_db_order.ended_transmission_at is not None
|
|
assert_order_state(second_order_uuid, 'transmitting')
|
|
assert second_db_order.tx_seq_num is not None
|
|
|
|
|
|
@patch('orders.new_invoice')
|
|
def test_assign_tx_seq_num(mock_new_invoice, client):
|
|
# make some orders
|
|
first_order_uuid = generate_test_order(mock_new_invoice, client)['uuid']
|
|
first_db_order = Order.query.filter_by(uuid=first_order_uuid).first()
|
|
assert first_db_order.tx_seq_num is None
|
|
|
|
second_order_uuid = generate_test_order(mock_new_invoice,
|
|
client,
|
|
order_id=2)['uuid']
|
|
second_db_order = Order.query.filter_by(uuid=second_order_uuid).first()
|
|
assert second_db_order.tx_seq_num is None
|
|
|
|
third_order_uuid = generate_test_order(mock_new_invoice,
|
|
client,
|
|
order_id=3)['uuid']
|
|
third_db_order = Order.query.filter_by(uuid=third_order_uuid).first()
|
|
assert third_db_order.tx_seq_num is None
|
|
|
|
transmitter.assign_tx_seq_num(first_db_order)
|
|
db.session.commit()
|
|
first_db_order = Order.query.filter_by(uuid=first_order_uuid).first()
|
|
assert first_db_order.tx_seq_num == 1
|
|
|
|
transmitter.assign_tx_seq_num(second_db_order)
|
|
db.session.commit()
|
|
second_db_order = Order.query.filter_by(uuid=second_order_uuid).first()
|
|
assert second_db_order.tx_seq_num == 2
|
|
|
|
transmitter.assign_tx_seq_num(third_db_order)
|
|
db.session.commit()
|
|
third_db_order = Order.query.filter_by(uuid=third_order_uuid).first()
|
|
assert third_db_order.tx_seq_num == 3
|
|
|
|
|
|
@patch('orders.new_invoice')
|
|
def test_startup_sequence(mock_new_invoice, client, mockredis):
|
|
# create an old transmitted order
|
|
transmitting_order_uuid = generate_test_order(
|
|
mock_new_invoice,
|
|
client,
|
|
invoice_status=InvoiceStatus.paid,
|
|
order_status=OrderStatus.transmitting)['uuid']
|
|
|
|
# create two paid orders
|
|
first_sendable_order_uuid = generate_test_order(mock_new_invoice,
|
|
client,
|
|
order_id=5,
|
|
bid=1000)['uuid']
|
|
second_sendable_order_uuid = generate_test_order(mock_new_invoice,
|
|
client,
|
|
order_id=6,
|
|
bid=2000)['uuid']
|
|
first_sendable_db_order = \
|
|
Order.query.filter_by(uuid=first_sendable_order_uuid).first()
|
|
second_sendable_db_order = \
|
|
Order.query.filter_by(uuid=second_sendable_order_uuid).first()
|
|
|
|
pay_invoice(first_sendable_db_order.invoices[0], client)
|
|
pay_invoice(second_sendable_db_order.invoices[0], client)
|
|
|
|
# At startup, tx_start() should trigger the transmission of the highest
|
|
# bidder among the two paid orders, namely the second sendable order.
|
|
# However, this transmission is not possible until the transmitting order
|
|
# from the previous session times out and changes to confirming state.
|
|
transmitter.tx_start()
|
|
assert_order_state(first_sendable_order_uuid, 'paid')
|
|
assert_order_state(second_sendable_order_uuid, 'paid')
|
|
|
|
# Force the timeout by manipulating the transmission timestamp.
|
|
transmitting_db_order = \
|
|
Order.query.filter_by(uuid=transmitting_order_uuid).first()
|
|
transmitting_db_order.started_transmission_at = datetime.utcnow(
|
|
) - timedelta(seconds=constants.TX_CONFIRM_TIMEOUT_SECS + 1)
|
|
db.session.commit()
|
|
refresh_retransmission_table()
|
|
assert_order_state(transmitting_order_uuid, 'confirming')
|
|
|
|
# Now, tx_start() can trigger the new transmission.
|
|
transmitter.tx_start()
|
|
assert_order_state(first_sendable_order_uuid, 'paid')
|
|
assert_order_state(second_sendable_order_uuid, 'transmitting')
|
|
|
|
# At this point, the Tx hosts send Tx confirmations
|
|
confirm_tx(second_sendable_db_order.tx_seq_num, all_region_numbers, client)
|
|
|
|
# Finally, tx_end() shall end the first transmission and trigger the second
|
|
# transmission (the first sendable order).
|
|
transmitter.tx_end(second_sendable_db_order)
|
|
assert_order_state(first_sendable_order_uuid, 'transmitting')
|
|
assert_order_state(second_sendable_order_uuid, 'sent')
|
|
|
|
# The sequence numbers should reflect the transmission order
|
|
assert first_sendable_db_order.tx_seq_num == 2
|
|
assert second_sendable_db_order.tx_seq_num == 1
|
|
|
|
|
|
@patch('orders.new_invoice')
|
|
def test_retransmission(mock_new_invoice, client, mockredis):
|
|
# 1) Order that requires retransmission due to not being confirmed
|
|
# by all Tx regions within the time limit.
|
|
first_order_uuid = generate_test_order(mock_new_invoice, client,
|
|
bid=1000)['uuid']
|
|
|
|
# Pay invoice -> State changes from pending to transmitting.
|
|
first_order = Order.query.filter_by(uuid=first_order_uuid).first()
|
|
pay_invoice(first_order.invoices[0], client)
|
|
assert_order_state(first_order_uuid, 'transmitting')
|
|
assert_redis_call(mockredis, first_order)
|
|
|
|
# Confirm Tx over a single region -> State changes from transmitting to
|
|
# confirming.
|
|
confirm_tx(1, [all_region_numbers[0]], client)
|
|
assert_order_state(first_order_uuid, 'confirming')
|
|
|
|
# Manipulate the Tx confirmation timestamp such that it exceeds the time
|
|
# limit and later leads to a retransmission.
|
|
last_tx_confirmation = TxConfirmation.query.filter_by(
|
|
order_id=first_order.id).order_by(
|
|
TxConfirmation.created_at.desc()).first()
|
|
last_tx_confirmation.created_at = datetime.utcnow() - timedelta(
|
|
seconds=constants.TX_CONFIRM_TIMEOUT_SECS + 1)
|
|
db.session.commit()
|
|
|
|
# 2) Order that needs retransmission due to not receiving any confirmation
|
|
# at all within the timeout limit.
|
|
second_order_uuid = generate_test_order(mock_new_invoice, client,
|
|
bid=2000)['uuid']
|
|
|
|
# Pay invoice -> State changes from pending to transmitting.
|
|
second_order = Order.query.filter_by(uuid=second_order_uuid).first()
|
|
pay_invoice(second_order.invoices[0], client)
|
|
assert_order_state(second_order_uuid, 'transmitting')
|
|
assert_redis_call(mockredis, second_order)
|
|
|
|
# Manipulate the Tx start timestamp such that it exceeds the time limit and
|
|
# later leads to a retransmission.
|
|
second_order.started_transmission_at = datetime.utcnow() - timedelta(
|
|
seconds=constants.TX_CONFIRM_TIMEOUT_SECS + 1)
|
|
db.session.commit()
|
|
|
|
# 3) Order that transmits normally with no need for retransmission.
|
|
third_order_regions = [Regions.g18.value, Regions.e113.value]
|
|
third_order_uuid = generate_test_order(mock_new_invoice,
|
|
client,
|
|
bid=2000,
|
|
regions=third_order_regions)['uuid']
|
|
|
|
# Pay invoice. In this case, the state changes from pending to paid (not to
|
|
# transmitting) since the Tx line is still blocked by the second order.
|
|
third_order = Order.query.filter_by(uuid=third_order_uuid).first()
|
|
pay_invoice(third_order.invoices[0], client)
|
|
assert_order_state(third_order_uuid, 'paid')
|
|
|
|
# Detect and update all the required retransmissions
|
|
refresh_retransmission_table()
|
|
|
|
# The first and second orders should require retransmission. Also, the
|
|
# second order should have changed from transmitting to confirming.
|
|
retry_order = TxRetry.query.all()
|
|
assert len(retry_order) == 2
|
|
assert retry_order[0].order_id == first_order.id
|
|
assert retry_order[1].order_id == second_order.id
|
|
assert_order_state(second_order_uuid, 'confirming')
|
|
|
|
# Check the next order for retransmission, which should be the highest
|
|
# bidder among the two with pending retransmission (the second order)
|
|
order, retry_info = get_next_retransmission()
|
|
assert order and retry_info
|
|
assert order.id == second_order.id
|
|
assert retry_info.order_id == second_order.id
|
|
|
|
# At this point, a worker would see orders requiring retransmission and
|
|
# call tx_start.
|
|
transmitter.tx_start()
|
|
|
|
# The third order should be prioritized because it's not a retransmission.
|
|
assert_order_state(third_order_uuid, 'transmitting')
|
|
assert_redis_call(mockredis, third_order)
|
|
|
|
# Confirm Tx for all regions. That should end the transmission and kick off
|
|
# the next, namely the retransmission of the second order. When the
|
|
# retransmission starts, the second order goes back from confirming to
|
|
# transmitting state.
|
|
confirm_tx(3, third_order_regions, client)
|
|
assert_order_state(third_order_uuid, 'sent')
|
|
assert_order_state(second_order_uuid, 'transmitting')
|
|
assert_redis_call(mockredis, second_order)
|
|
|
|
# Now, pretend the second order received all the required confirmations
|
|
# such that its transmission ended and the next started.
|
|
confirm_tx(second_order.tx_seq_num, all_region_numbers, client)
|
|
assert_order_state(second_order_uuid, 'sent')
|
|
|
|
# tx_end (called under the hood by the Tx confirmation handler) should
|
|
# remove the second order from the tx_retries table and start transmitting
|
|
# the next order, namely the retransmission of the first.
|
|
retry_order = TxRetry.query.all()
|
|
assert len(retry_order) == 1
|
|
assert retry_order[0].order_id == first_order.id
|
|
assert retry_order[0].retry_count == 1
|
|
assert retry_order[0].last_attempt is not None
|
|
assert_order_state(first_order_uuid, 'transmitting')
|
|
|
|
# Besides, since the first order was confirmed by the first region before,
|
|
# now the retransmission should go over the remaining regions only.
|
|
expected_redis_order = first_order
|
|
expected_redis_order.region_code = region_number_list_to_code(
|
|
all_region_numbers[1:])
|
|
assert_redis_call(mockredis, expected_redis_order)
|
|
|
|
# Next, suppose no confirmations are sent for the first retransmission.
|
|
# That should lead to a second retransmission. Manipulate the
|
|
# retransmission info to make that happen.
|
|
t_last_attempt = retry_order[0].last_attempt
|
|
retry_order[0].last_attempt = t_last_attempt - \
|
|
timedelta(seconds=constants.TX_CONFIRM_TIMEOUT_SECS + 1)
|
|
db.session.commit()
|
|
|
|
# A worker would timeout the order and put it back to confirming state.
|
|
refresh_retransmission_table()
|
|
assert_order_state(first_order_uuid, 'confirming')
|
|
|
|
# Another worker would see the confirming order and call tx_start(),
|
|
# leading to the second retransmission.
|
|
transmitter.tx_start()
|
|
assert_order_state(first_order_uuid, 'transmitting')
|
|
assert_redis_call(mockredis, expected_redis_order)
|
|
retry_order = TxRetry.query.all()
|
|
assert len(retry_order) == 1
|
|
assert retry_order[0].order_id == first_order.id
|
|
assert retry_order[0].retry_count == 2
|
|
|
|
# Lastly, suppose this second retransmission receives Tx confirmations, but
|
|
# not all of the required ones. Hence, it should be retransmitted one more
|
|
# time. Again, manipulate the retransmission info to make that happen. This
|
|
# time, note it's the wait interval that determines the retransmission, not
|
|
# the timeout interval. Also, the wait interval should be applied to the
|
|
# last (most recent) Tx confirmation, not the last retransmission time.
|
|
confirm_tx(first_order.tx_seq_num, all_region_numbers[1:3], client)
|
|
assert_order_state(first_order_uuid, 'confirming')
|
|
|
|
for order in TxConfirmation.query.filter_by(order_id=first_order.id).all():
|
|
order.created_at = datetime.utcnow() - timedelta(
|
|
seconds=constants.TX_CONFIRM_TIMEOUT_SECS + 1)
|
|
db.session.commit()
|
|
|
|
transmitter.tx_start()
|
|
retry_order = TxRetry.query.all()
|
|
assert len(retry_order) == 1
|
|
assert retry_order[0].order_id == first_order.id
|
|
assert retry_order[0].retry_count == 3
|