Add Tor Http Client Factory (#1445)

* WIP: Add Tor Http Client Factory

This PR adds a Tor Http Client Factory so that we can do HTTP requests over socks5( which we usually use tor's socks5).
Using it for payjoin when possible.

Currently have an issue where a 503 is always returned.  Must be something wrong with my tor config or the socks proxy lib Im using

* fix

* remove external dependency

* make payjoin client use tor client if available

* fix docker test

* use tor client only if available and necessary

* remove bip 79 mention

BTCPayServer.Tests/BTCPayServerTester.cs

@@ -93,6 +93,7 @@ namespace BTCPayServer.Tests
         }
 
         public bool MockRates { get; set; } = true;
+        public string SocksEndpoint { get; set; }
 
         public HashSet<string> Chains { get; set; } = new HashSet<string>(){"BTC"};
         public bool UseLightning { get; set; }
@@ -143,6 +144,7 @@ namespace BTCPayServer.Tests
                 config.AppendLine("allow-admin-registration=1");
            
             config.AppendLine($"torrcfile={TestUtils.GetTestDataFullPath("Tor/torrc")}");
+            config.AppendLine($"socksendpoint={SocksEndpoint}");
             config.AppendLine($"debuglog=debug.log");
 
 

BTCPayServer.Tests/ServerTester.cs

@@ -72,6 +72,7 @@ namespace BTCPayServer.Tests
             PayTester.SSHPassword = GetEnvironment("TESTS_SSHPASSWORD", "opD3i2282D");
             PayTester.SSHKeyFile = GetEnvironment("TESTS_SSHKEYFILE", "");
             PayTester.SSHConnection = GetEnvironment("TESTS_SSHCONNECTION", "root@127.0.0.1:21622");
+            PayTester.SocksEndpoint = GetEnvironment("TESTS_SOCKSENDPOINT", "localhost:9050");
         }
 
         public void ActivateLTC()

BTCPayServer.Tests/UnitTest1.cs

@@ -888,6 +888,25 @@ namespace BTCPayServer.Tests
             }
         }
 
+
+        [Fact(Timeout = TestTimeout)]
+        [Trait("Integration", "Integration")]
+        public async Task CanUseTorClient()
+        {
+            using (var tester = ServerTester.Create())
+            {
+                await tester.StartAsync();
+                var torFactory = tester.PayTester.GetService<Socks5HttpClientFactory>();
+                var client = torFactory.CreateClient("test");
+                Assert.NotNull(client);
+                var response = await client.GetAsync("https://check.torproject.org/");
+                response.EnsureSuccessStatusCode();
+                var result = await response.Content.ReadAsStringAsync();
+                Assert.DoesNotContain("You are not using Tor.", result);
+                Assert.Contains("Congratulations. This browser is configured to use Tor.", result);
+            }
+        }
+
         [Fact(Timeout = TestTimeout)]
         [Trait("Integration", "Integration")]
         public async Task CanRescanWallet()

BTCPayServer.Tests/docker-compose.yml

@@ -27,6 +27,7 @@ services:
       TESTS_SSHCONNECTION: "root@sshd:22"
       TESTS_SSHPASSWORD: ""
       TESTS_SSHKEYFILE: ""
+      TESTS_SOCKSENDPOINT: "tor:9050"
     expose:
       - "80"
     links:
@@ -51,6 +52,7 @@ services:
       - customer_lnd
       - merchant_lnd
       - sshd
+      - tor
 
   sshd:
     build:
@@ -318,6 +320,22 @@ services:
       - "bitcoin_datadir:/deps/.bitcoin"
     links:
       - bitcoind
+
+  tor:
+    restart: unless-stopped
+    image: btcpayserver/tor:0.4.1.5
+    container_name: tor
+    environment:
+        TOR_PASSWORD: btcpayserver
+    ports:
+        - "9050:9050" # SOCKS
+        - "9051:9051" # Tor Control
+    volumes:
+        - "tor_datadir:/home/tor/.tor"
+        - "torrcdir:/usr/local/etc/tor"
+        - "tor_servicesdir:/var/lib/tor/hidden_services"
+
+
 volumes:
     sshd_datadir:
     bitcoin_datadir:
@@ -327,3 +345,6 @@ volumes:
     lightning_charge_datadir:
     customer_lnd_datadir:
     merchant_lnd_datadir:
+    tor_datadir:
+    torrcdir:
+    tor_servicesdir:

BTCPayServer/Hosting/BTCPayServerServices.cs

@@ -68,6 +68,7 @@ namespace BTCPayServer.Hosting
             services.TryAddSingleton<SettingsRepository>();
             services.TryAddSingleton<TorServices>();
             services.TryAddSingleton<SocketFactory>();
+            services.TryAddSingleton<Socks5HttpClientFactory>();
             services.TryAddSingleton<LightningClientFactoryService>();
             services.TryAddSingleton<InvoicePaymentNotification>();
             services.TryAddSingleton<BTCPayServerOptions>(o =>

BTCPayServer/Models/StoreViewModels/StoreViewModel.cs

@@ -87,7 +87,7 @@ namespace BTCPayServer.Models.StoreViewModels
         [Display(Name = "Description template of the lightning invoice")]
         public string LightningDescriptionTemplate { get; set; }
         
-        [Display(Name = "Enable BIP79 Payjoin/P2EP")]
+        [Display(Name = "Enable Payjoin/P2EP")]
         public bool  PayJoinEnabled { get; set; }
 
         public class LightningNode

BTCPayServer/Models/WalletViewModels/WalletSendModel.cs

@@ -48,7 +48,7 @@ namespace BTCPayServer.Models.WalletViewModels
         public bool DisableRBF { get; set; }
 
         public bool NBXSeedAvailable { get; set; }
-        [Display(Name = "PayJoin Endpoint Url (BIP79)")]
+        [Display(Name = "PayJoin Endpoint Url")]
         public string PayJoinEndpointUrl { get; set; }
         public bool InputSelection { get; set; }
         public InputSelectionOption[] InputsAvailable { get; set; }

BTCPayServer/Properties/launchSettings.json

@@ -19,7 +19,8 @@
                 "BTCPAY_CHAINS": "btc,ltc",
                 "BTCPAY_POSTGRES": "User ID=postgres;Host=127.0.0.1;Port=39372;Database=btcpayserver",
                 "BTCPAY_DEBUGLOG": "debug.log",
-                "BTCPAY_TORRCFILE": "../BTCPayServer.Tests/TestData/Tor/torrc"
+                "BTCPAY_TORRCFILE": "../BTCPayServer.Tests/TestData/Tor/torrc",
+                "BTCPAY_SOCKSENDPOINT": "localhost:9050"
             },
             "applicationUrl": "http://127.0.0.1:14142/"
         },

BTCPayServer/Services/PayjoinClient.cs

@@ -43,14 +43,16 @@ namespace BTCPayServer.Services
         public const string BIP21EndpointKey = "bpu";
 
         private readonly ExplorerClientProvider _explorerClientProvider;
-        private HttpClient _httpClient;
+        private HttpClient _clearnetHttpClient;
+        private HttpClient _torHttpClient;
 
-        public PayjoinClient(ExplorerClientProvider explorerClientProvider, IHttpClientFactory httpClientFactory)
+        public PayjoinClient(ExplorerClientProvider explorerClientProvider, IHttpClientFactory httpClientFactory, Socks5HttpClientFactory socks5HttpClientFactory)
         {
             if (httpClientFactory == null) throw new ArgumentNullException(nameof(httpClientFactory));
             _explorerClientProvider =
                 explorerClientProvider ?? throw new ArgumentNullException(nameof(explorerClientProvider));
-            _httpClient = httpClientFactory.CreateClient("payjoin");
+            _clearnetHttpClient =  httpClientFactory.CreateClient("payjoin");
+            _torHttpClient = socks5HttpClientFactory.CreateClient("payjoin");
         }
 
         public async Task<PSBT> RequestPayjoin(Uri endpoint, DerivationSchemeSettings derivationSchemeSettings,
@@ -93,7 +95,12 @@ namespace BTCPayServer.Services
             }
 
             cloned.GlobalXPubs.Clear();
-            var bpuresponse = await _httpClient.PostAsync(endpoint,
+            HttpClient client = _clearnetHttpClient;
+            if (endpoint.IsOnion() && _torHttpClient != null)
+            {
+                client = _torHttpClient;
+            }
+            var bpuresponse = await client.PostAsync(endpoint,
                 new StringContent(cloned.ToHex(), Encoding.UTF8, "text/plain"), cancellationToken);
             if (!bpuresponse.IsSuccessStatusCode)
             {

BTCPayServer/Services/Proxy/ProxyClient.cs

@@ -0,0 +1,415 @@
+using System;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BTCPayServer.Services.Proxy
+{
+    /// <summary>
+    /// https://github.com/TheSuunny/Yove.Proxy
+    /// </summary>
+    public class ProxyClient : IDisposable, IWebProxy
+    {
+        public enum ProxyType
+        {
+            Socks4,
+            Socks5
+        }
+        #region IWebProxy
+
+        public ICredentials Credentials { get; set; }
+
+        public int ReadWriteTimeOut { get; set; } = 60000;
+
+        public Uri GetProxy(Uri destination) => HttpProxyURL;
+        public bool IsBypassed(Uri host) => false;
+
+        #endregion
+
+        #region ProxyClient
+
+        private Uri HttpProxyURL { get; set; }
+        private Socket InternalSocketServer { get; set; }
+        private int InternalSocketPort { get; set; }
+
+        private IPAddress Host { get; set; }
+        private int Port { get; set; }
+        private ProxyType Type { get; set; }
+        private int SocksVersion { get; set; }
+
+        public bool IsDisposed { get; set; }
+
+        #endregion
+
+        #region Constants
+
+        private const byte AddressTypeIPV4 = 0x01;
+        private const byte AddressTypeIPV6 = 0x04;
+        private const byte AddressTypeDomainName = 0x03;
+
+        #endregion
+
+        public ProxyClient(string Proxy, ProxyType Type)
+        {
+            string Host = Proxy.Split(':')[0]?.Trim();
+            int Port = Convert.ToInt32(Proxy.Split(':')[1]?.Trim());
+
+            if (string.IsNullOrEmpty(Host))
+                throw new ArgumentNullException("Host null or empty");
+
+            if (Port < 0 || Port > 65535)
+                throw new ArgumentOutOfRangeException("Port goes beyond");
+
+            this.Host = GetHost(Host);
+            this.Port = Port;
+            this.Type = Type;
+
+            SocksVersion = (Type == ProxyType.Socks4) ? 4 : 5;
+
+            CreateInternalServer();
+        }
+
+        public ProxyClient(string Host, int Port, ProxyType Type)
+        {
+            if (string.IsNullOrEmpty(Host))
+                throw new ArgumentNullException("Host null or empty");
+
+            if (Port < 0 || Port > 65535)
+                throw new ArgumentOutOfRangeException("Port goes beyond");
+
+            this.Host = GetHost(Host);
+            this.Port = Port;
+            this.Type = Type;
+
+            SocksVersion = (Type == ProxyType.Socks4) ? 4 : 5;
+
+            CreateInternalServer();
+        }
+
+        private void CreateInternalServer()
+        {
+            InternalSocketServer = CreateSocketServer();
+
+            InternalSocketServer.Bind(new IPEndPoint(IPAddress.Loopback, 0));
+            InternalSocketPort = ((IPEndPoint)(InternalSocketServer.LocalEndPoint)).Port;
+
+            HttpProxyURL = new Uri($"http://127.0.0.1:{InternalSocketPort}");
+
+            InternalSocketServer.Listen(512);
+            InternalSocketServer.BeginAccept(new AsyncCallback(AcceptCallback), null);
+        }
+
+        private async void AcceptCallback(IAsyncResult e)
+        {
+            if (IsDisposed)
+                return;
+
+            Socket Socket = InternalSocketServer.EndAccept(e);
+            InternalSocketServer.BeginAccept(new AsyncCallback(AcceptCallback), null);
+
+            byte[] HeaderBuffer = new byte[8192]; // Default Header size
+
+            Socket.Receive(HeaderBuffer, HeaderBuffer.Length, 0);
+
+            string Header = Encoding.ASCII.GetString(HeaderBuffer);
+
+            string HttpVersion = Header.Split(' ')[2].Split('\r')[0]?.Trim();
+            string TargetURL = Header.Split(' ')[1]?.Trim();
+
+            if (string.IsNullOrEmpty(HttpVersion) || string.IsNullOrEmpty(TargetURL))
+                throw new Exception("Unsupported request.");
+
+            string UriHostname = string.Empty;
+            int UriPort = 0;
+
+            if (TargetURL.Contains(":") && !TargetURL.Contains("http://"))
+            {
+                UriHostname = TargetURL.Split(':')[0];
+                UriPort = int.Parse(TargetURL.Split(':')[1]);
+            }
+            else
+            {
+                Uri URL = new Uri(TargetURL);
+
+                UriHostname = URL.Host;
+                UriPort = URL.Port;
+            }
+
+            Socket TargetSocket = CreateSocketServer();
+
+            SocketError Connection = await TrySocksConnection(UriHostname, UriPort, TargetSocket);
+
+            if (Connection != SocketError.Success)
+            {
+                if (Connection == SocketError.HostUnreachable || Connection == SocketError.ConnectionRefused || Connection == SocketError.ConnectionReset)
+                    Send(Socket, $"{HttpVersion} 502 Bad Gateway\r\n\r\n");
+                else if (Connection == SocketError.AccessDenied)
+                    Send(Socket, $"{HttpVersion} 401 Unauthorized\r\n\r\n");
+                else
+                    Send(Socket, $"{HttpVersion} 500 Internal Server Error\r\nX-Proxy-Error-Type: {Connection}\r\n\r\n");
+
+                Dispose(Socket);
+                Dispose(TargetSocket);
+            }
+            else
+            {
+                Send(Socket, $"{HttpVersion} 200 Connection established\r\n\r\n");
+
+                Relay(Socket, TargetSocket, false);
+            }
+        }
+
+        private async Task<SocketError> TrySocksConnection(string DestinationAddress, int DestinationPort, Socket Socket)
+        {
+            try
+            {
+                Socket.Connect(new IPEndPoint(Host, Port));
+
+                if (Type == ProxyType.Socks4)
+                    return await SendSocks4(Socket, DestinationAddress, DestinationPort).ConfigureAwait(false);
+                else if (Type == ProxyType.Socks5)
+                    return await SendSocks5(Socket, DestinationAddress, DestinationPort).ConfigureAwait(false);
+
+                return SocketError.ProtocolNotSupported;
+            }
+            catch (SocketException ex)
+            {
+                return ex.SocketErrorCode;
+            }
+        }
+
+        private void Relay(Socket Source, Socket Target, bool IsTarget)
+        {
+            try
+            {
+                if (!IsTarget)
+                    Task.Run(() => Relay(Target, Source, true));
+
+                int Read = 0;
+                byte[] Buffer = new byte[8192];
+
+                while ((Read = Source.Receive(Buffer, 0, Buffer.Length, SocketFlags.None)) > 0)
+                    Target.Send(Buffer, 0, Read, SocketFlags.None);
+            }
+            catch
+            {
+                // Ignored
+            }
+            finally
+            {
+                if (!IsTarget)
+                {
+                    Dispose(Source);
+                    Dispose(Target);
+                }
+            }
+        }
+
+        private async Task<SocketError> SendSocks4(Socket Socket, string DestinationHost, int DestinationPort)
+        {
+            byte AddressType = GetAddressType(DestinationHost);
+
+            if (AddressType == AddressTypeDomainName)
+                DestinationHost = GetHost(DestinationHost).ToString();
+
+            byte[] Address = GetIPAddressBytes(DestinationHost);
+            byte[] Port = GetPortBytes(DestinationPort);
+            byte[] UserId = new byte[0];
+
+            byte[] Request = new byte[9];
+
+            Request[0] = (byte)SocksVersion;
+            Request[1] = 0x01;
+            Address.CopyTo(Request, 4);
+            Port.CopyTo(Request, 2);
+            UserId.CopyTo(Request, 8);
+            Request[8] = 0x00;
+
+            byte[] Response = new byte[8];
+
+            Socket.Send(Request);
+
+            await WaitStream(Socket).ConfigureAwait(false);
+
+            Socket.Receive(Response);
+
+            if (Response[1] != 0x5a)
+                return SocketError.ConnectionRefused;
+
+            return SocketError.Success;
+        }
+
+        private async Task<SocketError> SendSocks5(Socket Socket, string DestinationHost, int DestinationPort)
+        {
+            byte[] Response = new byte[255];
+
+            byte[] Auth = new byte[3];
+            Auth[0] = (byte)SocksVersion;
+            Auth[1] = (byte)1;
+            Auth[2] = (byte)0;
+
+            Socket.Send(Auth);
+
+            await WaitStream(Socket).ConfigureAwait(false);
+
+            Socket.Receive(Response);
+
+            if (Response[1] != 0x00)
+                return SocketError.ConnectionRefused;
+
+            byte AddressType = GetAddressType(DestinationHost);
+
+            if (AddressType == AddressTypeDomainName)
+                DestinationHost = GetHost(DestinationHost).ToString();
+
+            byte[] Address = GetAddressBytes(AddressType, DestinationHost);
+            byte[] Port = GetPortBytes(DestinationPort);
+
+            byte[] Request = new byte[4 + Address.Length + 2];
+
+            Request[0] = (byte)SocksVersion;
+            Request[1] = 0x01;
+            Request[2] = 0x00;
+            Request[3] = AddressType;
+            Address.CopyTo(Request, 4);
+            Port.CopyTo(Request, 4 + Address.Length);
+
+            Socket.Send(Request);
+
+            await WaitStream(Socket).ConfigureAwait(false);
+
+            Socket.Receive(Response);
+
+            if (Response[1] != 0x00)
+                return SocketError.ConnectionRefused;
+
+            return SocketError.Success;
+        }
+
+        private async Task WaitStream(Socket Socket)
+        {
+            int Sleep = 0;
+            int Delay = (Socket.ReceiveTimeout < 10) ? 10 : Socket.ReceiveTimeout;
+
+            while (Socket.Available == 0)
+            {
+                if (Sleep < Delay)
+                {
+                    Sleep += 10;
+                    await Task.Delay(10).ConfigureAwait(false);
+
+                    continue;
+                }
+
+                throw new Exception($"Timeout waiting for data - {Host}:{Port}");
+            }
+        }
+
+        private Socket CreateSocketServer()
+        {
+            Socket Socket = new Socket(AddressFamily.InterNetworkV6, SocketType.Stream, ProtocolType.Tcp);
+
+            Socket.SetSocketOption(SocketOptionLevel.IPv6, SocketOptionName.IPv6Only, 0);
+            Socket.ExclusiveAddressUse = true;
+
+            Socket.ReceiveTimeout = Socket.SendTimeout = ReadWriteTimeOut;
+
+            return Socket;
+        }
+
+        private void Send(Socket Socket, string Message)
+        {
+            Socket.Send(Encoding.UTF8.GetBytes(Message));
+        }
+
+        private IPAddress GetHost(string Host)
+        {
+            if (IPAddress.TryParse(Host, out IPAddress Ip))
+                return Ip;
+
+            return Dns.GetHostAddresses(Host)[0];
+        }
+
+        private byte[] GetAddressBytes(byte AddressType, string Host)
+        {
+            switch (AddressType)
+            {
+                case AddressTypeIPV4:
+                case AddressTypeIPV6:
+                    return IPAddress.Parse(Host).GetAddressBytes();
+                case AddressTypeDomainName:
+                    byte[] Bytes = new byte[Host.Length + 1];
+
+                    Bytes[0] = (byte)Host.Length;
+                    Encoding.ASCII.GetBytes(Host).CopyTo(Bytes, 1);
+
+                    return Bytes;
+                default:
+                    return null;
+            }
+        }
+
+        private byte GetAddressType(string Host)
+        {
+            if (IPAddress.TryParse(Host, out IPAddress Ip))
+            {
+                if (Ip.AddressFamily == AddressFamily.InterNetwork)
+                    return AddressTypeIPV4;
+
+                return AddressTypeIPV6;
+            }
+
+            return AddressTypeDomainName;
+        }
+
+        private byte[] GetIPAddressBytes(string DestinationHost)
+        {
+            IPAddress Address = null;
+
+            if (!IPAddress.TryParse(DestinationHost, out Address))
+            {
+                IPAddress[] IPs = Dns.GetHostAddresses(DestinationHost);
+
+                if (IPs.Length > 0)
+                    Address = IPs[0];
+            }
+
+            return Address.GetAddressBytes();
+        }
+
+        private byte[] GetPortBytes(int Port)
+        {
+            byte[] ArrayBytes = new byte[2];
+
+            ArrayBytes[0] = (byte)(Port / 256);
+            ArrayBytes[1] = (byte)(Port % 256);
+
+            return ArrayBytes;
+        }
+
+        private void Dispose(Socket Socket)
+        {
+            try
+            {
+                Socket.Close();
+                Socket.Dispose();
+            }
+            catch
+            {
+                // Ignore
+            }
+        }
+
+        public void Dispose()
+        {
+            if (InternalSocketServer != null && !IsDisposed)
+            {
+                IsDisposed = true;
+
+                InternalSocketServer.Disconnect(false);
+                InternalSocketServer.Dispose();
+            }
+        }
+    }
+}

BTCPayServer/Services/SocketFactory.cs

@@ -1,13 +1,43 @@
-using System.Net;
+using System.Collections.Concurrent;
+using System.Net;
+using System.Net.Http;
 using System.Net.Sockets;
 using System.Threading;
 using System.Threading.Tasks;
 using BTCPayServer.Configuration;
+using BTCPayServer.Services.Proxy;
+using NBitcoin;
 using NBitcoin.Protocol.Connectors;
 using NBitcoin.Protocol;
 
 namespace BTCPayServer.Services
 {
+    public class Socks5HttpClientFactory : IHttpClientFactory
+    {
+        private readonly BTCPayServerOptions _options;
+
+        public Socks5HttpClientFactory(BTCPayServerOptions options)
+        {
+            _options = options;
+        }
+        private ConcurrentDictionary<string, HttpClient> cachedClients = new ConcurrentDictionary<string, HttpClient>();
+        public HttpClient CreateClient(string name)
+        {
+            return cachedClients.GetOrAdd(name, s =>
+            {
+                if (_options.SocksEndpoint == null)
+                {
+                    return null;
+                }
+
+                var proxy = new ProxyClient(_options.SocksEndpoint.ToEndpointString(), ProxyClient.ProxyType.Socks5);
+                return new HttpClient(
+                    new HttpClientHandler {Proxy = proxy, },
+                    true);
+            });
+        }
+    }
+
     public class SocketFactory
     {
         private readonly BTCPayServerOptions _options;
@@ -42,7 +72,7 @@ namespace BTCPayServer.Services
             return socket;
         }
 
-        internal static void SafeCloseSocket(System.Net.Sockets.Socket socket)
+        internal static void SafeCloseSocket(Socket socket)
         {
             try
             {