Auto-detect language on payment page (#2552)

* Auto-detect language on payment page

based on the requst Accept-Language header, which is the language you configured in your browser/OS and this 99.99% accurate

* Update BTCPayServer/Services/LanguageService.cs

Co-authored-by: britttttk <39231115+britttttk@users.noreply.github.com>

* Update BTCPayServer/Services/LanguageService.cs

Co-authored-by: Andrew Camilleri <evilkukka@gmail.com>

* Update BTCPayServer/Services/LanguageService.cs

Co-authored-by: Andrew Camilleri <evilkukka@gmail.com>

* Update BTCPayServer/Services/LanguageService.cs

Co-authored-by: Andrew Camilleri <evilkukka@gmail.com>

* Added loop for all locales in Accept-Language sorted by weight + check if know this language

* New public method so a unit test can be created for it

* Unit test for language detection

* Fix language service when not in browser context

* fall back to default lang

* Auto-detect setting + ?lang=auto support

* Added invoice param "?lang=auto" info to docs

* Using null-coalescing assignment operator

* Reduce complexity and http dependency in language service

Co-authored-by: britttttk <39231115+britttttk@users.noreply.github.com>
Co-authored-by: Andrew Camilleri <evilkukka@gmail.com>

BTCPayServer.Tests/GreenfieldAPITests.cs

@@ -1236,19 +1236,19 @@ namespace BTCPayServer.Tests
                 var langs = tester.PayTester.GetService<LanguageService>();
                 foreach (var match in new[] { "it", "it-IT", "it-LOL" })
                 {
-                    Assert.Equal("it-IT", langs.FindBestMatch(match).Code);
+                    Assert.Equal("it-IT", langs.FindLanguage(match).Code);
                 }
                 foreach (var match in new[] { "pt-BR" })
                 {
-                    Assert.Equal("pt-BR", langs.FindBestMatch(match).Code);
+                    Assert.Equal("pt-BR", langs.FindLanguage(match).Code);
                 }
                 foreach (var match in new[] { "en", "en-US" })
                 {
-                    Assert.Equal("en", langs.FindBestMatch(match).Code);
+                    Assert.Equal("en", langs.FindLanguage(match).Code);
                 }
                 foreach (var match in new[] { "pt", "pt-pt", "pt-PT" })
                 {
-                    Assert.Equal("pt-PT", langs.FindBestMatch(match).Code);
+                    Assert.Equal("pt-PT", langs.FindLanguage(match).Code);
                 }
 
                 //payment method activation tests

BTCPayServer.Tests/LanguageServiceTests.cs

@@ -0,0 +1,52 @@
+using System.Threading.Tasks;
+using BTCPayServer.Services;
+using BTCPayServer.Tests.Logging;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace BTCPayServer.Tests
+{
+    public class LanguageServiceTests
+    {
+        public const int TestTimeout = TestUtils.TestTimeout;
+        public LanguageServiceTests(ITestOutputHelper helper)
+        {
+            Logs.Tester = new XUnitLog(helper) { Name = "Tests" };
+            Logs.LogProvider = new XUnitLogProvider(helper);
+        }
+
+        [Fact(Timeout = TestTimeout)]
+        [Trait("Integration", "Integration")]
+        public async Task CanAutoDetectLanguage()
+        {
+            using (var tester = ServerTester.Create())
+            {
+                await tester.StartAsync();
+                var languageService = tester.PayTester.GetService<LanguageService>();
+                
+                // Most common format. First option does not have a quality score. Others do in descending order.
+                // Result should be nl-NL (because the default weight is 1 for nl)
+                var lang1 = languageService.FindLanguageInAcceptLanguageHeader("nl,fr;q=0.7,en;q=0.5");
+                Assert.NotNull(lang1);
+                Assert.Equal("nl-NL", lang1?.Code);
+
+                // Most common format. First option does not have a quality score. Others do in descending order. This time the first option includes a country.
+                // Result should be nl-NL (because the default weight is 1 for nl-BE and it does not exist in BTCPay Server, but nl-NL does and applies too for language "nl")
+                var lang2 = languageService.FindLanguageInAcceptLanguageHeader("nl-BE,fr;q=0.7,en;q=0.5");
+                Assert.NotNull(lang2);
+                Assert.Equal("nl-NL", lang2?.Code);
+
+                // Unusual format, but still valid. All values have a quality score and not ordered.
+                // Result should be fr-FR (because 0.7 is the highest quality score)
+                var lang3 = languageService.FindLanguageInAcceptLanguageHeader("nl;q=0.1,fr;q=0.7,en;q=0.5");
+                Assert.NotNull(lang3);
+                Assert.Equal("fr-FR", lang3?.Code);
+
+                // Unusual format, but still valid. Some language is given that we don't have and a wildcard for everything else. 
+                // Result should be NULL, because "xx" does not exist and * is a wildcard and has no meaning.
+                var lang4 = languageService.FindLanguageInAcceptLanguageHeader("xx,*;q=0.5");
+                Assert.Null(lang4);
+            }
+        }
+    }
+}

BTCPayServer/Controllers/GreenField/InvoiceController.cs

@@ -195,7 +195,7 @@ namespace BTCPayServer.Controllers.GreenField
 
             if (request.Checkout.DefaultLanguage != null)
             {
-                var lang = LanguageService.FindBestMatch(request.Checkout.DefaultLanguage);
+                var lang = LanguageService.FindLanguage(request.Checkout.DefaultLanguage);
                 if (lang == null)
                 {
                     request.AddModelError(invoiceRequest => invoiceRequest.Checkout.DefaultLanguage,

BTCPayServer/Controllers/InvoiceController.UI.cs

@@ -482,7 +482,7 @@ namespace BTCPayServer.Controllers
             return View(model);
         }
 
-        private async Task<PaymentModel?> GetInvoiceModel(string invoiceId, PaymentMethodId paymentMethodId, string lang)
+        private async Task<PaymentModel?> GetInvoiceModel(string invoiceId, PaymentMethodId paymentMethodId, string? lang)
         {
             var invoice = await _InvoiceRepository.GetInvoice(invoiceId);
             if (invoice == null)
@@ -531,6 +531,21 @@ namespace BTCPayServer.Controllers
             var paymentMethodHandler = _paymentMethodHandlerDictionary[paymentMethodId];
 
             var divisibility = _CurrencyNameTable.GetNumberFormatInfo(paymentMethod.GetId().CryptoCode, false)?.CurrencyDecimalDigits;
+
+            switch (lang?.ToLowerInvariant())
+            {
+                case "auto":
+                case null when storeBlob.AutoDetectLanguage:
+                    lang = _languageService.AutoDetectLanguageUsingHeader(HttpContext.Request.Headers, null).Code;
+                    break;
+                case { } langs when !string.IsNullOrEmpty(langs):
+                {
+                    lang = _languageService.FindLanguage(langs)?.Code;
+                    break;
+                }
+            }
+            lang ??= storeBlob.DefaultLang;
+            
             var model = new PaymentModel()
             {
                 Activated = paymentMethodDetails.Activated,

BTCPayServer/Controllers/InvoiceController.cs

@@ -14,6 +14,7 @@ using BTCPayServer.Models;
 using BTCPayServer.Payments;
 using BTCPayServer.Rating;
 using BTCPayServer.Security;
+using BTCPayServer.Services;
 using BTCPayServer.Services.Apps;
 using BTCPayServer.Services.Invoices;
 using BTCPayServer.Services.Rates;
@@ -44,6 +45,7 @@ namespace BTCPayServer.Controllers
         private readonly PaymentMethodHandlerDictionary _paymentMethodHandlerDictionary;
         private readonly ApplicationDbContextFactory _dbContextFactory;
         private readonly PullPaymentHostedService _paymentHostedService;
+        private readonly LanguageService _languageService;
 
         public WebhookNotificationManager WebhookNotificationManager { get; }
 
@@ -59,7 +61,8 @@ namespace BTCPayServer.Controllers
             PaymentMethodHandlerDictionary paymentMethodHandlerDictionary,
             ApplicationDbContextFactory dbContextFactory,
             PullPaymentHostedService paymentHostedService,
-            WebhookNotificationManager webhookNotificationManager)
+            WebhookNotificationManager webhookNotificationManager,
+            LanguageService languageService)
         {
             _CurrencyNameTable = currencyNameTable ?? throw new ArgumentNullException(nameof(currencyNameTable));
             _StoreRepository = storeRepository ?? throw new ArgumentNullException(nameof(storeRepository));
@@ -73,6 +76,7 @@ namespace BTCPayServer.Controllers
             _paymentHostedService = paymentHostedService;
             WebhookNotificationManager = webhookNotificationManager;
             _CSP = csp;
+            _languageService = languageService;
         }
 
 

BTCPayServer/Controllers/StoresController.cs

@@ -416,6 +416,7 @@ namespace BTCPayServer.Controllers
             vm.CustomCSS = storeBlob.CustomCSS;
             vm.CustomLogo = storeBlob.CustomLogo;
             vm.HtmlTitle = storeBlob.HtmlTitle;
+            vm.AutoDetectLanguage = storeBlob.AutoDetectLanguage;
             vm.SetLanguages(_LangService, storeBlob.DefaultLang);
             return View(vm);
         }
@@ -490,6 +491,7 @@ namespace BTCPayServer.Controllers
             blob.CustomLogo = model.CustomLogo;
             blob.CustomCSS = model.CustomCSS;
             blob.HtmlTitle = string.IsNullOrWhiteSpace(model.HtmlTitle) ? null : model.HtmlTitle;
+            blob.AutoDetectLanguage = model.AutoDetectLanguage;
             blob.DefaultLang = model.DefaultLang;
 
             if (CurrentStore.SetStoreBlob(blob))

BTCPayServer/Data/StoreBlob.cs

@@ -84,6 +84,8 @@ namespace BTCPayServer.Data
         public string CustomLogo { get; set; }
         public string HtmlTitle { get; set; }
         
+        public bool AutoDetectLanguage { get; set; }
+
         public bool RateScripting { get; set; }
 
         public string RateScript { get; set; }

BTCPayServer/Models/StoreViewModels/CheckoutExperienceViewModel.cs

@@ -56,6 +56,8 @@ namespace BTCPayServer.Models.StoreViewModels
         [Range(1, double.PositiveInfinity)]
         public int RecommendedFeeBlockTarget { get; set; }
         
+        [Display(Name = "Auto-detect language on checkout")]
+        public bool AutoDetectLanguage { get; set; }
         
         [Display(Name = "Default language on checkout")]
         public string DefaultLang { get; set; }

BTCPayServer/Services/LanguageService.cs

@@ -1,15 +1,29 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using BTCPayServer.Client.Models;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Newtonsoft.Json.Serialization;
 
 namespace BTCPayServer.Services
 {
+    public class Language
+    {
+        public Language(string code, string displayName)
+        {
+            DisplayName = displayName;
+            Code = code;
+        }
+
+        [JsonProperty("code")] public string Code { get; set; }
+        [JsonProperty("currentLanguage")] public string DisplayName { get; set; }
+    }
+
     public class LanguageService
     {
         private readonly Language[] _languages;
@@ -31,35 +45,95 @@ namespace BTCPayServer.Services
 
             _languages = result.ToArray();
         }
+
         public Language[] GetLanguages()
         {
             return _languages;
         }
 
-        public Language FindBestMatch(string defaultLang)
+        public Language FindLanguageInAcceptLanguageHeader(string acceptLanguageHeader)
         {
-            if (defaultLang is null)
-                return null;
-            defaultLang = defaultLang.Trim();
-            if (defaultLang.Length < 2)
-                return null;
-            var split = defaultLang.Split('-', StringSplitOptions.RemoveEmptyEntries);
-            if (split.Length != 1 && split.Length != 2)
+            IDictionary<string, float> acceptedLocales = new Dictionary<string, float>();
+            var locales = acceptLanguageHeader.Split(',', StringSplitOptions.RemoveEmptyEntries);
+            for (int i = 0; i < locales.Length; i++)
+            {
+                try
+                {
+                    var oneLocale = locales[i];
+                    var parts = oneLocale.Split(';', StringSplitOptions.RemoveEmptyEntries);
+                    var locale = parts[0];
+                    var qualityScore = 1.0f;
+                    if (parts.Length == 2)
+                    {
+                        var qualityScorePart = parts[1];
+                        if (qualityScorePart.StartsWith("q=", StringComparison.OrdinalIgnoreCase))
+                        {
+                            qualityScorePart = qualityScorePart.Substring(2);
+                            qualityScore = float.Parse(qualityScorePart, CultureInfo.InvariantCulture);
+                        }
+                        else
+                        {
+                            // Invalid format, continue with next
+                            continue;
+                        }
+                    }
+
+                    if (!locale.Equals("*", StringComparison.OrdinalIgnoreCase))
+                    {
+                        acceptedLocales[locale] = qualityScore;
+                    }
+                }
+                catch (System.FormatException e)
+                {
+                    // Can't use this piece, moving on...
+                }
+            }
+
+            var sortedAcceptedLocales = from entry in acceptedLocales orderby entry.Value descending select entry;
+            foreach (var pair in sortedAcceptedLocales)
+            {
+                var lang = FindLanguage(pair.Key);
+                if (lang != null)
+                {
+                    return lang;
+                }
+            }
+
             return null;
+        }
+
+        /**
+         * Look for a supported language that matches the given locale (can be in different notations like "nl" or "nl-NL").
+         * Example: "nl" is not supported, but we do have "nl-NL"
+         */
+        public Language FindLanguage(string locale)
+        {
+            var supportedLangs = GetLanguages();
+            var split = locale.Split('-', StringSplitOptions.RemoveEmptyEntries);
             var lang = split[0];
             var country = split.Length == 2 ? split[1] : split[0].ToUpperInvariant();
 
             var langStart = lang + "-";
-            var langMatches = GetLanguages()
+            var langMatches = supportedLangs
                 .Where(l => l.Code.Equals(lang, StringComparison.OrdinalIgnoreCase) ||
-                            l.Code.StartsWith(langStart, StringComparison.OrdinalIgnoreCase));
+                            l.Code.StartsWith(langStart, StringComparison.OrdinalIgnoreCase))
+                .ToList();
 
             var countryMatches = langMatches;
             var countryEnd = "-" + country;
-            countryMatches =
-                countryMatches
-                .Where(l => l.Code.EndsWith(countryEnd, StringComparison.OrdinalIgnoreCase));
+            countryMatches = countryMatches.Where(l =>
+                l.Code.EndsWith(countryEnd, StringComparison.OrdinalIgnoreCase)).ToList();
             return countryMatches.FirstOrDefault() ?? langMatches.FirstOrDefault();
         }
+
+        public Language AutoDetectLanguageUsingHeader(IHeaderDictionary headerDictionary, string defaultLang)
+        {
+            if (headerDictionary?.TryGetValue("Accept-Language",
+                out var acceptLanguage) is true && !string.IsNullOrEmpty(acceptLanguage))
+            {
+                return FindLanguageInAcceptLanguageHeader(acceptLanguage.ToString()) ?? FindLanguageInAcceptLanguageHeader(defaultLang);
+            }
+            return FindLanguageInAcceptLanguageHeader(defaultLang);
+        }
     }
 }

BTCPayServer/Views/Invoice/Checkout.cshtml

@@ -115,7 +115,7 @@
     </invoice>
     <script>
     var availableLanguages = @Safe.Json(langService.GetLanguages().Select((language) => language.Code));;
-    var storeDefaultLang = @Safe.Json(langService.FindBestMatch(Model.DefaultLang)?.Code ?? Model.DefaultLang);
+    var defaultLang = @Safe.Json(Model.DefaultLang);
     var fallbackLanguage = "en";
     startingLanguage = computeStartingLanguage();
     i18next
@@ -134,8 +134,8 @@
         if (urlParams.lang && isLanguageAvailable(urlParams.lang)) {
             return urlParams.lang;
         }
-        else if (isLanguageAvailable(storeDefaultLang)) {
-            return storeDefaultLang;
+        else if (isLanguageAvailable(defaultLang)) {
+            return defaultLang;
         } else {
             return fallbackLanguage;
         }

BTCPayServer/Views/Stores/CheckoutExperience.cshtml

@@ -82,6 +82,13 @@
             </div>
 
             <h4 class="mt-5 mb-3">Appearance</h4>
+            <div class="form-group">
+                <div class="form-check">
+                    <input asp-for="AutoDetectLanguage" type="checkbox" class="form-check-input"/>
+                    <label asp-for="AutoDetectLanguage" class="form-check-label"></label>
+                    <p class="form-text text-muted">Detects the language of the customer's browser with 99.9% accuracy.</p>
+                </div>
+            </div>
             <div class="form-group">
                 <label asp-for="DefaultLang" class="form-label"></label>
                 <select asp-for="DefaultLang" asp-items="Model.Languages" class="form-select w-auto"></select>

BTCPayServer/wwwroot/swagger/v1/swagger.template.misc.json

@@ -94,7 +94,7 @@
                     "name": "lang",
                     "in": "query",
                     "required": false,
-                    "description": "The preferred language of the checkout page. You can see the list of language codes with [this operation](#operation/langCodes).",
+                    "description": "The preferred language of the checkout page. You can use \"auto\" to use the language of the customer's browser or see the list of language codes with [this operation](#operation/langCodes).",
                     "schema": {
                         "type": "string"
                     }