using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; using System.Text; using System.Threading; using System.Threading.Tasks; using Amazon.Auth.AccessControlPolicy; using Amazon.Runtime.Internal; using BTCPayServer.Client; using BTCPayServer.Client.Models; using BTCPayServer.Controllers; using ExchangeSharp; using NBitcoin; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; using OpenQA.Selenium.DevTools.V100.Network; using OpenQA.Selenium.Support.UI; using Xunit; using Xunit.Abstractions; using static BTCPayServer.Tests.TransifexClient; namespace BTCPayServer.Tests { /// /// This class hold easy to run utilities for dev time /// public class UtilitiesTests { public ITestOutputHelper Logs { get; } public UtilitiesTests(ITestOutputHelper logs) { Logs = logs; } internal static string GetSecuritySchemeDescription() { var description = "BTCPay Server supports authenticating and authorizing users through an API Key that is generated by them. Send the API Key as a header value to Authorization with the format: `token {token}`. For a smoother experience, you can generate a url that redirects users to an API key creation screen.\n\n The following permissions are available to the context of the user creating the API Key:\n\n#OTHERPERMISSIONS#\n\nThe following permissions are available if the user is an administrator:\n\n#SERVERPERMISSIONS#\n\nThe following permissions applies to all stores of the user, you can limit to a specific store with the following format: `btcpay.store.cancreateinvoice:6HSHAEU4iYWtjxtyRs9KyPjM9GAQp8kw2T9VWbGG1FnZ`:\n\n#STOREPERMISSIONS#\n\nNote that API Keys only limits permission of a user and can never expand it. If an API Key has the permission `btcpay.server.canmodifyserversettings` but that the user account creating this API Key is not administrator, the API Key will not be able to modify the server settings.\nSome permissions may include other permissions, see [this operation](#operation/permissionsMetadata).\n"; var storePolicies = UIManageController.AddApiKeyViewModel.PermissionValueItem.PermissionDescriptions.Where(pair => Policies.IsStorePolicy(pair.Key) && !pair.Key.EndsWith(":", StringComparison.InvariantCulture)); var serverPolicies = UIManageController.AddApiKeyViewModel.PermissionValueItem.PermissionDescriptions.Where(pair => Policies.IsServerPolicy(pair.Key)); var otherPolicies = UIManageController.AddApiKeyViewModel.PermissionValueItem.PermissionDescriptions.Where(pair => !Policies.IsStorePolicy(pair.Key) && !Policies.IsServerPolicy(pair.Key)); description = description.Replace("#OTHERPERMISSIONS#", string.Join("\n", otherPolicies.Select(pair => $"* `{pair.Key}`: {pair.Value.Title}"))) .Replace("#SERVERPERMISSIONS#", string.Join("\n", serverPolicies.Select(pair => $"* `{pair.Key}`: {pair.Value.Title}"))) .Replace("#STOREPERMISSIONS#", string.Join("\n", storePolicies.Select(pair => $"* `{pair.Key}`: {pair.Value.Title}"))); return description; } // /// // /// This will take the translations from v1 or v2 // /// and upload them to transifex if not found // /// // [FactWithSecret("TransifexAPIToken")] // [Trait("Utilities", "Utilities")] //#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously // public async Task UpdateTransifex() // { // // DO NOT RUN IT, THIS WILL ERASE THE CURRENT TRANSIFEX TRANSLATIONS // var client = GetTransifexClient(); // var translations = JsonTranslation.GetTranslations(TranslationFolder.CheckoutV2); // var enTranslations = translations["en"]; // translations.Remove("en"); // foreach (var t in translations) // { // foreach (var w in t.Value.Words.ToArray()) // { // if (t.Value.Words[w.Key] == null) // t.Value.Words[w.Key] = enTranslations.Words[w.Key]; // } // t.Value.Words.Remove("code"); // t.Value.Words.Remove("NOTICE_WARN"); // } // await client.UpdateTranslations(translations); // } //#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously ///// ///// This utility will copy translations made on checkout v1 to checkout v2 ///// //[Fact] //[Trait("Utilities", "Utilities")] //public void SetTranslationV1ToV2() //{ // var mappings = new Dictionary(); // foreach (var kv in JsonTranslation.GetTranslations(TranslationFolder.CheckoutV1)) // { // var v1File = kv.Value; // var v2File = JsonTranslation.GetTranslation(TranslationFolder.CheckoutV2, v1File.Lang); // if (mappings.Count == 0) // { // foreach (var prop1 in v1File.Words) // foreach (var prop2 in v2File.Words) // { // if (Normalize(prop1.Key) == Normalize(prop2.Key)) // mappings.Add(prop1.Key, prop2.Key); // } // mappings.Add("Copied", "copy_confirm"); // mappings.Add("ConversionTab_BodyDesc", "conversion_body"); // mappings.Add("Return to StoreName", "return_to_store"); // } // foreach (var m in mappings) // { // var orig = v1File.Words[m.Key]; // v2File.Words[m.Value] = orig; // } // v2File.Words["currentLanguage"] = v1File.Words["currentLanguage"]; // v2File.Save(); // } //} //private string Normalize(string name) //{ // return name.Replace("_", "").ToLowerInvariant(); //} /// /// This utility will use selenium to pilot your browser to /// automatically translate a language. /// /// Step 1: Close all Chrome instances /// Step2: Edit "v1" variable if want to translate checkout v1 or v2 /// - Windows: "chrome.exe --remote-debugging-port=9222 https://chat.openai.com/" /// - Linux: "google-chrome --remote-debugging-port=9222 https://chat.openai.com/" /// Step 3: Run this. /// /// [Trait("Utilities", "Utilities")] [FactWithSecret("TransifexAPIToken")] public async Task AutoTranslateChatGPT() { var file = TranslationFolder.CheckoutV2; using var driver = new ChromeDriver(new ChromeOptions() { DebuggerAddress = "127.0.0.1:9222" }); var englishTranslations = JsonTranslation.GetTranslation(file, "en"); TransifexClient client = GetTransifexClient(); var langs = await client.GetLangs(englishTranslations.TransifexProject, englishTranslations.TransifexResource); foreach (var lang in langs) { if (lang == "en") continue; var jsonLangCode = GetLangCodeTransifexToJson(lang); var v1LangFile = JsonTranslation.GetTranslation(TranslationFolder.CheckoutV1, jsonLangCode); if (!v1LangFile.Exists()) continue; var languageCurrent = v1LangFile.Words["currentLanguage"]; if (v1LangFile.ShouldSkip()) { Logs.WriteLine("Skipped " + jsonLangCode); continue; } var langFile = JsonTranslation.GetTranslation(file, jsonLangCode); bool askedPrompt = false; foreach (var translation in langFile.Words) { if (translation.Key == "NOTICE_WARN" || translation.Key == "currentLanguage" || translation.Key == "code") continue; var english = englishTranslations.Words[translation.Key]; if (translation.Value != null) continue; // Already translated //TODO: A better way to avoid rate limits is to use this format: //I am translating a checkout crypto payment page, and I want you to translate it from English (en-US) to French (fr-FR). //## //English: This invoice will expire in //French: //## //English: Scan the QR code, or tap to copy the address. //French: //## //English: Your payment has been received and is now processing. //French: if (!askedPrompt) { driver.FindElement(By.XPath("//a[contains(text(), \"New chat\")]")).Click(); Thread.Sleep(200); var input = driver.FindElement(By.XPath("//textarea[@data-id]")); input.SendKeys($"I am translating a checkout crypto payment page, and I want you to translate it from English (en-US) to {languageCurrent} ({jsonLangCode})."); input.SendKeys(Keys.LeftShift + Keys.Enter); input.SendKeys("Reply only with the translation of the sentences I will give you and nothing more." + Keys.Enter); WaitCanWritePrompt(driver); askedPrompt = true; } english = english.Replace('\n', ' '); driver.FindElement(By.XPath("//textarea[@data-id]")).SendKeys(english + Keys.Enter); WaitCanWritePrompt(driver); var elements = driver.FindElements(By.XPath("//div[contains(@class,'markdown') and contains(@class,'prose')]//p")); var result = elements.Last().Text; langFile.Words[translation.Key] = result; } langFile.Save(); } } private static TransifexClient GetTransifexClient() { return new TransifexClient(FactWithSecretAttribute.GetFromSecrets("TransifexAPIToken")); } private void WaitCanWritePrompt(IWebDriver driver) { retry: Thread.Sleep(200); try { driver.FindElement(By.XPath("//*[contains(text(), \"Regenerate response\")]")); } catch { goto retry; } Thread.Sleep(200); } /// /// This utility will make sure that permission documentation is properly written in swagger.template.json /// [Trait("Utilities", "Utilities")] [Fact] public void UpdateSwagger() { var filePath = Path.Combine(TestUtils.TryGetSolutionDirectoryInfo().FullName, "BTCPayServer", "wwwroot", "swagger", "v1", "swagger.template.json"); var o = JObject.Parse(File.ReadAllText(filePath)); o["components"]["securitySchemes"]["API_Key"]["description"] = GetSecuritySchemeDescription(); File.WriteAllText(filePath, o.ToString(Newtonsoft.Json.Formatting.Indented)); } /// /// Download transifex transactions and put them in BTCPayServer\wwwroot\locales and BTCPayServer\wwwroot\locales\checkout /// [FactWithSecret("TransifexAPIToken")] [Trait("Utilities", "Utilities")] public async Task PullTransifexTranslations() { // 1. Generate an API Token on https://www.transifex.com/user/settings/api/ // 2. Run "dotnet user-secrets set TransifexAPIToken " await PullTransifexTranslationsCore(TranslationFolder.CheckoutV1); await PullTransifexTranslationsCore(TranslationFolder.CheckoutV2); } private async Task PullTransifexTranslationsCore(TranslationFolder folder) { var enTranslation = JsonTranslation.GetTranslation(folder, "en"); var client = GetTransifexClient(); var langs = await client.GetLangs(enTranslation.TransifexProject, enTranslation.TransifexResource); var resourceStrings = await client.GetResourceStrings(enTranslation.TransifexResource); enTranslation.Words.Clear(); enTranslation.Translate(resourceStrings.SourceTranslations); enTranslation.Save(); Task.WaitAll(langs.Select(async l => { if (l == "en") return; retry: try { var langCode = GetLangCodeTransifexToJson(l); var langTranslations = await client.GetTranslations(resourceStrings, l); var translation = JsonTranslation.GetTranslation(folder, langCode); if (translation.ShouldSkip()) { Logs.WriteLine("Skipping " + langCode); return; } if (translation.Words.ContainsKey("InvoiceExpired_Body_3") && translation.Words["InvoiceExpired_Body_3"] == enTranslation.Words["InvoiceExpired_Body_3"]) { translation.Words["InvoiceExpired_Body_3"] = string.Empty; } translation.Translate(langTranslations); translation.Save(); } catch { await Task.Delay(1000); goto retry; } }).ToArray()); } internal static string GetLangCodeTransifexToJson(string l) { if (l == "ne_NP") l = "np-NP"; if (l == "zh_CN") l = "zh-SP"; if (l == "kk") l = "kk-KZ"; return l.Replace("_", "-"); } internal static string GetLangCodeJsonToTransifex(string l) { if (l == "np-NP") l = "ne_NP"; if (l == "zh-SP") l = "zh_CN"; if (l == "kk-KZ") l = "kk"; return l.Replace("-", "_"); } } public class TransifexClient { public TransifexClient(string apiToken) { Client = new HttpClient(); APIToken = apiToken; } public HttpClient Client { get; } public string APIToken { get; } public async Task GetTransifexAsync(string uri) { var message = new HttpRequestMessage(HttpMethod.Get, uri); message.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", APIToken); message.Headers.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/vnd.api+json")); using var response = await Client.SendAsync(message); var str = await response.Content.ReadAsStringAsync(); return JObject.Parse(str); } public async Task UpdateTranslations(Dictionary translations) { var resourceStrings = await GetResourceStrings(translations.First().Value.TransifexResource); List patches = new List(); List batches = new List(); foreach (var translation in translations.Values) { foreach (var word in translation.Words) { if (word.Key == "NOTICE_WARN") continue; patches.Add(new JObject() { ["id"] = $"{translation.TransifexResource}:s:{resourceStrings.KeyToHashMapping[word.Key]}:l:{UtilitiesTests.GetLangCodeJsonToTransifex(translation.Lang)}", ["type"] = "resource_translations", ["attributes"] = new JObject() { ["strings"] = word.Value is null ? null : new JObject() { ["other"] = word.Value } } }); if (patches.Count >= 150) { batches.Add(patches.ToArray()); patches = new List(); } } if (patches.Count > 0) { batches.Add(patches.ToArray()); patches = new List(); } } if (patches.Count > 0) { batches.Add(patches.ToArray()); patches = new List(); } await Task.WhenAll(batches.Select(async batch => { var message = new HttpRequestMessage(HttpMethod.Get, "https://rest.api.transifex.com/resource_translations"); message.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", APIToken); message.Method = HttpMethod.Patch; var content = new StringContent(new JObject() { ["data"] = new JArray(batch.OfType().ToArray()) }.ToString(), Encoding.UTF8); content.Headers.Remove("Content-Type"); content.Headers.TryAddWithoutValidation("Content-Type", "application/vnd.api+json;profile=\"bulk\""); message.Content = content; using var response = await Client.SendAsync(message); var str = await response.Content.ReadAsStringAsync(); }).ToArray()); } public async Task> GetTranslations(ResourceStrings resourceStrings, string lang) { var j = await GetTransifexAsync($"https://rest.api.transifex.com/resource_translations?filter[resource]={resourceStrings.ResourceId}&filter[language]=l:{lang}"); if (j["data"] is null) { return resourceStrings.SourceTranslations.ToDictionary(kv => kv.Key, kv => null as string); } return j["data"].Select(o => (Key: resourceStrings.GetKey(o["id"].Value()), Strings: o["attributes"]["strings"])) .ToDictionary( o => o.Key, o => o.Strings.Type == JTokenType.Null ? null : o.Strings["other"].Value()); } public async Task GetLangs(string projectId, string resourceId) { var json = await GetTransifexAsync($"https://rest.api.transifex.com/resource_language_stats?filter[project]={projectId}&filter[resource]={resourceId}"); return json["data"].Select(o => o["id"].Value().Split(':').Last()).ToArray(); } public async Task GetResourceStrings(string resourceId) { var res = new ResourceStrings(); res.ResourceId = resourceId; var json = await GetTransifexAsync($"https://rest.api.transifex.com/resource_strings?filter[resource]={resourceId}"); res.HashToKeyMapping = json["data"] .ToDictionary( o => o["id"].Value().Split(':').Last(), o => o["attributes"]["key"].Value().Replace("\\.", ".")); res.KeyToHashMapping = res.HashToKeyMapping.ToDictionary(o => o.Value, o => o.Key); res.SourceTranslations = json["data"] .ToDictionary( o => o["attributes"]["key"].Value().Replace("\\.", "."), o => o["attributes"]["strings"]["other"].Value()); return res; } } public class ResourceStrings { public string ResourceId { get; set; } public Dictionary HashToKeyMapping { get; set; } public Dictionary SourceTranslations { get; set; } public Dictionary KeyToHashMapping { get; internal set; } public string GetKey(string hash) { if (HashToKeyMapping.TryGetValue(hash, out var v)) return v; hash = hash.Split(':')[^3]; if (HashToKeyMapping.TryGetValue(hash, out v)) return v; throw new InvalidOperationException(); } } public enum TranslationFolder { CheckoutV1, CheckoutV2 } public class JsonTranslation { public static Dictionary GetTranslations(TranslationFolder folder) { var res = new Dictionary(); var source = GetTranslation(null, folder, "en"); foreach (var f in Directory.GetFiles(GetFolder(folder))) { var lang = Path.GetFileNameWithoutExtension(f); res.Add(lang, GetTranslation(source, folder, lang)); } return res; } public static JsonTranslation GetTranslation(TranslationFolder folder, string lang) { var source = GetTranslation(null, folder, "en"); return GetTranslation(source, folder, lang); } private static JsonTranslation GetTranslation(JsonTranslation sourceTranslation, TranslationFolder folder, string lang) { var fullPath = Path.Combine(GetFolder(folder), $"{lang}.json"); var proj = "o:btcpayserver:p:btcpayserver"; string resource; if (folder == TranslationFolder.CheckoutV1) { resource = $"{proj}:r:enjson"; } else // file == v2 { resource = $"{proj}:r:checkout-v2"; } var words = new Dictionary(); if (File.Exists(fullPath)) { var obj = JObject.Parse(File.ReadAllText(fullPath)); foreach (var prop in obj.Properties()) words.Add(prop.Name, prop.Value.Value()); } if (sourceTranslation != null) { foreach (var w in sourceTranslation.Words) { if (!words.ContainsKey(w.Key)) words.Add(w.Key, null); } } return new JsonTranslation() { FullPath = fullPath, Lang = lang, Words = words, TransifexProject = proj, TransifexResource = resource }; } private static string GetFolder(TranslationFolder file) { if (file == TranslationFolder.CheckoutV1) return Path.Combine(TestUtils.TryGetSolutionDirectoryInfo().FullName, "BTCPayServer", "wwwroot", "locales"); else return Path.Combine(TestUtils.TryGetSolutionDirectoryInfo().FullName, "BTCPayServer", "wwwroot", "locales", "checkout"); } public string Lang { get; set; } public Dictionary Words { get; set; } public string FullPath { get; set; } public string TransifexProject { get; set; } public string TransifexResource { get; private set; } public void Save() { JObject obj = new JObject { { "NOTICE_WARN", "THIS CODE HAS BEEN AUTOMATICALLY GENERATED FROM TRANSIFEX, IF YOU WISH TO HELP TRANSLATION COME ON THE SLACK https://chat.btcpayserver.org/ TO REQUEST PERMISSION TO https://www.transifex.com/btcpayserver/btcpayserver/" }, { "code", Lang }, { "currentLanguage", Words["currentLanguage"] } }; foreach (var kv in Words) { if (obj[kv.Key] is not null) continue; if (kv.Value is null) continue; obj.Add(kv.Key, kv.Value); } try { File.WriteAllText(FullPath, obj.ToString(Newtonsoft.Json.Formatting.Indented)); } catch (FileNotFoundException) { File.Create(FullPath).Close(); File.WriteAllText(FullPath, obj.ToString(Newtonsoft.Json.Formatting.Indented)); } } public void Translate(Dictionary sourceTranslations) { foreach (var o in sourceTranslations) if (o.Value != null) Words.AddOrReplace(o.Key, o.Value); } public bool ShouldSkip() { if (!Words.ContainsKey("currentLanguage")) return true; if (Words["currentLanguage"] == "English") return true; if (Words["currentLanguage"] == "disable") return true; return false; } public bool Exists() { return File.Exists(FullPath); } } }