Can load external plugins during dev to debug more easily (#4518)

* Can load external plugins during dev to debug more easily

* Add again load plugin by project reference

* Make sure we don't load same plugin twice

.gitignore

@@ -298,3 +298,4 @@ Packed Plugins
 Plugins/packed
 
 BTCPayServer/wwwroot/swagger/v1/openapi.json
+BTCPayServer/appsettings.dev.json

BTCPayServer.Common/BTCPayServer.Common.csproj

@@ -5,7 +5,7 @@
   <ItemGroup>
     <FrameworkReference Include="Microsoft.AspNetCore.App" />
     <PackageReference Include="NBXplorer.Client" Version="4.2.3" />
-    <PackageReference Include="NicolasDorier.StandardConfiguration" Version="1.0.0.18" />
+    <PackageReference Include="NicolasDorier.StandardConfiguration" Version="2.0.1" />
   </ItemGroup>
   <ItemGroup Condition="'$(Altcoins)' != 'true'">
     <Compile Remove="Altcoins\**\*.cs"></Compile>

BTCPayServer/BTCPayServer.csproj

@@ -60,8 +60,8 @@
     <PackageReference Include="System.IO.Pipelines" Version="6.0.3" />
     <PackageReference Include="NBitpayClient" Version="1.0.0.39" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="NicolasDorier.CommandLine" Version="1.0.0.2" />
+    <PackageReference Include="NicolasDorier.CommandLine" Version="2.0.0" />
-    <PackageReference Include="NicolasDorier.CommandLine.Configuration" Version="1.0.0.3" />
+    <PackageReference Include="NicolasDorier.CommandLine.Configuration" Version="2.0.0" />
     <PackageReference Include="NicolasDorier.RateLimits" Version="1.2.3" />
     <PackageReference Include="Serilog" Version="2.9.0" />
     <PackageReference Include="Serilog.AspNetCore" Version="3.2.0" />

BTCPayServer/Plugins/PluginManager.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Specialized;
 using System.Globalization;
 using System.IO;
 using System.IO.Compression;
@@ -22,10 +23,7 @@ namespace BTCPayServer.Plugins
     {
         public const string BTCPayPluginSuffix = ".btcpay";
         private static readonly List<Assembly> _pluginAssemblies = new List<Assembly>();
-        private static readonly List<PluginLoader> _plugins = new List<PluginLoader>();
-        private static ILogger _logger;
 
-        private static List<(PluginLoader, Assembly, IFileProvider)> loadedPlugins;
         public static bool IsExceptionByPlugin(Exception exception)
         {
             return _pluginAssemblies.Any(assembly => assembly?.FullName?.Contains(exception.Source!, StringComparison.OrdinalIgnoreCase) is true);
@@ -33,82 +31,81 @@ namespace BTCPayServer.Plugins
         public static IMvcBuilder AddPlugins(this IMvcBuilder mvcBuilder, IServiceCollection serviceCollection,
             IConfiguration config, ILoggerFactory loggerFactory)
         {
-            _logger = loggerFactory.CreateLogger(typeof(PluginManager));
+            var logger = loggerFactory.CreateLogger(typeof(PluginManager));
             var pluginsFolder = new DataDirectories().Configure(config).PluginDir;
             var plugins = new List<IBTCPayServerPlugin>();
+            var loadedPluginIdentifiers = new HashSet<string>();
 
             serviceCollection.Configure<KestrelServerOptions>(options =>
             {
                 options.Limits.MaxRequestBodySize = int.MaxValue; // if don't set default value is: 30 MB
             });
-            _logger.LogInformation($"Loading plugins from {pluginsFolder}");
+            logger.LogInformation($"Loading plugins from {pluginsFolder}");
             Directory.CreateDirectory(pluginsFolder);
             ExecuteCommands(pluginsFolder);
-            loadedPlugins = new List<(PluginLoader, Assembly, IFileProvider)>();
-            var systemPlugins = GetDefaultLoadedPluginAssemblies();
-
-            foreach (Assembly systemExtension in systemPlugins)
-            {
-                var detectedPlugins = GetAllPluginTypesFromAssembly(systemExtension).Select(GetPluginInstanceFromType);
-                if (!detectedPlugins.Any())
-                {
-                    continue;
-
-                }
-
-                detectedPlugins = detectedPlugins.Select(plugin =>
-                {
-                    plugin.SystemPlugin = true;
-                    return plugin;
-                });
-
-                loadedPlugins.Add((null, systemExtension, CreateEmbeddedFileProviderForAssembly(systemExtension)));
-                plugins.AddRange(detectedPlugins);
-            }
-            var orderFilePath = Path.Combine(pluginsFolder, "order");
-
-            var availableDirs = Directory.GetDirectories(pluginsFolder);
-            var orderedDirs = new List<string>();
-            if (File.Exists(orderFilePath))
-            {
-                var order = File.ReadLines(orderFilePath);
-                foreach (var s in order)
-                {
-                    if (availableDirs.Contains(s))
-                    {
-                        orderedDirs.Add(s);
-                    }
-                }
-
-                orderedDirs.AddRange(availableDirs.Where(s => !orderedDirs.Contains(s)));
-            }
-            else
-            {
-                orderedDirs = availableDirs.ToList();
-            }
 
             var disabledPlugins = GetDisabledPlugins(pluginsFolder);
-
+            var systemAssembly = typeof(Program).Assembly;
-            foreach (var dir in orderedDirs)
+            // Load the referenced assembly plugins
-            {
+            // All referenced plugins should have at least one plugin with exact same plugin identifier
-                var pluginName = Path.GetFileName(dir);
+            // as the assembly. Except for the system assembly (btcpayserver assembly) which are fake plugins
-                var pluginFilePath = Path.Combine(dir, pluginName + ".dll");
+            foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
-                if (disabledPlugins.Contains(pluginName))
             {
+                var pluginIdentifier = assembly.GetName().Name;
+                bool isSystemPlugin = assembly == systemAssembly;
+                if (!isSystemPlugin && disabledPlugins.Contains(pluginIdentifier))
                     continue;
+
+                foreach (var plugin in GetPluginInstancesFromAssembly(assembly))
+                {
+                    if (!isSystemPlugin && plugin.Identifier != pluginIdentifier)
+                        continue;
+                    if (!loadedPluginIdentifiers.Add(plugin.Identifier))
+                        continue;
+                    plugins.Add(plugin);
+                    plugin.SystemPlugin = isSystemPlugin;
                 }
+            }
+
+            var pluginsToLoad = new List<(string PluginIdentifier, string PluginFilePath)>();
+
+#if DEBUG
+            // Load from DEBUG_PLUGINS, in an optional appsettings.dev.json
+            var debugPlugins = config["DEBUG_PLUGINS"] ?? "";
+            foreach (var plugin in debugPlugins.Split(';', StringSplitOptions.RemoveEmptyEntries))
+            {
+                // Formatted either as "<PLUGIN_IDENTIFIER>::<PathToDll>" or "<PathToDll>"
+                var idx = plugin.IndexOf("::");
+                if (idx != -1)
+                    pluginsToLoad.Add((plugin[0..idx], plugin[(idx+1)..]));
+                else
+                    pluginsToLoad.Add((Path.GetFileNameWithoutExtension(plugin), plugin));
+            }
+#endif
+
+            // Load from the plugins folder
+            foreach (var directory in Directory.GetDirectories(pluginsFolder))
+            {
+                var pluginIdentifier = Path.GetDirectoryName(directory);
+                var pluginFilePath = Path.Combine(directory, pluginIdentifier + ".dll");
                 if (!File.Exists(pluginFilePath))
-                {
-                    _logger.LogError(
-                        $"Error when loading plugin {pluginName} - {pluginFilePath} does not exist");
                     continue;
+                if (disabledPlugins.Contains(pluginIdentifier))
+                    continue;
+                pluginsToLoad.Add((pluginIdentifier, pluginFilePath));
             }
 
+            ReorderPlugins(pluginsFolder, pluginsToLoad);
+
+            foreach (var toLoad in pluginsToLoad)
+            {
+                if (!loadedPluginIdentifiers.Add(toLoad.PluginIdentifier))
+                    continue;
                 try
                 {
 
                     var plugin = PluginLoader.CreateFromAssemblyFile(
-                        pluginFilePath, // create a plugin from for the .dll file
+                        toLoad.PluginFilePath, // create a plugin from for the .dll file
                         config =>
                         {
 
@@ -116,25 +113,25 @@ namespace BTCPayServer.Plugins
                             config.PreferSharedTypes = true;
                             config.IsUnloadable = true;
                         });
-
-                    mvcBuilder.AddPluginLoader(plugin);
                     var pluginAssembly = plugin.LoadDefaultAssembly();
-                    _pluginAssemblies.Add(pluginAssembly);
+
-                    _plugins.Add(plugin);
+                    var p = GetPluginInstanceFromAssembly(toLoad.PluginIdentifier, pluginAssembly);
-                    var fileProvider = CreateEmbeddedFileProviderForAssembly(pluginAssembly);
+                    if (p == null)
-                    loadedPlugins.Add((plugin, pluginAssembly, fileProvider));
-                    foreach (var p in GetAllPluginTypesFromAssembly(pluginAssembly)
-                        .Select(GetPluginInstanceFromType))
                     {
+                        logger.LogError($"The plugin assembly doesn't contain the plugin {toLoad.PluginIdentifier}");
+                    }
+                    else
+                    {
+                        mvcBuilder.AddPluginLoader(plugin);
+                        _pluginAssemblies.Add(pluginAssembly);
                         p.SystemPlugin = false;
                         plugins.Add(p);
                     }
-
                 }
                 catch (Exception e)
                 {
-                    _logger.LogError(e,
+                    logger.LogError(e,
-                        $"Error when loading plugin {pluginName}");
+                        $"Error when loading plugin {toLoad.PluginIdentifier}");
                 }
             }
 
@@ -142,14 +139,14 @@ namespace BTCPayServer.Plugins
             {
                 try
                 {
-                    _logger.LogInformation(
+                    logger.LogInformation(
                         $"Adding and executing plugin {plugin.Identifier} - {plugin.Version}");
                     plugin.Execute(serviceCollection);
                     serviceCollection.AddSingleton(plugin);
                 }
                 catch (Exception e)
                 {
-                    _logger.LogError(
+                    logger.LogError(
                         $"Error when loading plugin {plugin.Identifier} - {plugin.Version}{Environment.NewLine}{e.Message}");
                 }
             }
@@ -157,42 +154,56 @@ namespace BTCPayServer.Plugins
             return mvcBuilder;
         }
 
+        private static void ReorderPlugins(string pluginsFolder, List<(string PluginIdentifier, string PluginFilePath)> pluginsToLoad)
+        {
+            Dictionary<string, int> ordersByPlugin = new Dictionary<string, int>();
+            var orderFilePath = Path.Combine(pluginsFolder, "order");
+            int order = 0;
+            if (File.Exists(orderFilePath))
+            {
+                foreach (var o in File.ReadLines(orderFilePath))
+                {
+                    if (ordersByPlugin.TryAdd(o, order))
+                        order++;
+                }
+            }
+            foreach (var p in pluginsToLoad)
+            {
+                if (ordersByPlugin.TryAdd(p.PluginIdentifier, order))
+                    order++;
+            }
+            pluginsToLoad.Sort((a,b) => ordersByPlugin[a.PluginIdentifier] - ordersByPlugin[b.PluginIdentifier]);
+        }
+
         public static void UsePlugins(this IApplicationBuilder applicationBuilder)
         {
+            HashSet<Assembly> assemblies = new HashSet<Assembly>();
             foreach (var extension in applicationBuilder.ApplicationServices
                 .GetServices<IBTCPayServerPlugin>())
             {
                 extension.Execute(applicationBuilder,
                     applicationBuilder.ApplicationServices);
+                assemblies.Add(extension.GetType().Assembly);
             }
 
             var webHostEnvironment = applicationBuilder.ApplicationServices.GetService<IWebHostEnvironment>();
             List<IFileProvider> providers = new List<IFileProvider>() { webHostEnvironment.WebRootFileProvider };
-            providers.AddRange(loadedPlugins.Select(tuple => tuple.Item3));
+            providers.AddRange(assemblies.Select(a => new EmbeddedFileProvider(a)));
             webHostEnvironment.WebRootFileProvider = new CompositeFileProvider(providers);
         }
 
-        private static Assembly[] GetDefaultLoadedPluginAssemblies()
+        private static IEnumerable<IBTCPayServerPlugin> GetPluginInstancesFromAssembly(Assembly assembly)
-        {
-            return AppDomain.CurrentDomain.GetAssemblies()
-                .ToArray();
-        }
-
-        private static Type[] GetAllPluginTypesFromAssembly(Assembly assembly)
         {
             return assembly.GetTypes().Where(type =>
                 typeof(IBTCPayServerPlugin).IsAssignableFrom(type) && type != typeof(PluginService.AvailablePlugin) &&
-                !type.IsAbstract).ToArray();
+                !type.IsAbstract).
+                Select(type => (IBTCPayServerPlugin)Activator.CreateInstance(type, Array.Empty<object>()));
         }
-
+        private static IBTCPayServerPlugin GetPluginInstanceFromAssembly(string pluginIdentifier, Assembly assembly)
-        private static IBTCPayServerPlugin GetPluginInstanceFromType(Type type)
         {
-            return (IBTCPayServerPlugin)Activator.CreateInstance(type, Array.Empty<object>());
+            return GetPluginInstancesFromAssembly(assembly)
-        }
+                .Where(plugin => plugin.Identifier == pluginIdentifier)
-
+                .FirstOrDefault();
-        private static IFileProvider CreateEmbeddedFileProviderForAssembly(Assembly assembly)
-        {
-            return new EmbeddedFileProvider(assembly);
         }
 
         private static void ExecuteCommands(string pluginsFolder)
@@ -318,20 +329,15 @@ namespace BTCPayServer.Plugins
             QueueCommands(pluginDir, ("disable", plugin));
         }
 
-        public static void Unload()
+        public static HashSet<string> GetDisabledPlugins(string pluginsFolder)
-        {
-            _plugins.ForEach(loader => loader.Dispose());
-        }
-
-        public static string[] GetDisabledPlugins(string pluginsFolder)
         {
             var disabledFilePath = Path.Combine(pluginsFolder, "disabled");
             if (File.Exists(disabledFilePath))
             {
-                return File.ReadLines(disabledFilePath).ToArray();
+                return File.ReadLines(disabledFilePath).ToHashSet();
             }
 
-            return Array.Empty<string>();
+            return new HashSet<string>();
         }
     }
 }

BTCPayServer/Plugins/PluginService.cs

@@ -142,7 +142,7 @@ namespace BTCPayServer.Plugins
 
         public string[] GetDisabledPlugins()
         {
-            return PluginManager.GetDisabledPlugins(_dataDirectories.Value.PluginDir);
+            return PluginManager.GetDisabledPlugins(_dataDirectories.Value.PluginDir).ToArray();
         }
     }
 }

BTCPayServer/Program.cs

@@ -35,7 +35,11 @@ namespace BTCPayServer
             IConfiguration conf = null;
             try
             {
-                conf = new DefaultConfiguration() { Logger = logger }.CreateConfiguration(args);
+                var confBuilder = new DefaultConfiguration() { Logger = logger }.CreateConfigurationBuilder(args);
+#if DEBUG
+                confBuilder.AddJsonFile("appsettings.dev.json", true, false);
+#endif
+                conf = confBuilder.Build();
                 if (conf == null)
                     return;