Plugin unhandled exception (#3349)

* [Neo Core] Part 1. Isolate Plugins Exceptions from the Node. (#3309) * catch plugin exceptions. * add UT test * udpate format * make the test more complete * complete the ut test * format * complete UT tests with NonPlugin case * async invoke * Update src/Neo/Ledger/Blockchain.cs Co-authored-by: Christopher Schuchardt <[email protected]> --------- Co-authored-by: Christopher Schuchardt <[email protected]> * [Neo Plugin New feature] UnhandledExceptionPolicy on Plugin Unhandled Exception (#3311) * catch plugin exceptions. * add UT test * udpate format * make the test more complete * complete the ut test * format * complete UT tests with NonPlugin case * async invoke * stop plugin on exception * remove watcher from blockchain if uint test is done to avoid cross test data pollution. * add missing file * 3 different policy on handling plugin exception * add missing file * fix null warning * format * Apply suggestions from code review Clean * Update src/Neo/Plugins/PluginSettings.cs Co-authored-by: Shargon <[email protected]> * Update src/Neo/Plugins/PluginSettings.cs Co-authored-by: Christopher Schuchardt <[email protected]> * Update src/Plugins/TokensTracker/TokensTracker.cs Co-authored-by: Christopher Schuchardt <[email protected]> * Update src/Plugins/TokensTracker/TokensTracker.json --------- Co-authored-by: Shargon <[email protected]> Co-authored-by: Christopher Schuchardt <[email protected]> * make the exception message clear --------- Co-authored-by: Christopher Schuchardt <[email protected]> Co-authored-by: Shargon <[email protected]> Co-authored-by: NGD Admin <[email protected]>
neo-project · Jun 21, 2024 · b2f060f · b2f060f
1 parent f379dab
commit b2f060f
Show file tree

Hide file tree

Showing 26 changed files with 417 additions and 29 deletions.
diff --git a/src/Neo/Ledger/Blockchain.cs b/src/Neo/Ledger/Blockchain.cs
@@ -12,18 +12,22 @@
 using Akka.Actor;
 using Akka.Configuration;
 using Akka.IO;
+using Akka.Util.Internal;
 using Neo.IO.Actors;
 using Neo.Network.P2P;
 using Neo.Network.P2P.Payloads;
 using Neo.Persistence;
+using Neo.Plugins;
 using Neo.SmartContract;
 using Neo.SmartContract.Native;
 using Neo.VM;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace Neo.Ledger
 {
@@ -468,10 +472,10 @@ private void Persist(Block block)
                     Context.System.EventStream.Publish(application_executed);
                     all_application_executed.Add(application_executed);
                 }
-                Committing?.Invoke(system, block, snapshot, all_application_executed);
+                _ = InvokeCommittingAsync(system, block, snapshot, all_application_executed);
                 snapshot.Commit();
             }
-            Committed?.Invoke(system, block);
+            _ = InvokeCommittedAsync(system, block);
             system.MemPool.UpdatePoolForBlockPersisted(block, system.StoreView);
             extensibleWitnessWhiteList = null;
             block_cache.Remove(block.PrevHash);
@@ -480,6 +484,64 @@ private void Persist(Block block)
                 Debug.Assert(header.Index == block.Index);
         }
 
+        internal static async Task InvokeCommittingAsync(NeoSystem system, Block block, DataCache snapshot, IReadOnlyList<ApplicationExecuted> applicationExecutedList)
+        {
+            await InvokeHandlersAsync(Committing?.GetInvocationList(), h => ((CommittingHandler)h)(system, block, snapshot, applicationExecutedList));
+        }
+
+        internal static async Task InvokeCommittedAsync(NeoSystem system, Block block)
+        {
+            await InvokeHandlersAsync(Committed?.GetInvocationList(), h => ((CommittedHandler)h)(system, block));
+        }
+
+        private static async Task InvokeHandlersAsync(Delegate[] handlers, Action<Delegate> handlerAction)
+        {
+            if (handlers == null) return;
+
+            var exceptions = new ConcurrentBag<Exception>();
+            var tasks = handlers.Select(handler => Task.Run(() =>
+            {
+                try
+                {
+                    // skip stopped plugin.
+                    if (handler.Target is Plugin { IsStopped: true })
+                    {
+                        return;
+                    }
+
+                    handlerAction(handler);
+                }
+                catch (Exception ex) when (handler.Target is Plugin plugin)
+                {
+                    switch (plugin.ExceptionPolicy)
+                    {
+                        case UnhandledExceptionPolicy.StopNode:
+                            exceptions.Add(ex);
+                            throw;
+                        case UnhandledExceptionPolicy.StopPlugin:
+                            //Stop plugin on exception
+                            plugin.IsStopped = true;
+                            break;
+                        case UnhandledExceptionPolicy.Ignore:
+                            // Log the exception and continue with the next handler
+                            break;
+                        default:
+                            throw new InvalidCastException($"The exception policy {plugin.ExceptionPolicy} is not valid.");
+                    }
+
+                    Utility.Log(nameof(plugin), LogLevel.Error, ex);
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                }
+            })).ToList();
+
+            await Task.WhenAll(tasks);
+
+            exceptions.ForEach(e => throw e);
+        }
+
         /// <summary>
         /// Gets a <see cref="Akka.Actor.Props"/> object used for creating the <see cref="Blockchain"/> actor.
         /// </summary>

diff --git a/src/Neo/Plugins/Plugin.cs b/src/Neo/Plugins/Plugin.cs
@@ -33,7 +33,8 @@ public abstract class Plugin : IDisposable
         /// <summary>
         /// The directory containing the plugin folders. Files can be contained in any subdirectory.
         /// </summary>
-        public static readonly string PluginsDirectory = Combine(GetDirectoryName(System.AppContext.BaseDirectory), "Plugins");
+        public static readonly string PluginsDirectory =
+            Combine(GetDirectoryName(System.AppContext.BaseDirectory), "Plugins");
 
         private static readonly FileSystemWatcher configWatcher;
 
@@ -67,14 +68,27 @@ public abstract class Plugin : IDisposable
         /// </summary>
         public virtual Version Version => GetType().Assembly.GetName().Version;
 
+        /// <summary>
+        /// If the plugin should be stopped when an exception is thrown.
+        /// Default is <see langword="true"/>.
+        /// </summary>
+        protected internal virtual UnhandledExceptionPolicy ExceptionPolicy { get; init; } = UnhandledExceptionPolicy.StopNode;
+
+        /// <summary>
+        /// The plugin will be stopped if an exception is thrown.
+        /// But it also depends on <see cref="UnhandledExceptionPolicy"/>.
+        /// </summary>
+        internal bool IsStopped { get; set; }
+
         static Plugin()
         {
             if (!Directory.Exists(PluginsDirectory)) return;
             configWatcher = new FileSystemWatcher(PluginsDirectory)
             {
                 EnableRaisingEvents = true,
                 IncludeSubdirectories = true,
-                NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.CreationTime | NotifyFilters.LastWrite | NotifyFilters.Size,
+                NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.CreationTime |
+                               NotifyFilters.LastWrite | NotifyFilters.Size,
             };
             configWatcher.Changed += ConfigWatcher_Changed;
             configWatcher.Created += ConfigWatcher_Changed;
@@ -106,7 +120,8 @@ private static void ConfigWatcher_Changed(object sender, FileSystemEventArgs e)
             {
                 case ".json":
                 case ".dll":
-                    Utility.Log(nameof(Plugin), LogLevel.Warning, $"File {e.Name} is {e.ChangeType}, please restart node.");
+                    Utility.Log(nameof(Plugin), LogLevel.Warning,
+                        $"File {e.Name} is {e.ChangeType}, please restart node.");
                     break;
             }
         }
@@ -119,7 +134,8 @@ private static Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEven
             AssemblyName an = new(args.Name);
 
             Assembly assembly = AppDomain.CurrentDomain.GetAssemblies().FirstOrDefault(a => a.FullName == args.Name) ??
-                                AppDomain.CurrentDomain.GetAssemblies().FirstOrDefault(a => a.GetName().Name == an.Name);
+                                AppDomain.CurrentDomain.GetAssemblies()
+                                    .FirstOrDefault(a => a.GetName().Name == an.Name);
             if (assembly != null) return assembly;
 
             string filename = an.Name + ".dll";
@@ -150,7 +166,8 @@ public virtual void Dispose()
         /// <returns>The content of the configuration file read.</returns>
         protected IConfigurationSection GetConfiguration()
         {
-            return new ConfigurationBuilder().AddJsonFile(ConfigFile, optional: true).Build().GetSection("PluginConfiguration");
+            return new ConfigurationBuilder().AddJsonFile(ConfigFile, optional: true).Build()
+                .GetSection("PluginConfiguration");
         }
 
         private static void LoadPlugin(Assembly assembly)
@@ -187,6 +204,7 @@ internal static void LoadPlugins()
                     catch { }
                 }
             }
+
             foreach (Assembly assembly in assemblies)
             {
                 LoadPlugin(assembly);
@@ -229,7 +247,33 @@ protected internal virtual void OnSystemLoaded(NeoSystem system)
         /// <returns><see langword="true"/> if the <paramref name="message"/> is handled by a plugin; otherwise, <see langword="false"/>.</returns>
         public static bool SendMessage(object message)
         {
-            return Plugins.Any(plugin => plugin.OnMessage(message));
+
+            return Plugins.Any(plugin =>
+                {
+                    try
+                    {
+                        return !plugin.IsStopped &&
+                               plugin.OnMessage(message);
+                    }
+                    catch (Exception ex)
+                    {
+                        switch (plugin.ExceptionPolicy)
+                        {
+                            case UnhandledExceptionPolicy.StopNode:
+                                throw;
+                            case UnhandledExceptionPolicy.StopPlugin:
+                                plugin.IsStopped = true;
+                                break;
+                            case UnhandledExceptionPolicy.Ignore:
+                                break;
+                            default:
+                                throw new InvalidCastException($"The exception policy {plugin.ExceptionPolicy} is not valid.");
+                        }
+                        Utility.Log(nameof(Plugin), LogLevel.Error, ex);
+                        return false;
+                    }
+                }
+            );
         }
     }
 }
diff --git a/src/Neo/Plugins/PluginSettings.cs b/src/Neo/Plugins/PluginSettings.cs
@@ -0,0 +1,33 @@
+// Copyright (C) 2015-2024 The Neo Project.
+//
+// PluginSettings.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using Microsoft.Extensions.Configuration;
+using Org.BouncyCastle.Security;
+using System;
+
+namespace Neo.Plugins;
+
+public abstract class PluginSettings(IConfigurationSection section)
+{
+    public UnhandledExceptionPolicy ExceptionPolicy
+    {
+        get
+        {
+            var policyString = section.GetValue(nameof(UnhandledExceptionPolicy), nameof(UnhandledExceptionPolicy.StopNode));
+            if (Enum.TryParse(policyString, out UnhandledExceptionPolicy policy))
+            {
+                return policy;
+            }
+
+            throw new InvalidParameterException($"{policyString} is not a valid UnhandledExceptionPolicy");
+        }
+    }
+}
diff --git a/src/Neo/Plugins/UnhandledExceptionPolicy.cs b/src/Neo/Plugins/UnhandledExceptionPolicy.cs
@@ -0,0 +1,20 @@
+// Copyright (C) 2015-2024 The Neo Project.
+//
+// UnhandledExceptionPolicy.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+namespace Neo.Plugins
+{
+    public enum UnhandledExceptionPolicy
+    {
+        Ignore = 0,
+        StopPlugin = 1,
+        StopNode = 2,
+    }
+}
diff --git a/src/Plugins/ApplicationLogs/ApplicationLogs.json b/src/Plugins/ApplicationLogs/ApplicationLogs.json
@@ -3,7 +3,8 @@
     "Path": "ApplicationLogs_{0}",
     "Network": 860833102,
     "MaxStackSize": 65535,
-    "Debug": false
+    "Debug": false,
+    "UnhandledExceptionPolicy": "StopPlugin"
   },
   "Dependency": [
     "RpcServer"

diff --git a/src/Plugins/ApplicationLogs/LogReader.cs b/src/Plugins/ApplicationLogs/LogReader.cs
@@ -38,6 +38,7 @@ public class LogReader : Plugin, ICommittingHandler, ICommittedHandler, ILogHand
 
         public override string Name => "ApplicationLogs";
         public override string Description => "Synchronizes smart contract VM executions and notifications (NotifyLog) on blockchain.";
+        protected override UnhandledExceptionPolicy ExceptionPolicy => Settings.Default.ExceptionPolicy;
 
         #region Ctor
 

diff --git a/src/Plugins/ApplicationLogs/Settings.cs b/src/Plugins/ApplicationLogs/Settings.cs
@@ -13,7 +13,7 @@
 
 namespace Neo.Plugins.ApplicationLogs
 {
-    internal class Settings
+    internal class Settings : PluginSettings
     {
         public string Path { get; }
         public uint Network { get; }
@@ -23,7 +23,7 @@ internal class Settings
 
         public static Settings Default { get; private set; }
 
-        private Settings(IConfigurationSection section)
+        private Settings(IConfigurationSection section) : base(section)
         {
             Path = section.GetValue("Path", "ApplicationLogs_{0}");
             Network = section.GetValue("Network", 5195086u);

diff --git a/src/Plugins/DBFTPlugin/DBFTPlugin.cs b/src/Plugins/DBFTPlugin/DBFTPlugin.cs
@@ -31,6 +31,8 @@ public class DBFTPlugin : Plugin, IServiceAddedHandler, IMessageReceivedHandler,
 
         public override string ConfigFile => System.IO.Path.Combine(RootPath, "DBFTPlugin.json");
 
+        protected override UnhandledExceptionPolicy ExceptionPolicy => settings.ExceptionPolicy;
+
         public DBFTPlugin()
         {
             RemoteNode.MessageReceived += ((IMessageReceivedHandler)this).RemoteNode_MessageReceived_Handler;

diff --git a/src/Plugins/DBFTPlugin/DBFTPlugin.json b/src/Plugins/DBFTPlugin/DBFTPlugin.json
@@ -5,6 +5,7 @@
     "AutoStart": false,
     "Network": 860833102,
     "MaxBlockSize": 2097152,
-    "MaxBlockSystemFee": 150000000000
+    "MaxBlockSystemFee": 150000000000,
+    "UnhandledExceptionPolicy": "StopNode"
   }
 }
diff --git a/src/Plugins/DBFTPlugin/Settings.cs b/src/Plugins/DBFTPlugin/Settings.cs
@@ -13,7 +13,7 @@
 
 namespace Neo.Plugins.DBFTPlugin
 {
-    public class Settings
+    public class Settings : PluginSettings
     {
         public string RecoveryLogs { get; }
         public bool IgnoreRecoveryLogs { get; }
@@ -22,7 +22,7 @@ public class Settings
         public uint MaxBlockSize { get; }
         public long MaxBlockSystemFee { get; }
 
-        public Settings(IConfigurationSection section)
+        public Settings(IConfigurationSection section) : base(section)
         {
             RecoveryLogs = section.GetValue("RecoveryLogs", "ConsensusState");
             IgnoreRecoveryLogs = section.GetValue("IgnoreRecoveryLogs", false);

diff --git a/src/Plugins/OracleService/OracleService.cs b/src/Plugins/OracleService/OracleService.cs
@@ -62,6 +62,8 @@ public class OracleService : Plugin, ICommittingHandler, IServiceAddedHandler, I
 
         public override string Description => "Built-in oracle plugin";
 
+        protected override UnhandledExceptionPolicy ExceptionPolicy => Settings.Default.ExceptionPolicy;
+
         public override string ConfigFile => System.IO.Path.Combine(RootPath, "OracleService.json");
 
         public OracleService()

diff --git a/src/Plugins/OracleService/OracleService.json b/src/Plugins/OracleService/OracleService.json
@@ -6,6 +6,7 @@
     "MaxOracleTimeout": 10000,
     "AllowPrivateHost": false,
     "AllowedContentTypes": [ "application/json" ],
+    "UnhandledExceptionPolicy": "Ignore",
     "Https": {
       "Timeout": 5000
     },

diff --git a/src/Plugins/OracleService/Settings.cs b/src/Plugins/OracleService/Settings.cs
@@ -37,7 +37,7 @@ public NeoFSSettings(IConfigurationSection section)
         }
     }
 
-    class Settings
+    class Settings : PluginSettings
     {
         public uint Network { get; }
         public Uri[] Nodes { get; }
@@ -51,7 +51,7 @@ class Settings
 
         public static Settings Default { get; private set; }
 
-        private Settings(IConfigurationSection section)
+        private Settings(IConfigurationSection section) : base(section)
         {
             Network = section.GetValue("Network", 5195086u);
             Nodes = section.GetSection("Nodes").GetChildren().Select(p => new Uri(p.Get<string>(), UriKind.Absolute)).ToArray();

diff --git a/src/Plugins/RpcServer/RpcServer.json b/src/Plugins/RpcServer/RpcServer.json
@@ -1,5 +1,6 @@
 {
   "PluginConfiguration": {
+    "UnhandledExceptionPolicy": "Ignore",
     "Servers": [
       {
         "Network": 860833102,

diff --git a/src/Plugins/RpcServer/RpcServerPlugin.cs b/src/Plugins/RpcServer/RpcServerPlugin.cs
@@ -24,6 +24,7 @@ public class RpcServerPlugin : Plugin
         private static readonly Dictionary<uint, List<object>> handlers = new();
 
         public override string ConfigFile => System.IO.Path.Combine(RootPath, "RpcServer.json");
+        protected override UnhandledExceptionPolicy ExceptionPolicy => settings.ExceptionPolicy;
 
         protected override void Configure()
         {