Skip to content

Commit

Permalink
3.2.0 (#2682)
Browse files Browse the repository at this point in the history 
* Add ToJson overload (#2671)

* Add ToJson overload

* change

* Update src/neo/VM/Helper.cs

* Update src/neo/VM/Helper.cs

* Update src/neo/VM/Helper.cs

* Update src/neo/VM/Helper.cs

Co-authored-by: Jinghui Liao <[email protected]>

* Update src/neo/VM/Helper.cs

Co-authored-by: Jinghui Liao <[email protected]>

Co-authored-by: Shargon <[email protected]>
Co-authored-by: Jinghui Liao <[email protected]>

* Fix oom (#2665)

* Fix oom

* Revert reorder

* parameters order

Co-authored-by: Erik Zhang <[email protected]>

* Optimize inventory (#2659)

* add `murmur32` to crypto lib (#2604)

* 3.2.0

* fix

Co-authored-by: Shargon <[email protected]>
Co-authored-by: Jinghui Liao <[email protected]>
  • Loading branch information
@erikzhang @shargon @Jim8y
3 people committed Apr 6, 2022
1 parent 2a64c1c commit 41055be
Show file tree
Hide file tree
Showing 8 changed files with 152 additions and 68 deletions.
58 changes: 32 additions & 26 deletions src/neo/IO/Json/JPathToken.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,67 +122,68 @@ private static JPathToken DequeueToken(Queue<JPathToken> tokens)
public static void ProcessJsonPath(ref JObject[] objects, Queue<JPathToken> tokens)
{
int maxDepth = 6;
int maxObjects = 1024;
while (tokens.Count > 0)
{
JPathToken token = DequeueToken(tokens);
switch (token.Type)
{
case JPathTokenType.Dot:
ProcessDot(ref objects, ref maxDepth, tokens);
ProcessDot(ref objects, ref maxDepth, maxObjects, tokens);
break;
case JPathTokenType.LeftBracket:
ProcessBracket(ref objects, ref maxDepth, tokens);
ProcessBracket(ref objects, ref maxDepth, maxObjects, tokens);
break;
default:
throw new FormatException();
}
}
}

private static void ProcessDot(ref JObject[] objects, ref int maxDepth, Queue<JPathToken> tokens)
private static void ProcessDot(ref JObject[] objects, ref int maxDepth, int maxObjects, Queue<JPathToken> tokens)
{
JPathToken token = DequeueToken(tokens);
switch (token.Type)
{
case JPathTokenType.Asterisk:
Descent(ref objects, ref maxDepth);
Descent(ref objects, ref maxDepth, maxObjects);
break;
case JPathTokenType.Dot:
ProcessRecursiveDescent(ref objects, ref maxDepth, tokens);
ProcessRecursiveDescent(ref objects, ref maxDepth, maxObjects, tokens);
break;
case JPathTokenType.Identifier:
Descent(ref objects, ref maxDepth, token.Content);
Descent(ref objects, ref maxDepth, maxObjects, token.Content);
break;
default:
throw new FormatException();
}
}

private static void ProcessBracket(ref JObject[] objects, ref int maxDepth, Queue<JPathToken> tokens)
private static void ProcessBracket(ref JObject[] objects, ref int maxDepth, int maxObjects, Queue<JPathToken> tokens)
{
JPathToken token = DequeueToken(tokens);
switch (token.Type)
{
case JPathTokenType.Asterisk:
if (DequeueToken(tokens).Type != JPathTokenType.RightBracket)
throw new FormatException();
Descent(ref objects, ref maxDepth);
Descent(ref objects, ref maxDepth, maxObjects);
break;
case JPathTokenType.Colon:
ProcessSlice(ref objects, ref maxDepth, tokens, 0);
ProcessSlice(ref objects, ref maxDepth, maxObjects, tokens, 0);
break;
case JPathTokenType.Number:
JPathToken next = DequeueToken(tokens);
switch (next.Type)
{
case JPathTokenType.Colon:
ProcessSlice(ref objects, ref maxDepth, tokens, int.Parse(token.Content));
ProcessSlice(ref objects, ref maxDepth, maxObjects, tokens, int.Parse(token.Content));
break;
case JPathTokenType.Comma:
ProcessUnion(ref objects, ref maxDepth, tokens, token);
ProcessUnion(ref objects, ref maxDepth, maxObjects, tokens, token);
break;
case JPathTokenType.RightBracket:
Descent(ref objects, ref maxDepth, int.Parse(token.Content));
Descent(ref objects, ref maxDepth, maxObjects, int.Parse(token.Content));
break;
default:
throw new FormatException();
Expand All @@ -193,10 +194,10 @@ private static void ProcessBracket(ref JObject[] objects, ref int maxDepth, Queu
switch (next.Type)
{
case JPathTokenType.Comma:
ProcessUnion(ref objects, ref maxDepth, tokens, token);
ProcessUnion(ref objects, ref maxDepth, maxObjects, tokens, token);
break;
case JPathTokenType.RightBracket:
Descent(ref objects, ref maxDepth, JObject.Parse($"\"{token.Content.Trim('\'')}\"").GetString());
Descent(ref objects, ref maxDepth, maxObjects, JObject.Parse($"\"{token.Content.Trim('\'')}\"").GetString());
break;
default:
throw new FormatException();
Expand All @@ -207,38 +208,39 @@ private static void ProcessBracket(ref JObject[] objects, ref int maxDepth, Queu
}
}

private static void ProcessRecursiveDescent(ref JObject[] objects, ref int maxDepth, Queue<JPathToken> tokens)
private static void ProcessRecursiveDescent(ref JObject[] objects, ref int maxDepth, int maxObjects, Queue<JPathToken> tokens)
{
List<JObject> results = new();
JPathToken token = DequeueToken(tokens);
if (token.Type != JPathTokenType.Identifier) throw new FormatException();
while (objects.Length > 0)
{
results.AddRange(objects.Where(p => p is not null).SelectMany(p => p.Properties).Where(p => p.Key == token.Content).Select(p => p.Value));
Descent(ref objects, ref maxDepth);
Descent(ref objects, ref maxDepth, maxObjects);
if (results.Count > maxObjects) throw new InvalidOperationException(nameof(maxObjects));
}
objects = results.ToArray();
}

private static void ProcessSlice(ref JObject[] objects, ref int maxDepth, Queue<JPathToken> tokens, int start)
private static void ProcessSlice(ref JObject[] objects, ref int maxDepth, int maxObjects, Queue<JPathToken> tokens, int start)
{
JPathToken token = DequeueToken(tokens);
switch (token.Type)
{
case JPathTokenType.Number:
if (DequeueToken(tokens).Type != JPathTokenType.RightBracket)
throw new FormatException();
DescentRange(ref objects, ref maxDepth, start, int.Parse(token.Content));
DescentRange(ref objects, ref maxDepth, maxObjects, start, int.Parse(token.Content));
break;
case JPathTokenType.RightBracket:
DescentRange(ref objects, ref maxDepth, start, 0);
DescentRange(ref objects, ref maxDepth, maxObjects, start, 0);
break;
default:
throw new FormatException();
}
}

private static void ProcessUnion(ref JObject[] objects, ref int maxDepth, Queue<JPathToken> tokens, JPathToken first)
private static void ProcessUnion(ref JObject[] objects, ref int maxDepth, int maxObjects, Queue<JPathToken> tokens, JPathToken first)
{
List<JPathToken> items = new() { first };
while (true)
Expand All @@ -255,24 +257,25 @@ private static void ProcessUnion(ref JObject[] objects, ref int maxDepth, Queue<
switch (first.Type)
{
case JPathTokenType.Number:
Descent(ref objects, ref maxDepth, items.Select(p => int.Parse(p.Content)).ToArray());
Descent(ref objects, ref maxDepth, maxObjects, items.Select(p => int.Parse(p.Content)).ToArray());
break;
case JPathTokenType.String:
Descent(ref objects, ref maxDepth, items.Select(p => JObject.Parse($"\"{p.Content.Trim('\'')}\"").GetString()).ToArray());
Descent(ref objects, ref maxDepth, maxObjects, items.Select(p => JObject.Parse($"\"{p.Content.Trim('\'')}\"").GetString()).ToArray());
break;
default:
throw new FormatException();
}
}

private static void Descent(ref JObject[] objects, ref int maxDepth)
private static void Descent(ref JObject[] objects, ref int maxDepth, int maxObjects)
{
if (maxDepth <= 0) throw new InvalidOperationException();
--maxDepth;
objects = objects.Where(p => p is not null).SelectMany(p => p is JArray array ? array : p.Properties.Values).ToArray();
if (objects.Length > maxObjects) throw new InvalidOperationException(nameof(maxObjects));
}

private static void Descent(ref JObject[] objects, ref int maxDepth, params string[] names)
private static void Descent(ref JObject[] objects, ref int maxDepth, int maxObjects, params string[] names)
{
static IEnumerable<JObject> GetProperties(JObject obj, string[] names)
{
Expand All @@ -283,9 +286,10 @@ static IEnumerable<JObject> GetProperties(JObject obj, string[] names)
if (maxDepth <= 0) throw new InvalidOperationException();
--maxDepth;
objects = objects.SelectMany(p => GetProperties(p, names)).ToArray();
if (objects.Length > maxObjects) throw new InvalidOperationException(nameof(maxObjects));
}

private static void Descent(ref JObject[] objects, ref int maxDepth, params int[] indexes)
private static void Descent(ref JObject[] objects, ref int maxDepth, int maxObjects, params int[] indexes)
{
static IEnumerable<JObject> GetElements(JArray array, int[] indexes)
{
Expand All @@ -299,9 +303,10 @@ static IEnumerable<JObject> GetElements(JArray array, int[] indexes)
if (maxDepth <= 0) throw new InvalidOperationException();
--maxDepth;
objects = objects.OfType<JArray>().SelectMany(p => GetElements(p, indexes)).ToArray();
if (objects.Length > maxObjects) throw new InvalidOperationException(nameof(maxObjects));
}

private static void DescentRange(ref JObject[] objects, ref int maxDepth, int start, int end)
private static void DescentRange(ref JObject[] objects, ref int maxDepth, int maxObjects, int start, int end)
{
if (maxDepth <= 0) throw new InvalidOperationException();
--maxDepth;
Expand All @@ -313,6 +318,7 @@ private static void DescentRange(ref JObject[] objects, ref int maxDepth, int st
int count = iEnd - iStart;
return p.Skip(iStart).Take(count);
}).ToArray();
if (objects.Length > maxObjects) throw new InvalidOperationException(nameof(maxObjects));
}
}
}
2 changes: 1 addition & 1 deletion src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ private void OnHeadersMessageReceived(HeadersPayload payload)

private void OnInventoryReceived(IInventory inventory)
{
knownHashes.Add(inventory.Hash);
if (!knownHashes.Add(inventory.Hash)) return;
pendingKnownHashes.Remove(inventory.Hash);
system.TaskManager.Tell(inventory);
switch (inventory)
Expand Down
13 changes: 13 additions & 0 deletions src/neo/SmartContract/Native/CryptoLib.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,19 @@ public static byte[] Sha256(byte[] data)
return data.Sha256();
}

/// <summary>
/// Computes the hash value for the specified byte array using the murmur32 algorithm.
/// </summary>
/// <param name="data">The input to compute the hash code for.</param>
/// <param name="seed">The seed of the murmur32 hash function</param>
/// <returns>The computed hash code.</returns>
[ContractMethod(CpuFee = 1 << 13)]
public static byte[] Murmur32(byte[] data, uint seed)
{
using Murmur32 murmur = new(seed);
return murmur.ComputeHash(data);
}

/// <summary>
/// Verifies that a digital signature is appropriate for the provided key and message using the ECDSA algorithm.
/// </summary>
Expand Down
109 changes: 83 additions & 26 deletions src/neo/VM/Helper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,48 +278,105 @@ public static byte[] MakeScript(this UInt160 scriptHash, string method, params o
/// Converts the <see cref="StackItem"/> to a JSON object.
/// </summary>
/// <param name="item">The <see cref="StackItem"/> to convert.</param>
/// <param name="maxSize">The maximum size in bytes of the result.</param>
/// <returns>The <see cref="StackItem"/> represented by a JSON object.</returns>
public static JObject ToJson(this StackItem item)
public static JObject ToJson(this StackItem item, int maxSize = int.MaxValue)
{
return ToJson(item, null);
return ToJson(item, null, ref maxSize);
}

private static JObject ToJson(StackItem item, HashSet<StackItem> context)
/// <summary>
/// Converts the <see cref="EvaluationStack"/> to a JSON object.
/// </summary>
/// <param name="stack">The <see cref="EvaluationStack"/> to convert.</param>
/// <param name="maxSize">The maximum size in bytes of the result.</param>
/// <returns>The <see cref="EvaluationStack"/> represented by a JSON object.</returns>
public static JArray ToJson(this EvaluationStack stack, int maxSize = int.MaxValue)
{
if (maxSize <= 0) throw new ArgumentOutOfRangeException(nameof(maxSize));
maxSize -= 2/*[]*/+ Math.Max(0, (stack.Count - 1))/*,*/;
JArray result = new();
foreach (var item in stack)
result.Add(ToJson(item, null, ref maxSize));
if (maxSize < 0) throw new InvalidOperationException("Max size reached.");
return result;
}

private static JObject ToJson(StackItem item, HashSet<StackItem> context, ref int maxSize)
{
JObject json = new();
json["type"] = item.Type;
JObject json = new()
{
["type"] = item.Type
};
JObject value = null;
maxSize -= 11/*{"type":""}*/+ item.Type.ToString().Length;
switch (item)
{
case Array array:
context ??= new HashSet<StackItem>(ReferenceEqualityComparer.Instance);
if (!context.Add(array)) throw new InvalidOperationException();
json["value"] = new JArray(array.Select(p => ToJson(p, context)));
break;
{
context ??= new HashSet<StackItem>(ReferenceEqualityComparer.Instance);
if (!context.Add(array)) throw new InvalidOperationException();
maxSize -= 2/*[]*/+ Math.Max(0, (array.Count - 1))/*,*/;
JArray a = new();
foreach (StackItem stackItem in array)
a.Add(ToJson(stackItem, context, ref maxSize));
value = a;
break;
}
case Boolean boolean:
json["value"] = boolean.GetBoolean();
break;
{
bool b = boolean.GetBoolean();
maxSize -= b ? 4/*true*/: 5/*false*/;
value = b;
break;
}
case Buffer _:
case ByteString _:
json["value"] = Convert.ToBase64String(item.GetSpan());
break;
{
string s = Convert.ToBase64String(item.GetSpan());
maxSize -= 2/*""*/+ s.Length;
value = s;
break;
}
case Integer integer:
json["value"] = integer.GetInteger().ToString();
break;
{
string s = integer.GetInteger().ToString();
maxSize -= 2/*""*/+ s.Length;
value = s;
break;
}
case Map map:
context ??= new HashSet<StackItem>(ReferenceEqualityComparer.Instance);
if (!context.Add(map)) throw new InvalidOperationException();
json["value"] = new JArray(map.Select(p =>
{
JObject item = new();
item["key"] = ToJson(p.Key, context);
item["value"] = ToJson(p.Value, context);
return item;
}));
break;
context ??= new HashSet<StackItem>(ReferenceEqualityComparer.Instance);
if (!context.Add(map)) throw new InvalidOperationException();
maxSize -= 2/*[]*/+ Math.Max(0, (map.Count - 1))/*,*/;
JArray a = new();
foreach (var (k, v) in map)
{
maxSize -= 17/*{"key":,"value":}*/;
JObject i = new()
{
["key"] = ToJson(k, context, ref maxSize),
["value"] = ToJson(v, context, ref maxSize)
};
a.Add(i);
}
value = a;
break;
}
case Pointer pointer:
json["value"] = pointer.Position;
break;
{
maxSize -= pointer.Position.ToString().Length;
value = pointer.Position;
break;
}
}
if (value is not null)
{
maxSize -= 9/*,"value":*/;
json["value"] = value;
}
if (maxSize < 0) throw new InvalidOperationException("Max size reached.");
return json;
}

Expand Down
14 changes: 7 additions & 7 deletions src/neo/neo.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<Copyright>2015-2021 The Neo Project</Copyright>
<Copyright>2015-2022 The Neo Project</Copyright>
<AssemblyTitle>Neo</AssemblyTitle>
<VersionPrefix>3.1.0</VersionPrefix>
<VersionPrefix>3.2.0</VersionPrefix>
<Authors>The Neo Project</Authors>
<TargetFramework>net6.0</TargetFramework>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
Expand All @@ -25,11 +25,11 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Akka" Version="1.4.28" />
<PackageReference Include="BouncyCastle.NetCore" Version="1.8.8" />
<PackageReference Include="K4os.Compression.LZ4" Version="1.2.15" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.0" />
<PackageReference Include="Neo.VM" Version="3.1.0" />
<PackageReference Include="Akka" Version="1.4.35" />
<PackageReference Include="BouncyCastle.NetCore" Version="1.8.10" />
<PackageReference Include="K4os.Compression.LZ4" Version="1.2.16" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.3" />
<PackageReference Include="Neo.VM" Version="3.2.0" />
</ItemGroup>

</Project>
Loading

0 comments on commit 41055be

Please sign in to comment.