Add a 'sortLeaves' options

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+
+- Added an option to disable leaf sorting.
+
 ## 1.0.5
 
 - Make `processMultiProof` more robust by validating invariants.

README.md

@@ -130,6 +130,18 @@ bytes32 leaf = keccak256(bytes.concat(keccak256(abi.encode(addr, amount))));
 
 This is an opinionated design that we believe will offer the best out of the box experience for most users. We may introduce options for customization in the future based on user requests.
 
+### Leaf ordering
+
+Each leaf of a merkle tree can be proven individually. The relative ordering of leaves is mostly irrelevant when the only objective is to prove inclusion of individual leaves in the tree. Proving multiple leaves at once is however a little bit more difficult.
+
+This library proposes a mechanism to prove (and verify) that sets of leaves are included in the tree. These "multiproofs" can also be verified onchain using the implementation available in `@openzeppelin/contracts`. This mechanism requires the leaves to be ordered respective to their position in the tree. For example, if the tree leaves are (in hex form) `[ 0xAA...AA, 0xBB...BB, 0xCC...CC, 0xDD...DD]`, then you'd be able to prove `[0xBB...BB, 0xDD...DD]` as a subset of the leaves, but not `[0xDD...DD, 0xBB...BB]`.
+
+Since this library knows the entire tree, you can generate a multiproof with the requested leaves in any order. The library will re-order them so that they appear inside the proof in the correct order. The `MultiProof` object returned by `tree.getMultiProof(...)` will have the leaves ordered according to their position in the tree, and not in the order in which you provided them.
+
+By default, the library orders the leaves according to their hash when building the tree. This is so that a smart contract can build the hashes of a set of leaves and order them correctly without any knowledge of the tree itself. Said differently, it is simpler for a smart contract to process a multiproof for leaves it rebuilt itself if the corresponding tree is ordered.
+
+However, some trees are constructed iteratively, from unsorted data. In that case, the leaves are unlikelly to be sorted. In order for this library to be able to represent such trees, the call to `StandardMerkleTree.of` includes an option to disable sorting. With that option, the leaves are kept in the order in which they were provided. Note that this option has no effect on your ability to generate and verify proofs and multiproofs in JavaScript, but that it may introduce challenges when verifying multiproofs onchain. We recommend only using it for building a representation of trees that are built (onchain) using an iterative process.
+
 ## API & Examples
 
 ### `StandardMerkleTree`
@@ -141,14 +153,17 @@ import { StandardMerkleTree } from "@openzeppelin/merkle-tree";
 ### `StandardMerkleTree.of`
 
 ```typescript
-const tree = StandardMerkleTree.of([[alice, '100'], [bob, '200']], ['address', 'uint']);
+const tree = StandardMerkleTree.of([[alice, '100'], [bob, '200']], ['address', 'uint'], { sortLeaves: true });
 ```
 
 Creates a standard merkle tree out of an array of the elements in the tree, along with their types for ABI encoding. For documentation on the syntax of the types, including how to encode structs, refer to the documentation for Ethers.js's [`AbiCoder`](https://docs.ethers.org/v5/api/utils/abi/coder/#AbiCoder-encode).
 
 > **Note**
 > Consider reading the array of elements from a CSV file for easy interoperability with spreadsheets or other data processing pipelines.
 
+> **Note**
+> By default, leaves are sorted according to their hash. This is done so that multiproof generated by the library can more easily be verified onchain. This can be disabled using the optional third argument. See the [Leaf ordering](#leaf-ordering) section for more details.
+
 ### `StandardMerkleTree.verify`
 
 ```typescript

src/options.ts

@@ -0,0 +1,11 @@
+// MerkleTree building options
+export type MerkleTreeOptions = Partial<{
+  /** Enable or disable sorted leaves. Sorting is strongly recommended for multiproofs. */
+  sortLeaves: boolean;
+}>;
+
+// Recommended (default) options.
+// - leaves are sorted by default to facilitate onchain verification of multiproofs.
+export const defaultOptions: Required<MerkleTreeOptions> = {
+  sortLeaves: true,
+};

src/standard.test.ts

@@ -1,131 +1,149 @@
 import assert from 'assert/strict';
 import { keccak256 } from 'ethereum-cryptography/keccak';
 import { hex } from './bytes';
+import { MerkleTreeOptions } from './options';
 import { StandardMerkleTree } from './standard';
 
 const zeroBytes = new Uint8Array(32);
 const zero = hex(zeroBytes);
 
-const characters = (s: string) => {
+const makeTree = (s: string, opts: MerkleTreeOptions = {}) => {
   const l = s.split('').map(c => [c]);
-  const t = StandardMerkleTree.of(l, ['string']);
+  const t = StandardMerkleTree.of(l, ['string'], opts);
   return { l, t };
 }
 
 describe('standard merkle tree', () => {
-  it('generates valid single proofs for all leaves', () => {
-    const { t } = characters('abcdef');
-    t.validate();
-  });
-
-  it('generates valid single proofs for all leaves', () => {
-    const { t } = characters('abcdef');
-
-    for (const [id, leaf] of t.entries()) {
-      const proof1 = t.getProof(id);
-      const proof2 = t.getProof(leaf);
-
-      assert.deepEqual(proof1, proof2);
-
-      assert(t.verify(id, proof1));
-      assert(t.verify(leaf, proof1));
-      assert(StandardMerkleTree.verify(t.root, ['string'], leaf, proof1));
-    }
-  });
-
-  it('rejects invalid proofs', () => {
-    const { t } = characters('abcdef');
-    const { t: otherTree } = characters('abc');
-
-    const leaf = ['a'];
-    const invalidProof = otherTree.getProof(leaf);
-
-    assert(!t.verify(leaf, invalidProof));
-    assert(!StandardMerkleTree.verify(t.root, ['string'], leaf, invalidProof));
-  });
-
-  it('generates valid multiproofs', () => {
-    const { t, l } = characters('abcdef');
-
-    for (const ids of [[], [0, 1], [0, 1, 5], [1, 3, 4, 5], [0, 2, 4, 5], [0, 1, 2, 3, 4, 5]]) {
-      const proof1 = t.getMultiProof(ids);
-      const proof2 = t.getMultiProof(ids.map(i => l[i]!));
-
-      assert.deepEqual(proof1, proof2);
-
-      assert(t.verifyMultiProof(proof1));
-      assert(StandardMerkleTree.verifyMultiProof(t.root, ['string'], proof1));
-    }
-  });
-
-  it('rejects invalid multiproofs', () => {
-    const { t } = characters('abcdef');
-    const { t: otherTree } = characters('abc');
-
-    const leaves = [['a'], ['b'], ['c']];
-    const multiProof = otherTree.getMultiProof(leaves);
-
-    assert(!t.verifyMultiProof(multiProof));
-    assert(!StandardMerkleTree.verifyMultiProof(t.root, ['string'], multiProof));
-  });
-
-  it('renders tree representation', () => {
-    const { t } = characters('abc');
-
-    const expected = `\
-0) f2129b5a697531ef818f644564a6552b35c549722385bc52aa7fe46c0b5f46b1
-├─ 1) fa914d99a18dc32d9725b3ef1c50426deb40ec8d0885dac8edcc5bfd6d030016
-│  ├─ 3) 9c15a6a0eaeed500fd9eed4cbeab71f797cefcc67bfd46683e4d2e6ff7f06d1c
-│  └─ 4) 19ba6c6333e0e9a15bf67523e0676e2f23eb8e574092552d5e888c64a4bb3681
-└─ 2) 9cf5a63718145ba968a01c1d557020181c5b252f665cf7386d370eddb176517b`;
-
-    assert.equal(t.render(), expected);
-  });
-
-  it('dump and load', () => {
-    const { t } = characters('abcdef');
-    const t2 = StandardMerkleTree.load(t.dump());
-
-    t2.validate();
-    assert.deepEqual(t2, t);
-  });
-
-  it('reject out of bounds value index', () => {
-    const { t } = characters('a');
-    assert.throws(
-      () => t.getProof(1),
-      /^Error: Index out of bounds$/,
-    );
-  });
-
-  it('reject unrecognized tree dump', () => {
-    assert.throws(
-      () => StandardMerkleTree.load({ format: 'nonstandard' } as any),
-      /^Error: Unknown format 'nonstandard'$/,
-    );
-  });
-
-  it('reject malformed tree dump', () => {
-    const t1 = StandardMerkleTree.load({
-      format: 'standard-v1',
-      tree: [zero],
-      values: [{ value: ['0'], treeIndex: 0 }],
-      leafEncoding: ['uint256'],
+  for (const opts of [
+    {},
+    { sortLeaves: true },
+    { sortLeaves: false },
+  ]) {
+    describe(`with options '${JSON.stringify(opts)}'`, () => {
+      const { l: leaves, t: tree } = makeTree('abcdef', opts);
+      const { l: otherLeaves, t: otherTree } = makeTree('abc', opts);
+
+      it('generates valid single proofs for all leaves', () => {
+        tree.validate();
+      });
+
+      it('generates valid single proofs for all leaves', () => {
+        for (const [id, leaf] of tree.entries()) {
+          const proof1 = tree.getProof(id);
+          const proof2 = tree.getProof(leaf);
+
+          assert.deepEqual(proof1, proof2);
+
+          assert(tree.verify(id, proof1));
+          assert(tree.verify(leaf, proof1));
+          assert(StandardMerkleTree.verify(tree.root, ['string'], leaf, proof1));
+        }
+      });
+
+      it('rejects invalid proofs', () => {
+        const leaf = ['a'];
+        const invalidProof = otherTree.getProof(leaf);
+
+        assert(!tree.verify(leaf, invalidProof));
+        assert(!StandardMerkleTree.verify(tree.root, ['string'], leaf, invalidProof));
+      });
+
+      it('generates valid multiproofs', () => {
+        for (const ids of [[], [0, 1], [0, 1, 5], [1, 3, 4, 5], [0, 2, 4, 5], [0, 1, 2, 3, 4, 5], [4, 1, 5, 0, 2]]) {
+          const proof1 = tree.getMultiProof(ids);
+          const proof2 = tree.getMultiProof(ids.map(i => leaves[i]!));
+
+          assert.deepEqual(proof1, proof2);
+
+          assert(tree.verifyMultiProof(proof1));
+          assert(StandardMerkleTree.verifyMultiProof(tree.root, ['string'], proof1));
+        }
+      });
+
+      it('rejects invalid multiproofs', () => {
+        const multiProof = otherTree.getMultiProof([['a'], ['b'], ['c']]);
+
+        assert(!tree.verifyMultiProof(multiProof));
+        assert(!StandardMerkleTree.verifyMultiProof(tree.root, ['string'], multiProof));
+      });
+
+      it('renders tree representation', () => {
+        assert.equal(
+          tree.render(),
+          opts.sortLeaves == false
+            ? [
+                "0) 23be0977360f08bb0bd7f709a7d543d2cd779c79c66d74e0441919871647de2b",
+                "├─ 1) 8f7234e8cfe39c08ca84a3a3e3274f574af26fd15165fe29e09cbab742daccd9",
+                "│  ├─ 3) 03707d7802a71ca56a8ad8028da98c4f1dbec55b31b4a25d536b5309cc20eda9",
+                "│  │  ├─ 7) eba909cf4bb90c6922771d7f126ad0fd11dfde93f3937a196274e1ac20fd2f5b",
+                "│  │  └─ 8) 9cf5a63718145ba968a01c1d557020181c5b252f665cf7386d370eddb176517b",
+                "│  └─ 4) fa914d99a18dc32d9725b3ef1c50426deb40ec8d0885dac8edcc5bfd6d030016",
+                "│     ├─ 9) 19ba6c6333e0e9a15bf67523e0676e2f23eb8e574092552d5e888c64a4bb3681",
+                "│     └─ 10) 9c15a6a0eaeed500fd9eed4cbeab71f797cefcc67bfd46683e4d2e6ff7f06d1c",
+                "└─ 2) 7b0c6cd04b82bfc0e250030a5d2690c52585e0cc6a4f3bc7909d7723b0236ece",
+                "   ├─ 5) c62a8cfa41edc0ef6f6ae27a2985b7d39c7fea770787d7e104696c6e81f64848",
+                "   └─ 6) 9a4f64e953595df82d1b4f570d34c4f4f0cfaf729a61e9d60e83e579e1aa283e",
+              ].join("\n")
+            : [
+                "0) 6deb52b5da8fd108f79fab00341f38d2587896634c646ee52e49f845680a70c8",
+                "├─ 1) 52426e0f1f65ff7e209a13b8c29cffe82e3acaf3dad0a9b9088f3b9a61a929c3",
+                "│  ├─ 3) 8076923e76cf01a7c048400a2304c9a9c23bbbdac3a98ea3946340fdafbba34f",
+                "│  │  ├─ 7) 9cf5a63718145ba968a01c1d557020181c5b252f665cf7386d370eddb176517b",
+                "│  │  └─ 8) 9c15a6a0eaeed500fd9eed4cbeab71f797cefcc67bfd46683e4d2e6ff7f06d1c",
+                "│  └─ 4) 965b92c6cf08303cc4feb7f3e0819c436c2cec17c6f0688a6af139c9a368707c",
+                "│     ├─ 9) 9a4f64e953595df82d1b4f570d34c4f4f0cfaf729a61e9d60e83e579e1aa283e",
+                "│     └─ 10) 19ba6c6333e0e9a15bf67523e0676e2f23eb8e574092552d5e888c64a4bb3681",
+                "└─ 2) fd3cf45654e88d1cc5d663578c82c76f4b5e3826bacaa1216441443504538f51",
+                "   ├─ 5) eba909cf4bb90c6922771d7f126ad0fd11dfde93f3937a196274e1ac20fd2f5b",
+                "   └─ 6) c62a8cfa41edc0ef6f6ae27a2985b7d39c7fea770787d7e104696c6e81f64848",
+              ].join("\n"),
+        );
+      });
+
+      it('dump and load', () => {
+        const recoveredTree = StandardMerkleTree.load(tree.dump());
+
+        recoveredTree.validate();
+        assert.deepEqual(tree, recoveredTree);
+      });
+
+      it('reject out of bounds value index', () => {
+        assert.throws(
+          () => tree.getProof(leaves.length),
+          /^Error: Index out of bounds$/,
+        );
+      });
+
+      it('reject unrecognized tree dump', () => {
+        assert.throws(
+          () => StandardMerkleTree.load({ format: 'nonstandard' } as any),
+          /^Error: Unknown format 'nonstandard'$/,
+        );
+      });
+
+      it('reject malformed tree dump', () => {
+        const loadedTree1 = StandardMerkleTree.load({
+          format: 'standard-v1',
+          tree: [zero],
+          values: [{ value: ['0'], treeIndex: 0 }],
+          leafEncoding: ['uint256'],
+        });
+        assert.throws(
+          () => loadedTree1.getProof(0),
+          /^Error: Merkle tree does not contain the expected value$/,
+        );
+
+        const loadedTree2 = StandardMerkleTree.load({
+          format: 'standard-v1',
+          tree: [zero, zero, hex(keccak256(keccak256(zeroBytes)))],
+          values: [{ value: ['0'], treeIndex: 2 }],
+          leafEncoding: ['uint256'],
+        });
+        assert.throws(
+          () => loadedTree2.getProof(0),
+          /^Error: Unable to prove value$/,
+        );
+      });
     });
-    assert.throws(
-      () => t1.getProof(0),
-      /^Error: Merkle tree does not contain the expected value$/,
-    );
-
-    const t2 = StandardMerkleTree.load({
-      format: 'standard-v1',
-      tree: [zero, zero, hex(keccak256(keccak256(zeroBytes)))],
-      values: [{ value: ['0'], treeIndex: 2 }],
-      leafEncoding: ['uint256'],
-    });
-    assert.throws(
-      () => t2.getProof(0),
-      /^Error: Unable to prove value$/,
-    );
-  });
+  }
 });

src/standard.ts

@@ -1,14 +1,10 @@
-import { keccak256 } from 'ethereum-cryptography/keccak';
 import { equalsBytes, hexToBytes } from 'ethereum-cryptography/utils';
-import { defaultAbiCoder } from '@ethersproject/abi';
 import { Bytes, compareBytes, hex } from './bytes';
 import { getProof, isValidMerkleTree, makeMerkleTree, processProof, renderMerkleTree, MultiProof, getMultiProof, processMultiProof } from './core';
+import { MerkleTreeOptions, defaultOptions } from './options';
 import { checkBounds } from './utils/check-bounds';
 import { throwError } from './utils/throw-error';
-
-function standardLeafHash<T extends any[]>(value: T, types: string[]): Bytes {
-  return keccak256(keccak256(hexToBytes(defaultAbiCoder.encode(types, value))));
-}
+import { standardLeafHash } from './utils/standard-leaf-hash';
 
 interface StandardMerkleTreeData<T extends any[]> {
   format: 'standard-v1';
@@ -35,10 +31,14 @@ export class StandardMerkleTree<T extends any[]> {
       ]));
   }
 
-  static of<T extends any[]>(values: T[], leafEncoding: string[]) {
-    const hashedValues = values
-      .map((value, valueIndex) => ({ value, valueIndex, hash: standardLeafHash(value, leafEncoding) }))
-      .sort((a, b) => compareBytes(a.hash, b.hash));
+  static of<T extends any[]>(values: T[], leafEncoding: string[], options: MerkleTreeOptions = {}) {
+    const sortLeaves = options.sortLeaves ?? defaultOptions.sortLeaves;
+
+    const hashedValues = values.map((value, valueIndex) => ({ value, valueIndex, hash: standardLeafHash(value, leafEncoding) }));
+
+    if (sortLeaves) {
+      hashedValues.sort((a, b) => compareBytes(a.hash, b.hash));
+    }
 
     const tree = makeMerkleTree(hashedValues.map(v => v.hash));
 

src/utils/standard-leaf-hash.ts

@@ -0,0 +1,8 @@
+import { keccak256 } from 'ethereum-cryptography/keccak';
+import { hexToBytes } from 'ethereum-cryptography/utils';
+import { defaultAbiCoder } from '@ethersproject/abi';
+import { Bytes } from '../bytes';
+
+export function standardLeafHash<T extends any[]>(value: T, types: string[]): Bytes {
+  return keccak256(keccak256(hexToBytes(defaultAbiCoder.encode(types, value))));
+}