Add RLP and TrieProof libraries

[ernestognw]

Requires:

• Add Memory utility library #5189
• Add reverseBits operations to Bytes.sol #5724
• Add new clz(bytes) and clz(uint256) functions #5725

PR Checklist

• Tests
• Documentation
• Changeset entry (run npx changeset add)

[changeset-bot]

🦋 Changeset detected

Latest commit: aa26e48

The changes in this PR will be included in the next version bump.

This PR includes changesets to release 1 package

Name	Type
openzeppelin-solidity	Minor

Not sure what this means? Click here to learn what changesets are.

Click here if you're a maintainer who wants to add another changeset to this PR

[ernestognw]

This PR must be updated if any of #5792 or #5726 is merged. Essentially replacing these RLP private functions and string's equal

[0xClandestine]

On my point before I don't think it's necessary to convert the key to nibbles beforehand, makes more sense to extract the Nth nibble here.

[0xClandestine]

The toNibbles function converts a bytes array like 0xABCD into a nibble-expanded format like 0xA00BC00D. The existing implementation does this by iterating linearly over the input, which works correctly but is extremely inefficient.

My approach processes the input in parallel, reducing time complexity from O(n) to O(n / 32). This is inspired by bit-hacking techniques for interleaving bits, as described in the following resources:

https://stackoverflow.com/questions/38881877/bit-hack-expanding-bits
https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN

    // 0xABCD -> 0xA0B0C0D0
    function expandNibbles(uint128 x) internal pure returns (uint256) {
        uint256 y = uint256(x);
        y = (y | (y << 64)) & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF;
        y = (y | (y << 32)) & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF;
        y = (y | (y << 16)) & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF;
        y = (y | (y << 8)) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF;
        y = (y | (y << 4)) & 0x0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F;
        return y;
    }
    
    // Puts nibbles in their respective height (high/low). 
    // 0xA0B0C0D0 -> 0xA00BC00D
    function toNibbles(uint128 x) internal pure returns (uint256) {
        uint256 y = expandNibbles(x);
        uint256 high = y & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF;
        uint256 low = y & 0xFF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00;
        high = high >> 4;
        return high | low;
    }

[0xClandestine]

Bringing it all together:

function toNibbles(bytes memory input) internal pure returns (bytes memory output) {
    /// @solidity memory-safe-assembly
    assembly {
        output := mload(0x40)
        mstore(output, mul(mload(input), 2))
        mstore(0x40, add(output, add(0x20, mul(mload(input), 2))))
        
        for { let i := 0 } lt(i, mload(input)) { i := add(i, 16) } {
            let x := shr(128, mload(add(add(input, 0x20), i)))
            x := and(or(x, shl(64, x)), 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF)
            x := and(or(x, shl(32, x)), 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF)
            x := and(or(x, shl(16, x)), 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF)
            x := and(or(x, shl(8, x)),  0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF)
            let mask := 0x0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F
            x := and(or(x, shl(4, x)),  mask)
            x := or(shl(4, and(shr(4, x), mask)), and(x, mask))
            mstore(add(add(output, 0x20), mul(i, 2)), x)
        }
    }
}

[0xClandestine]

Can probably be simplified further by adjusting the first 4 masks, working on that now.

[0xClandestine]

This can also be much more efficient by comparing 32 bytes at a time. Once a mismatch is found, use xor and clz to count leading matching bytes within the word.

O(n) -> O(n/32)

[0xClandestine]

function _getSharedNibbleLength(bytes memory _a, bytes memory _b) private pure returns (uint256 shared) {
    uint256 minLen = _a.length < _b.length ? _a.length : _b.length;

    unchecked {
        // Compare 32 bytes at a time
        for (; shared < minLen; shared += 32) {
            uint256 wordA;
            uint256 wordB;

            assembly {
                wordA := mload(add(_a, add(32, shared)))
                wordB := mload(add(_b, add(32, shared)))
            }

            if (wordA != wordB) {
                uint256 diff = wordA ^ wordB;
                uint256 leadingBits = clz(diff); // clz counts bits
                return shared + (leadingBits / 8);
            }
        }

        // Fallback to byte-by-byte for remaining bytes
        while (shared < minLen && _a[shared] == _b[shared]) shared++;
    }
}

[0xClandestine]

With proofs in calldata, and the above the optimizations I'm hitting 175k gas for an account proof verification (down from over 400k).

[0xClandestine]

Mind you I'm working off of optimisms implementation, this version looks a bit more expensive.

[0xClandestine]

Feel like we should use an if statement with a revert case after.