From 9eb5f1cac83afe4d0000ea5d0a9aaf296a8c21d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Wed, 4 Sep 2024 11:13:57 -0600 Subject: [PATCH 001/114] Add memory utils --- .changeset/dull-students-eat.md | 5 ++++ contracts/utils/Memory.sol | 34 +++++++++++++++++++++++++++ contracts/utils/README.adoc | 1 + test/utils/Memory.t.sol | 16 +++++++++++++ test/utils/Memory.test.js | 41 +++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+) create mode 100644 .changeset/dull-students-eat.md create mode 100644 contracts/utils/Memory.sol create mode 100644 test/utils/Memory.t.sol create mode 100644 test/utils/Memory.test.js diff --git a/.changeset/dull-students-eat.md b/.changeset/dull-students-eat.md new file mode 100644 index 00000000000..94c4fc21ef2 --- /dev/null +++ b/.changeset/dull-students-eat.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Memory`: Add library with utilities to manipulate memory diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol new file mode 100644 index 00000000000..a0fc881e318 --- /dev/null +++ b/contracts/utils/Memory.sol @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +/// @dev Memory utility library. +library Memory { + type Pointer is bytes32; + + /// @dev Returns a memory pointer to the current free memory pointer. + function getFreePointer() internal pure returns (Pointer ptr) { + assembly ("memory-safe") { + ptr := mload(0x40) + } + } + + /// @dev Sets the free memory pointer to a specific value. + /// + /// WARNING: Everything after the pointer may be overwritten. + function setFreePointer(Pointer ptr) internal pure { + assembly ("memory-safe") { + mstore(0x40, ptr) + } + } + + /// @dev Pointer to `bytes32`. + function asBytes32(Pointer ptr) internal pure returns (bytes32) { + return Pointer.unwrap(ptr); + } + + /// @dev `bytes32` to pointer. + function asPointer(bytes32 value) internal pure returns (Pointer) { + return Pointer.wrap(value); + } +} diff --git a/contracts/utils/README.adoc b/contracts/utils/README.adoc index 0ef3e5387c8..87af4fd4b7b 100644 --- a/contracts/utils/README.adoc +++ b/contracts/utils/README.adoc @@ -40,6 +40,7 @@ Miscellaneous contracts and libraries containing utility functions you can use t * {Packing}: A library for packing and unpacking multiple values into bytes32 * {Panic}: A library to revert with https://docs.soliditylang.org/en/v0.8.20/control-structures.html#panic-via-assert-and-error-via-require[Solidity panic codes]. * {Comparators}: A library that contains comparator functions to use with with the {Heap} library. + * {Memory}: A utility library to manipulate memory. [NOTE] ==== diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol new file mode 100644 index 00000000000..4cc60b88f9c --- /dev/null +++ b/test/utils/Memory.t.sol @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +import {Test} from "forge-std/Test.sol"; +import {Memory} from "@openzeppelin/contracts/utils/Memory.sol"; + +contract MemoryTest is Test { + using Memory for *; + + function testSymbolicGetSetFreePointer(bytes32 ptr) public { + Memory.Pointer memoryPtr = ptr.asPointer(); + Memory.setFreePointer(memoryPtr); + assertEq(Memory.getFreePointer().asBytes32(), memoryPtr.asBytes32()); + } +} diff --git a/test/utils/Memory.test.js b/test/utils/Memory.test.js new file mode 100644 index 00000000000..5698728dcfd --- /dev/null +++ b/test/utils/Memory.test.js @@ -0,0 +1,41 @@ +const { ethers } = require('hardhat'); +const { expect } = require('chai'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +async function fixture() { + const mock = await ethers.deployContract('$Memory'); + + return { mock }; +} + +describe('Memory', function () { + beforeEach(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + describe('free pointer', function () { + it('sets memory pointer', async function () { + const ptr = '0x00000000000000000000000000000000000000000000000000000000000000a0'; + expect(await this.mock.$setFreePointer(ptr)).to.not.be.reverted; + }); + + it('gets memory pointer', async function () { + expect(await this.mock.$getFreePointer()).to.equal( + // Default pointer + '0x0000000000000000000000000000000000000000000000000000000000000080', + ); + }); + + it('asBytes32', async function () { + const ptr = ethers.toBeHex('0x1234', 32); + await this.mock.$setFreePointer(ptr); + expect(await this.mock.$asBytes32(ptr)).to.equal(ptr); + }); + + it('asPointer', async function () { + const ptr = ethers.toBeHex('0x1234', 32); + await this.mock.$setFreePointer(ptr); + expect(await this.mock.$asPointer(ptr)).to.equal(ptr); + }); + }); +}); From 2d397f467f202ccc1dc3c2b240a9f3353f13af83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Wed, 4 Sep 2024 11:43:04 -0600 Subject: [PATCH 002/114] Fix tests upgradeable --- contracts/mocks/Stateless.sol | 1 + 1 file changed, 1 insertion(+) diff --git a/contracts/mocks/Stateless.sol b/contracts/mocks/Stateless.sol index 846c77d98e8..a96dd48cc87 100644 --- a/contracts/mocks/Stateless.sol +++ b/contracts/mocks/Stateless.sol @@ -37,6 +37,7 @@ import {SignatureChecker} from "../utils/cryptography/SignatureChecker.sol"; import {SignedMath} from "../utils/math/SignedMath.sol"; import {StorageSlot} from "../utils/StorageSlot.sol"; import {Strings} from "../utils/Strings.sol"; +import {Memory} from "../utils/Memory.sol"; import {Time} from "../utils/types/Time.sol"; contract Dummy1234 {} From 2a0fb7e5db92a563991ff7b447596b8191d22381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Thu, 5 Sep 2024 12:21:13 -0600 Subject: [PATCH 003/114] Add docs --- contracts/utils/Memory.sol | 8 +++++- docs/modules/ROOT/pages/utilities.adoc | 36 +++++++++++++++++++++++++- test/utils/Memory.t.sol | 5 ++-- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index a0fc881e318..abb6f100bc6 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -2,7 +2,13 @@ pragma solidity ^0.8.20; -/// @dev Memory utility library. +/// @dev Utilities to manipulate memory. +/// +/// Memory is a contiguous and dynamic byte array in which Solidity stores non-primitive types. +/// This library provides functions to manipulate pointers to this dynamic array. +/// +/// WARNING: When manipulating memory, make sure to follow the Solidity documentation +/// guidelines for https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[Memory Safety]. library Memory { type Pointer is bytes32; diff --git a/docs/modules/ROOT/pages/utilities.adoc b/docs/modules/ROOT/pages/utilities.adoc index b8afec4eabd..d1cf470d60a 100644 --- a/docs/modules/ROOT/pages/utilities.adoc +++ b/docs/modules/ROOT/pages/utilities.adoc @@ -189,7 +189,7 @@ Some use cases require more powerful data structures than arrays and mappings of - xref:api:utils.adoc#EnumerableSet[`EnumerableSet`]: A https://en.wikipedia.org/wiki/Set_(abstract_data_type)[set] with enumeration capabilities. - xref:api:utils.adoc#EnumerableMap[`EnumerableMap`]: A `mapping` variant with enumeration capabilities. - xref:api:utils.adoc#MerkleTree[`MerkleTree`]: An on-chain https://wikipedia.org/wiki/Merkle_Tree[Merkle Tree] with helper functions. -- xref:api:utils.adoc#Heap.sol[`Heap`]: A +- xref:api:utils.adoc#Heap.sol[`Heap`]: A https://en.wikipedia.org/wiki/Binary_heap[binary heap] to store elements with priority defined by a compartor function. The `Enumerable*` structures are similar to mappings in that they store and remove elements in constant time and don't allow for repeated entries, but they also support _enumeration_, which means you can easily query all stored entries both on and off-chain. @@ -386,3 +386,37 @@ await instance.multicall([ instance.interface.encodeFunctionData("bar") ]); ---- + +=== Memory + +The `Memory` library provides functions for advanced use cases that require granular memory management. A common use case is to avoid unnecessary memory expansion costs when iterating over a section of the code that allocates new memory. Consider the following example: + +[source,solidity] +---- +function callFoo(address target) internal { + bytes memory callData = abi.encodeWithSelector( + bytes4(keccak256("foo()")) + ) + (bool success, /* bytes memory returndata */) = target.call(callData); + require(success); +} +---- + +Note the function allocates memory for both the `callData` argument and for the returndata even if it's ignored. As such, it may be desirable to reset the free memory pointer after the end of the function. + +[source,solidity] +---- +function callFoo(address target) internal { + Memory.Pointer ptr = Memory.getFreePointer(); // Cache pointer + bytes memory callData = abi.encodeWithSelector( + bytes4(keccak256("foo()")) + ) + (bool success, /* bytes memory returndata */) = target.call(callData); + require(success); + Memory.setFreePointer(ptr); // Reset pointer +} +---- + +In this way, new memory will be allocated in the space where the `returndata` and `callData` used to be, potentially reducing memory expansion costs by shrinking the its size at the end of the transaction and resulting in gas savings. + +IMPORTANT: By default, Solidity handles memory safely. Using this library without understanding how memory works may be dangerous. Consider thoroughly reading the Solidity documentation about the https://docs.soliditylang.org/en/v0.8.20/internals/layout_in_memory.html[memory layout] and how the language defines https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[memory safety]. diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol index 4cc60b88f9c..99cb75fb095 100644 --- a/test/utils/Memory.t.sol +++ b/test/utils/Memory.t.sol @@ -9,8 +9,7 @@ contract MemoryTest is Test { using Memory for *; function testSymbolicGetSetFreePointer(bytes32 ptr) public { - Memory.Pointer memoryPtr = ptr.asPointer(); - Memory.setFreePointer(memoryPtr); - assertEq(Memory.getFreePointer().asBytes32(), memoryPtr.asBytes32()); + Memory.setFreePointer(ptr.asPointer()); + assertEq(Memory.getFreePointer().asBytes32(), ptr); } } From a7e61c3bd521822e7a8c932c5596c6f3cea8739e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Thu, 5 Sep 2024 12:32:19 -0600 Subject: [PATCH 004/114] Make use of the library --- contracts/access/manager/AuthorityUtils.sol | 3 +++ contracts/token/ERC20/extensions/ERC4626.sol | 3 +++ contracts/token/ERC20/utils/SafeERC20.sol | 7 +++++++ contracts/utils/cryptography/SignatureChecker.sol | 6 +++++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/contracts/access/manager/AuthorityUtils.sol b/contracts/access/manager/AuthorityUtils.sol index fb3018ca805..4cc77123716 100644 --- a/contracts/access/manager/AuthorityUtils.sol +++ b/contracts/access/manager/AuthorityUtils.sol @@ -4,6 +4,7 @@ pragma solidity ^0.8.20; import {IAuthority} from "./IAuthority.sol"; +import {Memory} from "../../utils/Memory.sol"; library AuthorityUtils { /** @@ -17,6 +18,7 @@ library AuthorityUtils { address target, bytes4 selector ) internal view returns (bool immediate, uint32 delay) { + Memory.Pointer ptr = Memory.getFreePointer(); (bool success, bytes memory data) = authority.staticcall( abi.encodeCall(IAuthority.canCall, (caller, target, selector)) ); @@ -27,6 +29,7 @@ library AuthorityUtils { immediate = abi.decode(data, (bool)); } } + Memory.setFreePointer(ptr); return (immediate, delay); } } diff --git a/contracts/token/ERC20/extensions/ERC4626.sol b/contracts/token/ERC20/extensions/ERC4626.sol index c71b14ad48c..121d729bc96 100644 --- a/contracts/token/ERC20/extensions/ERC4626.sol +++ b/contracts/token/ERC20/extensions/ERC4626.sol @@ -7,6 +7,7 @@ import {IERC20, IERC20Metadata, ERC20} from "../ERC20.sol"; import {SafeERC20} from "../utils/SafeERC20.sol"; import {IERC4626} from "../../../interfaces/IERC4626.sol"; import {Math} from "../../../utils/math/Math.sol"; +import {Memory} from "../../../utils/Memory.sol"; /** * @dev Implementation of the ERC-4626 "Tokenized Vault Standard" as defined in @@ -84,6 +85,7 @@ abstract contract ERC4626 is ERC20, IERC4626 { * @dev Attempts to fetch the asset decimals. A return value of false indicates that the attempt failed in some way. */ function _tryGetAssetDecimals(IERC20 asset_) private view returns (bool, uint8) { + Memory.Pointer ptr = Memory.getFreePointer(); (bool success, bytes memory encodedDecimals) = address(asset_).staticcall( abi.encodeCall(IERC20Metadata.decimals, ()) ); @@ -93,6 +95,7 @@ abstract contract ERC4626 is ERC20, IERC4626 { return (true, uint8(returnedDecimals)); } } + Memory.setFreePointer(ptr); return (false, 0); } diff --git a/contracts/token/ERC20/utils/SafeERC20.sol b/contracts/token/ERC20/utils/SafeERC20.sol index ed41fb042c9..4d06ded819d 100644 --- a/contracts/token/ERC20/utils/SafeERC20.sol +++ b/contracts/token/ERC20/utils/SafeERC20.sol @@ -6,6 +6,7 @@ pragma solidity ^0.8.20; import {IERC20} from "../IERC20.sol"; import {IERC1363} from "../../../interfaces/IERC1363.sol"; import {Address} from "../../../utils/Address.sol"; +import {Memory} from "../../../utils/Memory.sol"; /** * @title SafeERC20 @@ -32,7 +33,9 @@ library SafeERC20 { * non-reverting calls are assumed to be successful. */ function safeTransfer(IERC20 token, address to, uint256 value) internal { + Memory.Pointer ptr = Memory.getFreePointer(); _callOptionalReturn(token, abi.encodeCall(token.transfer, (to, value))); + Memory.setFreePointer(ptr); } /** @@ -40,7 +43,9 @@ library SafeERC20 { * calling contract. If `token` returns no value, non-reverting calls are assumed to be successful. */ function safeTransferFrom(IERC20 token, address from, address to, uint256 value) internal { + Memory.Pointer ptr = Memory.getFreePointer(); _callOptionalReturn(token, abi.encodeCall(token.transferFrom, (from, to, value))); + Memory.setFreePointer(ptr); } /** @@ -72,12 +77,14 @@ library SafeERC20 { * to be set to zero before setting it to a non-zero value, such as USDT. */ function forceApprove(IERC20 token, address spender, uint256 value) internal { + Memory.Pointer ptr = Memory.getFreePointer(); bytes memory approvalCall = abi.encodeCall(token.approve, (spender, value)); if (!_callOptionalReturnBool(token, approvalCall)) { _callOptionalReturn(token, abi.encodeCall(token.approve, (spender, 0))); _callOptionalReturn(token, approvalCall); } + Memory.setFreePointer(ptr); } /** diff --git a/contracts/utils/cryptography/SignatureChecker.sol b/contracts/utils/cryptography/SignatureChecker.sol index 9aaa2e0716c..16e038d2d87 100644 --- a/contracts/utils/cryptography/SignatureChecker.sol +++ b/contracts/utils/cryptography/SignatureChecker.sol @@ -5,6 +5,7 @@ pragma solidity ^0.8.20; import {ECDSA} from "./ECDSA.sol"; import {IERC1271} from "../../interfaces/IERC1271.sol"; +import {Memory} from "../Memory.sol"; /** * @dev Signature verification helper that can be used instead of `ECDSA.recover` to seamlessly support both ECDSA @@ -40,11 +41,14 @@ library SignatureChecker { bytes32 hash, bytes memory signature ) internal view returns (bool) { + Memory.Pointer ptr = Memory.getFreePointer(); (bool success, bytes memory result) = signer.staticcall( abi.encodeCall(IERC1271.isValidSignature, (hash, signature)) ); - return (success && + bool valid = (success && result.length >= 32 && abi.decode(result, (bytes32)) == bytes32(IERC1271.isValidSignature.selector)); + Memory.setFreePointer(ptr); + return valid; } } From 1aae8bbd7e77f20600c4661a56a1d1d95ceb76b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Tue, 8 Oct 2024 23:48:04 -0600 Subject: [PATCH 005/114] Update docs/modules/ROOT/pages/utilities.adoc Co-authored-by: Hadrien Croubois --- docs/modules/ROOT/pages/utilities.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/utilities.adoc b/docs/modules/ROOT/pages/utilities.adoc index d1cf470d60a..84ede5a4b5e 100644 --- a/docs/modules/ROOT/pages/utilities.adoc +++ b/docs/modules/ROOT/pages/utilities.adoc @@ -417,6 +417,6 @@ function callFoo(address target) internal { } ---- -In this way, new memory will be allocated in the space where the `returndata` and `callData` used to be, potentially reducing memory expansion costs by shrinking the its size at the end of the transaction and resulting in gas savings. +This way, memory is allocated to accommodate the `callData`, and the `returndata` is freed. This allows other memory operations to reuse that space, thus reducing the memory expansion costs of these operations. In particular, this allows many `callFoo` to be performed in a loop with limited memory expansion costs. IMPORTANT: By default, Solidity handles memory safely. Using this library without understanding how memory works may be dangerous. Consider thoroughly reading the Solidity documentation about the https://docs.soliditylang.org/en/v0.8.20/internals/layout_in_memory.html[memory layout] and how the language defines https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[memory safety]. From d514606a9cba7c1fc081427b5d7dd19017f26f9c Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 6 Mar 2025 17:14:59 +0100 Subject: [PATCH 006/114] fix tests --- test/utils/Memory.t.sol | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol index 99cb75fb095..0affe3234c4 100644 --- a/test/utils/Memory.t.sol +++ b/test/utils/Memory.t.sol @@ -8,7 +8,11 @@ import {Memory} from "@openzeppelin/contracts/utils/Memory.sol"; contract MemoryTest is Test { using Memory for *; - function testSymbolicGetSetFreePointer(bytes32 ptr) public { + function testSymbolicGetSetFreePointer(uint256 seed) public pure { + // - first 0x80 bytes are reserved (scratch + FMP + zero) + // - moving the free memory pointer to far causes OOG errors + bytes32 ptr = bytes32(bound(seed, 0x80, type(uint24).max)); + Memory.setFreePointer(ptr.asPointer()); assertEq(Memory.getFreePointer().asBytes32(), ptr); } From 14fa04ef86a0c78af70fae316d7cee2cae71304c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Wed, 7 May 2025 12:30:10 -0600 Subject: [PATCH 007/114] Update contracts/utils/Memory.sol Co-authored-by: Arr00 <13561405+arr00@users.noreply.github.com> --- contracts/utils/Memory.sol | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index abb6f100bc6..33842a6eb4d 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -2,13 +2,15 @@ pragma solidity ^0.8.20; -/// @dev Utilities to manipulate memory. -/// -/// Memory is a contiguous and dynamic byte array in which Solidity stores non-primitive types. -/// This library provides functions to manipulate pointers to this dynamic array. -/// -/// WARNING: When manipulating memory, make sure to follow the Solidity documentation -/// guidelines for https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[Memory Safety]. +/** + * @dev Utilities to manipulate memory. + * + * Memory is a contiguous and dynamic byte array in which Solidity stores non-primitive types. + * This library provides functions to manipulate pointers to this dynamic array. + * + * WARNING: When manipulating memory, make sure to follow the Solidity documentation + * guidelines for https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[Memory Safety]. + */ library Memory { type Pointer is bytes32; From d0d55fcc356d813d5563687a229ee6710e12d5aa Mon Sep 17 00:00:00 2001 From: Arr00 <13561405+arr00@users.noreply.github.com> Date: Wed, 7 May 2025 14:32:03 -0400 Subject: [PATCH 008/114] Update contracts/utils/Memory.sol --- contracts/utils/Memory.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 33842a6eb4d..e5cc0e06cc8 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -3,7 +3,7 @@ pragma solidity ^0.8.20; /** - * @dev Utilities to manipulate memory. + * @dev Utilities to manipulate memory. * * Memory is a contiguous and dynamic byte array in which Solidity stores non-primitive types. * This library provides functions to manipulate pointers to this dynamic array. From 7b3cb6638ac4f56015d3ada00dc19c776624bbe3 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 10 May 2025 17:23:03 -0600 Subject: [PATCH 009/114] Add RLP library --- contracts/utils/RLP.sol | 228 ++++++++++++++++++++++++++++ contracts/utils/math/Endianness.sol | 49 ++++++ 2 files changed, 277 insertions(+) create mode 100644 contracts/utils/RLP.sol create mode 100644 contracts/utils/math/Endianness.sol diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol new file mode 100644 index 00000000000..4eafd3b58ad --- /dev/null +++ b/contracts/utils/RLP.sol @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.25; + +import {Math} from "./math/Math.sol"; +import {Endianness} from "./math/Endianness.sol"; +import {Bytes} from "./Bytes.sol"; + +library RLP { + using Bytes for bytes; + + struct Item { + uint256 length; + bytes32 ptr; + } + + enum ItemType { + DATA_ITEM, + LIST_ITEM + } + + uint8 internal constant SHORT_THRESHOLD = 55; + + uint8 internal constant SHORT_OFFSET = 128; + uint8 internal constant LONG_LENGTH_OFFSET = SHORT_OFFSET + SHORT_THRESHOLD + 1; // 184 + uint8 internal constant LONG_OFFSET = LONG_LENGTH_OFFSET + 8; // 192 + uint8 internal constant SHORT_LIST_OFFSET = LONG_OFFSET + SHORT_THRESHOLD + 1; // 248 + + function encode(bytes memory buffer) internal pure returns (bytes memory) { + return _isSingleByte(buffer) ? buffer : bytes.concat(_encodeLength(buffer.length, SHORT_OFFSET), buffer); + } + + function encode(bytes[] memory list) internal pure returns (bytes memory) { + bytes memory flattened = _flatten(list); + return bytes.concat(_encodeLength(flattened.length, LONG_OFFSET), flattened); + } + + function encode(string memory str) internal pure returns (bytes memory) { + return encode(bytes(str)); + } + + function encode(address addr) internal pure returns (bytes memory) { + return encode(abi.encodePacked(addr)); + } + + function encode(uint256 value) internal pure returns (bytes memory) { + return encode(_toBinaryBuffer(value)); + } + + function encode(bytes32 value) internal pure returns (bytes memory) { + return encode(_toBinaryBuffer(uint256(value))); + } + + function encode(bool value) internal pure returns (bytes memory) { + bytes memory encoded = new bytes(1); + encoded[0] = value ? bytes1(0x01) : bytes1(SHORT_OFFSET); // false is encoded as an empty string + return encoded; + } + + function toItem(bytes memory value) internal pure returns (Item memory) { + require(value.length != 0); // Empty arrays are not RLP items. + return Item(value.length, _skippedLengthPtr(value)); + } + + function readList(Item memory item) internal pure returns (Item[] memory) { + (uint256 listOffset, uint256 listLength, ItemType itemType) = _decodeLength(item); + require(itemType == ItemType.LIST_ITEM); + require(listOffset + listLength == item.length); + Item[] memory items = new Item[](32); + + uint256 itemCount = item.length; + + for (uint256 i; listOffset < itemCount; i++) { + (uint256 itemOffset, uint256 itemLength, ) = _decodeLength( + Item(itemCount - listOffset, bytes32(uint256(item.ptr) + listOffset)) + ); + items[i] = Item(itemLength + itemOffset, bytes32(uint256(item.ptr) + listOffset)); + listOffset += itemOffset + itemLength; + } + + // Decrease the array size to match the actual item count. + assembly ("memory-safe") { + mstore(items, itemCount) + } + return items; + } + + function readList(bytes memory value) internal pure returns (Item[] memory) { + return readList(toItem(value)); + } + + function readBytes(Item memory item) internal pure returns (bytes memory) { + (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); + require(itemType == ItemType.DATA_ITEM); + require(item.length == itemOffset + itemLength); + return _copy(item.ptr, bytes32(itemOffset), itemLength); + } + + function readBytes(bytes memory item) internal pure returns (bytes memory) { + return readBytes(toItem(item)); + } + + function readRawBytes(Item memory item) internal pure returns (bytes memory) { + return _copy(item.ptr, 0, item.length); + } + + function _isSingleByte(bytes memory buffer) private pure returns (bool) { + return buffer.length == 1 && uint8(buffer[0]) <= SHORT_OFFSET - 1; + } + + function _encodeLength(uint256 length, uint256 offset) private pure returns (bytes memory) { + return + length <= SHORT_THRESHOLD + ? abi.encodePacked(bytes1(uint8(length) + uint8(offset))) + : _encodeLongLength(length, offset); + } + + function _encodeLongLength(uint256 length, uint256 offset) private pure returns (bytes memory) { + uint256 bytesLength = Math.log256(length) + 1; // Result is floored + return + abi.encodePacked( + bytes1(uint8(bytesLength) + uint8(offset) + SHORT_THRESHOLD), + Endianness.reverseUint256(length) // to big-endian + ); + } + + function _toBinaryBuffer(uint256 value) private pure returns (bytes memory) { + uint256 leadingZeroes = _countLeadingZeroBytes(value); + return abi.encodePacked(value).slice(leadingZeroes); + } + + function _countLeadingZeroBytes(uint256 x) private pure returns (uint256) { + uint256 r = 0; + if (x > 0xffffffffffffffffffffffffffffffff) r = 128; // Upper 128 bits + if ((x >> r) > 0xffffffffffffffff) r |= 64; // Next 64 bits + if ((x >> r) > 0xffffffff) r |= 32; // Next 32 bits + if ((x >> r) > 0xffff) r |= 16; // Next 16 bits + if ((x >> r) > 0xff) r |= 8; // Next 8 bits + return 31 ^ (r >> 3); // Convert to leading zero bytes count + } + + function _flatten(bytes[] memory list) private pure returns (bytes memory) { + bytes memory flattened = new bytes(_totalLength(list)); + bytes32 dataPtr = _skippedLengthPtr(flattened); + for (uint256 i = 0; i < list.length; i++) { + bytes memory item = list[i]; + uint256 length = item.length; + _copy(dataPtr, _skippedLengthPtr(item), length); + dataPtr = bytes32(uint256(dataPtr) + length); + } + return flattened; + } + + function _totalLength(bytes[] memory list) private pure returns (uint256) { + uint256 totalLength; + for (uint256 i = 0; i < list.length; i++) { + totalLength += list[i].length; + } + return totalLength; + } + + function _decodeLength(Item memory item) private pure returns (uint256 offset, uint256 length, ItemType) { + require(item.length != 0); + bytes32 ptr = item.ptr; + uint256 prefix = uint8(_extractMemoryByte(ptr)); + if (prefix < SHORT_OFFSET) return (0, 1, ItemType.DATA_ITEM); // Single byte. + if (prefix < LONG_LENGTH_OFFSET) return _decodeShortString(prefix - SHORT_OFFSET, item); + if (prefix < LONG_OFFSET) { + (offset, length) = _decodeLong(prefix - LONG_LENGTH_OFFSET, item); + return (offset, length, ItemType.DATA_ITEM); + } + if (prefix < SHORT_LIST_OFFSET) return _decodeShortList(prefix - LONG_OFFSET, item); + (offset, length) = _decodeLong(prefix - SHORT_LIST_OFFSET, item); + return (offset, length, ItemType.LIST_ITEM); + } + + function _decodeShortString( + uint256 strLength, + Item memory item + ) private pure returns (uint256 offset, uint256 length, ItemType) { + require(item.length > strLength); + require(strLength != 1 || _extractMemoryByte(bytes32(uint256(item.ptr) + 1)) >= bytes1(SHORT_OFFSET)); + return (1, strLength, ItemType.DATA_ITEM); + } + + function _decodeShortList( + uint256 listLength, + Item memory item + ) private pure returns (uint256 offset, uint256 length, ItemType) { + require(item.length > listLength); + return (1, listLength, ItemType.LIST_ITEM); + } + + function _decodeLong(uint256 lengthLength, Item memory item) private pure returns (uint256 offset, uint256 length) { + lengthLength += 1; // 1 byte for the length itself + require(item.length > lengthLength); + require(_extractMemoryByte(item.ptr) != 0x00); + + uint256 len = _extractMemoryWord(bytes32(uint256(item.ptr) + 1)) >> (256 - 8 * lengthLength); + require(len > SHORT_OFFSET); + require(item.length <= lengthLength + len); + return (lengthLength + 1, len); + } + + function _copy(bytes32 destPtr, bytes32 srcPtr, uint256 length) private pure returns (bytes memory src) { + assembly ("memory-safe") { + mcopy(destPtr, srcPtr, length) + src := mload(src) + } + } + + function _skippedLengthPtr(bytes memory buffer) private pure returns (bytes32 ptr) { + assembly ("memory-safe") { + ptr := add(buffer, 32) + } + } + + function _extractMemoryByte(bytes32 ptr) private pure returns (bytes1 v) { + assembly ("memory-safe") { + v := byte(0, mload(ptr)) + } + } + + function _extractMemoryWord(bytes32 ptr) private pure returns (uint256 v) { + assembly ("memory-safe") { + v := mload(ptr) + } + } +} diff --git a/contracts/utils/math/Endianness.sol b/contracts/utils/math/Endianness.sol new file mode 100644 index 00000000000..e506fa7a7d7 --- /dev/null +++ b/contracts/utils/math/Endianness.sol @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +library Endianness { + function reverseUint256(uint256 value) internal pure returns (uint256) { + value = // swap bytes + ((value >> 8) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) | + ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) << 8); + value = // swap 2-byte long pairs + ((value >> 16) & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) | + ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) << 16); + value = // swap 4-byte long pairs + ((value >> 32) & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) | + ((value & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) << 32); + value = // swap 8-byte long pairs + ((value >> 64) & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) | + ((value & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) << 64); + return (value >> 128) | (value << 128); // swap 16-byte long pairs + } + + function reverseUint128(uint128 value) internal pure returns (uint256) { + value = // swap bytes + ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | + ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); + value = // swap 2-byte long pairs + ((value & 0xFFFF0000FFFF0000FFFF0000FFFF0000) >> 16) | + ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF) << 16); + value = // swap 4-byte long pairs + ((value & 0xFFFFFFFF00000000FFFFFFFF00000000) >> 32) | + ((value & 0x00000000FFFFFFFF00000000FFFFFFFF) << 32); + return (value >> 64) | (value << 64); // swap 8-byte long pairs + } + + function reverseUint64(uint64 value) internal pure returns (uint256) { + value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes + value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs + return (value >> 32) | (value << 32); // swap 4-byte long pairs + } + + function reverseUint32(uint64 value) internal pure returns (uint256) { + value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes + return (value >> 16) | (value << 16); // swap 2-byte long pairs + } + + function reverseUint16(uint64 value) internal pure returns (uint256) { + return (value >> 8) | (value << 8); + } +} From 95149f803f2ad492fe964ea259a5c665c5ae12f6 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 10 May 2025 20:50:53 -0600 Subject: [PATCH 010/114] Add TrieProof library --- contracts/utils/Bytes.sol | 17 +++ contracts/utils/RLP.sol | 2 +- contracts/utils/Strings.sol | 3 +- contracts/utils/TrieProof.sol | 188 ++++++++++++++++++++++++++++++++++ 4 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 contracts/utils/TrieProof.sol diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 1234b845513..725bc06cafe 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -99,6 +99,23 @@ library Bytes { return result; } + /// @dev Split each byte in `value` into two nibbles (4 bits each). + function nibbles(bytes memory value) internal pure returns (bytes memory) { + uint256 length = value.length; + bytes memory nibbles_ = new bytes(length * 2); + for (uint256 i = 0; i < length; i++) { + (nibbles_[i * 2], nibbles_[i * 2 + 1]) = (value[i] & 0xf0, value[i] & 0x0f); + } + return nibbles_; + } + + /** + * @dev Returns true if the two byte buffers are equal. + */ + function equal(bytes memory a, bytes memory b) internal pure returns (bool) { + return a.length == b.length && keccak256(a) == keccak256(b); + } + /** * @dev Reads a bytes32 from a bytes array without bounds checking. * diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 4eafd3b58ad..05a56d7e9e1 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.25; +pragma solidity ^0.8.27; import {Math} from "./math/Math.sol"; import {Endianness} from "./math/Endianness.sol"; diff --git a/contracts/utils/Strings.sol b/contracts/utils/Strings.sol index 4cc597646f2..a865bfbc785 100644 --- a/contracts/utils/Strings.sol +++ b/contracts/utils/Strings.sol @@ -6,6 +6,7 @@ pragma solidity ^0.8.20; import {Math} from "./math/Math.sol"; import {SafeCast} from "./math/SafeCast.sol"; import {SignedMath} from "./math/SignedMath.sol"; +import {Bytes} from "./Bytes.sol"; /** * @dev String operations. @@ -132,7 +133,7 @@ library Strings { * @dev Returns true if the two strings are equal. */ function equal(string memory a, string memory b) internal pure returns (bool) { - return bytes(a).length == bytes(b).length && keccak256(bytes(a)) == keccak256(bytes(b)); + return Bytes.equal(bytes(a), bytes(b)); } /** diff --git a/contracts/utils/TrieProof.sol b/contracts/utils/TrieProof.sol new file mode 100644 index 00000000000..7071b750475 --- /dev/null +++ b/contracts/utils/TrieProof.sol @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.27; + +import {Bytes} from "./Bytes.sol"; +import {RLP} from "./RLP.sol"; +import {Math} from "./math/Math.sol"; + +library TrieProof { + using Bytes for bytes; + using RLP for *; + + enum Prefix { + EXTENSION_EVEN, // 0 - Extension node with even length path + EXTENSION_ODD, // 1 - Extension node with odd length path + LEAF_EVEN, // 2 - Leaf node with even length path + LEAF_ODD // 3 - Leaf node with odd length path + } + + enum ProofError { + NO_ERROR, // No error occurred during proof verification + EMPTY_KEY, // The provided key is empty + INDEX_OUT_OF_BOUNDS, // Array index access is out of bounds + INVALID_ROOT_HASH, // The provided root hash doesn't match the proof + INVALID_LARGE_INTERNAL_HASH, // Internal node hash exceeds expected size + INVALID_INTERNAL_NODE_HASH, // Internal node hash doesn't match expected value + EMPTY_VALUE, // The value to verify is empty + INVALID_EXTRA_PROOF_ELEMENT, // Proof contains unexpected additional elements + INVALID_PATH_REMAINDER, // Path remainder doesn't match expected value + INVALID_KEY_REMAINDER, // Key remainder doesn't match expected value + UNKNOWN_NODE_PREFIX, // Node prefix is not recognized + UNPARSEABLE_NODE, // Node cannot be parsed from RLP encoding + INVALID_PROOF // General proof validation failure + } + + struct Node { + bytes encoded; // Raw RLP encoded node + RLP.Item[] decoded; // Decoded RLP items + } + + uint256 internal constant EVM_TREE_RADIX = 16; // Ethereum uses 16 as its trie radix (hex) + uint256 internal constant LEAF_OR_EXTENSION_NODE_LENGTH = 2; // Leaf and extension nodes have exactly 2 items + + function verify( + bytes memory key, + bytes memory value, + bytes[] memory proof, + bytes32 root + ) internal pure returns (bool) { + return verify(key, value, proof, root, EVM_TREE_RADIX); + } + + function verify( + bytes memory key, + bytes memory value, + bytes[] memory proof, + bytes32 root, + uint256 radix + ) internal pure returns (bool) { + (bytes memory processedValue, ProofError err) = processProof(key, proof, root, radix); + return processedValue.equal(value) && err == ProofError.NO_ERROR; + } + + function processProof( + bytes memory key_, + bytes[] memory proof, + bytes32 root + ) internal pure returns (bytes memory value, ProofError) { + return processProof(key_, proof, root, EVM_TREE_RADIX); + } + + function processProof( + bytes memory key_, + bytes[] memory proof, + bytes32 root, + uint256 radix + ) internal pure returns (bytes memory value, ProofError) { + if (key_.length == 0) return ("", ProofError.EMPTY_KEY); + // Convert key to nibbles (4-bit values) and begin processing from the root + return _processInclusionProof(_decodeProof(proof), key_.nibbles(), bytes.concat(root), 0, radix); + } + + // Main recursive function that traverses the trie using the provided proof + function _processInclusionProof( + Node[] memory trieProof, + bytes memory key, + bytes memory nodeId, + uint256 keyIndex, + uint256 radix + ) private pure returns (bytes memory value, ProofError err) { + uint256 branchNodeLength = radix + 1; // Branch nodes have radix+1 items (values + 1 for stored value) + + for (uint256 i = 0; i < trieProof.length; i++) { + Node memory node = trieProof[i]; + + // ensure we haven't overshot the key + if (keyIndex > key.length) return ("", ProofError.INDEX_OUT_OF_BOUNDS); + err = _validateNodeHashes(nodeId, node, keyIndex); + if (err != ProofError.NO_ERROR) return ("", err); + + uint256 nodeLength = node.decoded.length; + + // must be either a branch or leaf/extension node + if (nodeLength != branchNodeLength && nodeLength != LEAF_OR_EXTENSION_NODE_LENGTH) + return ("", ProofError.UNPARSEABLE_NODE); + + if (nodeLength == branchNodeLength) { + // If we've consumed the entire key, the value must be in the last slot + if (keyIndex == key.length) return _validateLastItem(node.decoded[radix], trieProof, i); + + // Otherwise, continue down the branch specified by the next nibble in the key + uint8 branchKey = uint8(key[keyIndex]); + (nodeId, keyIndex) = (_id(node.decoded[branchKey]), keyIndex + 1); + } else if (nodeLength == LEAF_OR_EXTENSION_NODE_LENGTH) { + bytes memory path = _path(node); + uint8 prefix = uint8(path[0]); + uint8 offset = 2 - (prefix % 2); // Calculate offset based on even/odd path length + bytes memory pathRemainder = Bytes.slice(path, offset); // Path after the prefix + bytes memory keyRemainder = Bytes.slice(key, keyIndex); // Remaining key to match + uint256 sharedNibbleLength = _sharedNibbleLength(pathRemainder, keyRemainder); + + // Path must match at least partially with our key + if (sharedNibbleLength == 0) return ("", ProofError.INVALID_PATH_REMAINDER); + if (prefix > uint8(type(Prefix).max)) return ("", ProofError.UNKNOWN_NODE_PREFIX); + + // Leaf node (terminal) - return its value if key matches completely + if (Prefix(prefix) == Prefix.LEAF_EVEN || Prefix(prefix) == Prefix.LEAF_ODD) { + if (keyRemainder.length == 0) return ("", ProofError.INVALID_KEY_REMAINDER); + return _validateLastItem(node.decoded[1], trieProof, i); + } + + // Extension node (non-terminal) - continue to next node + // Increment keyIndex by the number of nibbles consumed + (nodeId, keyIndex) = (_id(node.decoded[1]), keyIndex + sharedNibbleLength); + } + } + + // If we've gone through all proof elements without finding a value, the proof is invalid + return ("", ProofError.INVALID_PROOF); + } + + function _validateNodeHashes( + bytes memory nodeId, + Node memory node, + uint256 keyIndex + ) private pure returns (ProofError) { + if (keyIndex == 0 && !bytes.concat(keccak256(node.encoded)).equal(nodeId)) return ProofError.INVALID_ROOT_HASH; // Root node must match root hash + if (node.encoded.length >= 32 && !bytes.concat(keccak256(node.encoded)).equal(nodeId)) + return ProofError.INVALID_LARGE_INTERNAL_HASH; // Large nodes are stored as hashes + if (!node.encoded.equal(nodeId)) return ProofError.INVALID_INTERNAL_NODE_HASH; // Small nodes must match directly + return ProofError.NO_ERROR; // No error + } + + function _validateLastItem( + RLP.Item memory item, + Node[] memory trieProof, + uint256 i + ) private pure returns (bytes memory value, ProofError) { + bytes memory value_ = item.readBytes(); + if (value_.length == 0) return ("", ProofError.EMPTY_VALUE); + if (i != trieProof.length - 1) return ("", ProofError.INVALID_EXTRA_PROOF_ELEMENT); + return (value_, ProofError.NO_ERROR); + } + + function _decodeProof(bytes[] memory proof) private pure returns (Node[] memory proof_) { + uint256 length = proof.length; + proof_ = new Node[](length); + for (uint256 i = 0; i < length; i++) { + proof_[i] = Node(proof[i], proof[i].readList()); + } + } + + function _id(RLP.Item memory node) private pure returns (bytes memory) { + return node.length < 32 ? node.readRawBytes() : node.readBytes(); + } + + function _path(Node memory node) private pure returns (bytes memory) { + return node.decoded[0].readBytes().nibbles(); + } + + function _sharedNibbleLength(bytes memory _a, bytes memory _b) private pure returns (uint256 shared_) { + uint256 max = Math.max(_a.length, _b.length); + uint256 length; + while (length < max && _a[length] == _b[length]) { + length++; + } + return length; + } +} From ad5d4ac8b6d899fac87d44b42ad5a9057e4b019f Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 10 May 2025 20:55:51 -0600 Subject: [PATCH 011/114] up --- contracts/utils/RLP.sol | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 05a56d7e9e1..6ded8d2b8c5 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -9,23 +9,24 @@ library RLP { using Bytes for bytes; struct Item { - uint256 length; - bytes32 ptr; + uint256 length; // Total length of the item in bytes + bytes32 ptr; // Memory pointer to the start of the item } enum ItemType { - DATA_ITEM, - LIST_ITEM + DATA_ITEM, // Single data value + LIST_ITEM // List of RLP encoded items } - uint8 internal constant SHORT_THRESHOLD = 55; + uint8 internal constant SHORT_THRESHOLD = 55; // Maximum length for data that will be encoded using the short format - uint8 internal constant SHORT_OFFSET = 128; - uint8 internal constant LONG_LENGTH_OFFSET = SHORT_OFFSET + SHORT_THRESHOLD + 1; // 184 - uint8 internal constant LONG_OFFSET = LONG_LENGTH_OFFSET + 8; // 192 - uint8 internal constant SHORT_LIST_OFFSET = LONG_OFFSET + SHORT_THRESHOLD + 1; // 248 + uint8 internal constant SHORT_OFFSET = 128; // Prefix for short string (0-55 bytes) + uint8 internal constant LONG_LENGTH_OFFSET = SHORT_OFFSET + SHORT_THRESHOLD + 1; // 184 - Prefix for long string length + uint8 internal constant LONG_OFFSET = LONG_LENGTH_OFFSET + 8; // 192 - Prefix for list items + uint8 internal constant SHORT_LIST_OFFSET = LONG_OFFSET + SHORT_THRESHOLD + 1; // 248 - Prefix for long list length function encode(bytes memory buffer) internal pure returns (bytes memory) { + // Single bytes below 128 are encoded as themselves, otherwise as length prefix + data return _isSingleByte(buffer) ? buffer : bytes.concat(_encodeLength(buffer.length, SHORT_OFFSET), buffer); } @@ -162,13 +163,23 @@ library RLP { require(item.length != 0); bytes32 ptr = item.ptr; uint256 prefix = uint8(_extractMemoryByte(ptr)); - if (prefix < SHORT_OFFSET) return (0, 1, ItemType.DATA_ITEM); // Single byte. + + // Single byte below 128 + if (prefix < SHORT_OFFSET) return (0, 1, ItemType.DATA_ITEM); + + // Short string (0-55 bytes) if (prefix < LONG_LENGTH_OFFSET) return _decodeShortString(prefix - SHORT_OFFSET, item); + + // Long string (>55 bytes) if (prefix < LONG_OFFSET) { (offset, length) = _decodeLong(prefix - LONG_LENGTH_OFFSET, item); return (offset, length, ItemType.DATA_ITEM); } + + // Short list if (prefix < SHORT_LIST_OFFSET) return _decodeShortList(prefix - LONG_OFFSET, item); + + // Long list (offset, length) = _decodeLong(prefix - SHORT_LIST_OFFSET, item); return (offset, length, ItemType.LIST_ITEM); } @@ -195,6 +206,7 @@ library RLP { require(item.length > lengthLength); require(_extractMemoryByte(item.ptr) != 0x00); + // Extract the length value from the next bytes uint256 len = _extractMemoryWord(bytes32(uint256(item.ptr) + 1)) >> (256 - 8 * lengthLength); require(len > SHORT_OFFSET); require(item.length <= lengthLength + len); From 18540efec69acdba62bf97a112788825af4d2076 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 10 May 2025 23:23:47 -0600 Subject: [PATCH 012/114] Add docs --- contracts/utils/Bytes.sol | 11 +++ contracts/utils/RLP.sol | 125 +++++++++++++++++++++------- contracts/utils/TrieProof.sol | 52 ++++++++++-- contracts/utils/math/Endianness.sol | 49 ----------- contracts/utils/math/Math.sol | 52 ++++++++++++ 5 files changed, 203 insertions(+), 86 deletions(-) delete mode 100644 contracts/utils/math/Endianness.sol diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 725bc06cafe..3391f31753d 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -116,6 +116,17 @@ library Bytes { return a.length == b.length && keccak256(a) == keccak256(b); } + /// @dev Counts the number of leading zero bytes in a uint256. + function countLeadingZeroes(uint256 x) internal pure returns (uint256) { + uint256 r = 0; + if (x > 0xffffffffffffffffffffffffffffffff) r = 128; // Upper 128 bits + if ((x >> r) > 0xffffffffffffffff) r |= 64; // Next 64 bits + if ((x >> r) > 0xffffffff) r |= 32; // Next 32 bits + if ((x >> r) > 0xffff) r |= 16; // Next 16 bits + if ((x >> r) > 0xff) r |= 8; // Next 8 bits + return 31 ^ (r >> 3); // Convert to leading zero bytes count + } + /** * @dev Reads a bytes32 from a bytes array without bounds checking. * diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 6ded8d2b8c5..abc671f37cb 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -2,11 +2,16 @@ pragma solidity ^0.8.27; import {Math} from "./math/Math.sol"; -import {Endianness} from "./math/Endianness.sol"; import {Bytes} from "./Bytes.sol"; +/** + * @dev Library for encoding and decoding data in RLP format. + * Recursive Length Prefix (RLP) is the main encoding method used to serialize objects in Ethereum. + * It's used for encoding everything from transactions to blocks to Patricia-Merkle tries. + */ library RLP { - using Bytes for bytes; + using Math for uint256; + using Bytes for *; struct Item { uint256 length; // Total length of the item in bytes @@ -18,50 +23,75 @@ library RLP { LIST_ITEM // List of RLP encoded items } - uint8 internal constant SHORT_THRESHOLD = 55; // Maximum length for data that will be encoded using the short format - - uint8 internal constant SHORT_OFFSET = 128; // Prefix for short string (0-55 bytes) - uint8 internal constant LONG_LENGTH_OFFSET = SHORT_OFFSET + SHORT_THRESHOLD + 1; // 184 - Prefix for long string length - uint8 internal constant LONG_OFFSET = LONG_LENGTH_OFFSET + 8; // 192 - Prefix for list items - uint8 internal constant SHORT_LIST_OFFSET = LONG_OFFSET + SHORT_THRESHOLD + 1; // 248 - Prefix for long list length - + /** + * @dev Maximum length for data that will be encoded using the short format. + * If `data.length <= 55 bytes`, it will be encoded as: `[0x80 + length]` + data. + */ + uint8 internal constant SHORT_THRESHOLD = 55; + + /// @dev Single byte prefix for short strings (0-55 bytes) + uint8 internal constant SHORT_OFFSET = 128; + /// @dev Prefix for long string length (0xB8) + uint8 internal constant LONG_LENGTH_OFFSET = SHORT_OFFSET + SHORT_THRESHOLD + 1; // 184 + /// @dev Prefix for list items (0xC0) + uint8 internal constant LONG_OFFSET = LONG_LENGTH_OFFSET + 8; // 192 + /// @dev Prefix for long list length (0xF8) + uint8 internal constant SHORT_LIST_OFFSET = LONG_OFFSET + SHORT_THRESHOLD + 1; // 248 + + /** + * @dev Encodes a bytes array using RLP rules. + * Single bytes below 128 are encoded as themselves, otherwise as length prefix + data. + */ function encode(bytes memory buffer) internal pure returns (bytes memory) { - // Single bytes below 128 are encoded as themselves, otherwise as length prefix + data return _isSingleByte(buffer) ? buffer : bytes.concat(_encodeLength(buffer.length, SHORT_OFFSET), buffer); } + /** + * @dev Encodes an array of bytes using RLP (as a list). + * First it {_flatten}s the list of byte arrays, then encodes it with the list prefix. + */ function encode(bytes[] memory list) internal pure returns (bytes memory) { bytes memory flattened = _flatten(list); return bytes.concat(_encodeLength(flattened.length, LONG_OFFSET), flattened); } + /// @dev Convenience method to encode a string as RLP. function encode(string memory str) internal pure returns (bytes memory) { return encode(bytes(str)); } + /// @dev Convenience method to encode an address as RLP bytes (i.e. encoded as packed 20 bytes). function encode(address addr) internal pure returns (bytes memory) { return encode(abi.encodePacked(addr)); } + /// @dev Convenience method to encode a uint256 as RLP. See {_binaryBuffer}. function encode(uint256 value) internal pure returns (bytes memory) { - return encode(_toBinaryBuffer(value)); + return encode(_binaryBuffer(value)); } + /// @dev Same as {encode-uint256-}, but for bytes32. function encode(bytes32 value) internal pure returns (bytes memory) { - return encode(_toBinaryBuffer(uint256(value))); + return encode(uint256(value)); } + /** + * @dev Convenience method to encode a boolean as RLP. + * Boolean `true` is encoded as single byte 0x01, false as an empty string (0x80). + */ function encode(bool value) internal pure returns (bytes memory) { bytes memory encoded = new bytes(1); encoded[0] = value ? bytes1(0x01) : bytes1(SHORT_OFFSET); // false is encoded as an empty string return encoded; } + /// @dev Creates an RLP Item from a bytes array. function toItem(bytes memory value) internal pure returns (Item memory) { require(value.length != 0); // Empty arrays are not RLP items. return Item(value.length, _skippedLengthPtr(value)); } + /// @dev Decodes an RLP encoded list into an array of RLP Items. See {_decodeLength} function readList(Item memory item) internal pure returns (Item[] memory) { (uint256 listOffset, uint256 listLength, ItemType itemType) = _decodeLength(item); require(itemType == ItemType.LIST_ITEM); @@ -85,29 +115,55 @@ library RLP { return items; } + /// @dev Same as {readList} but for `bytes`. See {toItem}. function readList(bytes memory value) internal pure returns (Item[] memory) { return readList(toItem(value)); } + /// @dev Decodes an RLP encoded item. function readBytes(Item memory item) internal pure returns (bytes memory) { (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); require(itemType == ItemType.DATA_ITEM); require(item.length == itemOffset + itemLength); - return _copy(item.ptr, bytes32(itemOffset), itemLength); + + uint256 start = itemOffset; + bytes32 itemPtr = item.ptr; + bytes memory result = new bytes(itemLength); + assembly ("memory-safe") { + mcopy(add(result, 0x20), add(itemPtr, start), itemLength) + } + + return result; } + /// @dev Same as {readBytes} but for `bytes`. See {toItem}. function readBytes(bytes memory item) internal pure returns (bytes memory) { return readBytes(toItem(item)); } + /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. function readRawBytes(Item memory item) internal pure returns (bytes memory) { - return _copy(item.ptr, 0, item.length); + bytes32 itemPtr = item.ptr; + uint256 itemLength = item.length; + bytes memory result = new bytes(itemLength); + assembly ("memory-safe") { + mcopy(add(result, 0x20), itemPtr, itemLength) + } + + return result; } + /// @dev Checks if a buffer is a single byte below 128 (0x80). Encoded as-is in RLP. function _isSingleByte(bytes memory buffer) private pure returns (bool) { return buffer.length == 1 && uint8(buffer[0]) <= SHORT_OFFSET - 1; } + /** + * @dev Encodes a length with appropriate RLP prefix. + * + * Uses short encoding for lengths <= 55 bytes (i.e. `abi.encodePacked(bytes1(uint8(length) + uint8(offset)))`). + * Uses long encoding for lengths > 55 bytes See {_encodeLongLength}. + */ function _encodeLength(uint256 length, uint256 offset) private pure returns (bytes memory) { return length <= SHORT_THRESHOLD @@ -115,31 +171,27 @@ library RLP { : _encodeLongLength(length, offset); } + /** + * @dev Encodes a long length value (>55 bytes) with a length-of-length prefix. + * Format: [prefix + length of the length] + [length in big-endian] + */ function _encodeLongLength(uint256 length, uint256 offset) private pure returns (bytes memory) { - uint256 bytesLength = Math.log256(length) + 1; // Result is floored + uint256 bytesLength = length.log256() + 1; // Result is floored return abi.encodePacked( bytes1(uint8(bytesLength) + uint8(offset) + SHORT_THRESHOLD), - Endianness.reverseUint256(length) // to big-endian + length.reverseBitsUint256() // to big-endian ); } - function _toBinaryBuffer(uint256 value) private pure returns (bytes memory) { - uint256 leadingZeroes = _countLeadingZeroBytes(value); - return abi.encodePacked(value).slice(leadingZeroes); - } - - function _countLeadingZeroBytes(uint256 x) private pure returns (uint256) { - uint256 r = 0; - if (x > 0xffffffffffffffffffffffffffffffff) r = 128; // Upper 128 bits - if ((x >> r) > 0xffffffffffffffff) r |= 64; // Next 64 bits - if ((x >> r) > 0xffffffff) r |= 32; // Next 32 bits - if ((x >> r) > 0xffff) r |= 16; // Next 16 bits - if ((x >> r) > 0xff) r |= 8; // Next 8 bits - return 31 ^ (r >> 3); // Convert to leading zero bytes count + /// @dev Converts a uint256 to minimal binary representation, removing leading zeros. + function _binaryBuffer(uint256 value) private pure returns (bytes memory) { + return abi.encodePacked(value).slice(value.countLeadingZeroes()); } + /// @dev Concatenates all byte arrays in the `list` sequentially. Returns a flattened buffer. function _flatten(bytes[] memory list) private pure returns (bytes memory) { + // TODO: Move to Arrays.sol bytes memory flattened = new bytes(_totalLength(list)); bytes32 dataPtr = _skippedLengthPtr(flattened); for (uint256 i = 0; i < list.length; i++) { @@ -151,7 +203,9 @@ library RLP { return flattened; } + /// @dev Sums up the length of each array in the list. function _totalLength(bytes[] memory list) private pure returns (uint256) { + // TODO: Move to Arrays.sol uint256 totalLength; for (uint256 i = 0; i < list.length; i++) { totalLength += list[i].length; @@ -159,6 +213,10 @@ library RLP { return totalLength; } + /** + * @dev Decodes an RLP `item`'s `length and type from its prefix. + * Returns the offset, length, and type of the RLP item based on the encoding rules. + */ function _decodeLength(Item memory item) private pure returns (uint256 offset, uint256 length, ItemType) { require(item.length != 0); bytes32 ptr = item.ptr; @@ -184,6 +242,7 @@ library RLP { return (offset, length, ItemType.LIST_ITEM); } + /// @dev Decodes a short string (0-55 bytes). The first byte contains the length, and the rest is the payload. function _decodeShortString( uint256 strLength, Item memory item @@ -193,6 +252,7 @@ library RLP { return (1, strLength, ItemType.DATA_ITEM); } + /// @dev Decodes a short list (0-55 bytes). The first byte contains the length of the entire list. function _decodeShortList( uint256 listLength, Item memory item @@ -201,6 +261,7 @@ library RLP { return (1, listLength, ItemType.LIST_ITEM); } + /// @dev Decodes a long string or list (>55 bytes). The first byte indicates the length of the length, followed by the length itself. function _decodeLong(uint256 lengthLength, Item memory item) private pure returns (uint256 offset, uint256 length) { lengthLength += 1; // 1 byte for the length itself require(item.length > lengthLength); @@ -237,4 +298,10 @@ library RLP { v := mload(ptr) } } + + function _buffer(bytes32 ptr) private pure returns (bytes memory buffer) { + assembly ("memory-safe") { + buffer := ptr + } + } } diff --git a/contracts/utils/TrieProof.sol b/contracts/utils/TrieProof.sol index 7071b750475..ac0e5060bba 100644 --- a/contracts/utils/TrieProof.sol +++ b/contracts/utils/TrieProof.sol @@ -5,6 +5,12 @@ import {Bytes} from "./Bytes.sol"; import {RLP} from "./RLP.sol"; import {Math} from "./math/Math.sol"; +/** + * @dev Library for verifying Ethereum Merkle-Patricia trie inclusion proofs. + * + * Ethereum's State Trie state layout is a 4-item array of `[nonce, balance, storageRoot, codeHash]` + * See https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie[Merkle-Patricia trie] + */ library TrieProof { using Bytes for bytes; using RLP for *; @@ -37,9 +43,15 @@ library TrieProof { RLP.Item[] decoded; // Decoded RLP items } - uint256 internal constant EVM_TREE_RADIX = 16; // Ethereum uses 16 as its trie radix (hex) - uint256 internal constant LEAF_OR_EXTENSION_NODE_LENGTH = 2; // Leaf and extension nodes have exactly 2 items + /// @dev The radix of the Ethereum trie (hexadecimal = 16) + uint256 internal constant EVM_TREE_RADIX = 16; + /// @dev Number of items in leaf or extension nodes (always 2) + uint256 internal constant LEAF_OR_EXTENSION_NODE_LENGTH = 2; + /** + * @dev Verifies a `proof` against a given `key`, `value`, `and root` hash + * using the default Ethereum radix (16). + */ function verify( bytes memory key, bytes memory value, @@ -49,6 +61,7 @@ library TrieProof { return verify(key, value, proof, root, EVM_TREE_RADIX); } + /// @dev Same as {verify} but with a custom radix. function verify( bytes memory key, bytes memory value, @@ -60,26 +73,28 @@ library TrieProof { return processedValue.equal(value) && err == ProofError.NO_ERROR; } + /// @dev Processes a proof for a given key using default Ethereum radix (16) and returns the processed value. function processProof( - bytes memory key_, + bytes memory key, bytes[] memory proof, bytes32 root ) internal pure returns (bytes memory value, ProofError) { - return processProof(key_, proof, root, EVM_TREE_RADIX); + return processProof(key, proof, root, EVM_TREE_RADIX); } + /// @dev Same as {processProof} but with a custom radix. function processProof( - bytes memory key_, + bytes memory key, bytes[] memory proof, bytes32 root, uint256 radix ) internal pure returns (bytes memory value, ProofError) { - if (key_.length == 0) return ("", ProofError.EMPTY_KEY); + if (key.length == 0) return ("", ProofError.EMPTY_KEY); // Convert key to nibbles (4-bit values) and begin processing from the root - return _processInclusionProof(_decodeProof(proof), key_.nibbles(), bytes.concat(root), 0, radix); + return _processInclusionProof(_decodeProof(proof), key.nibbles(), bytes.concat(root), 0, radix); } - // Main recursive function that traverses the trie using the provided proof + /// @dev Main recursive function that traverses the trie using the provided proof. function _processInclusionProof( Node[] memory trieProof, bytes memory key, @@ -138,6 +153,7 @@ library TrieProof { return ("", ProofError.INVALID_PROOF); } + /// @dev Validates the node hashes at different levels of the proof. function _validateNodeHashes( bytes memory nodeId, Node memory node, @@ -150,6 +166,10 @@ library TrieProof { return ProofError.NO_ERROR; // No error } + /** + * @dev Validates that we've reached a valid leaf value and this is the last proof element. + * Ensures the value is not empty and no extra proof elements exist. + */ function _validateLastItem( RLP.Item memory item, Node[] memory trieProof, @@ -161,6 +181,10 @@ library TrieProof { return (value_, ProofError.NO_ERROR); } + /** + * @dev Converts raw proof bytes into structured Node objects with RLP parsing. + * Transforms each proof element into a Node with both encoded and decoded forms. + */ function _decodeProof(bytes[] memory proof) private pure returns (Node[] memory proof_) { uint256 length = proof.length; proof_ = new Node[](length); @@ -169,14 +193,26 @@ library TrieProof { } } + /** + * @dev Extracts the node ID (hash or raw data based on size). + * For small nodes (<32 bytes), returns the raw bytes; for large nodes, returns the hash. + */ function _id(RLP.Item memory node) private pure returns (bytes memory) { return node.length < 32 ? node.readRawBytes() : node.readBytes(); } + /** + * @dev Extracts the path from a leaf or extension node. + * The path is stored as the first element in the node's decoded array. + */ function _path(Node memory node) private pure returns (bytes memory) { return node.decoded[0].readBytes().nibbles(); } + /** + * @dev Calculates the number of shared nibbles between two byte arrays. + * Used to determine how much of a path matches a key during trie traversal. + */ function _sharedNibbleLength(bytes memory _a, bytes memory _b) private pure returns (uint256 shared_) { uint256 max = Math.max(_a.length, _b.length); uint256 length; diff --git a/contracts/utils/math/Endianness.sol b/contracts/utils/math/Endianness.sol deleted file mode 100644 index e506fa7a7d7..00000000000 --- a/contracts/utils/math/Endianness.sol +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: MIT - -pragma solidity ^0.8.20; - -library Endianness { - function reverseUint256(uint256 value) internal pure returns (uint256) { - value = // swap bytes - ((value >> 8) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) | - ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) << 8); - value = // swap 2-byte long pairs - ((value >> 16) & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) | - ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) << 16); - value = // swap 4-byte long pairs - ((value >> 32) & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) | - ((value & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) << 32); - value = // swap 8-byte long pairs - ((value >> 64) & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) | - ((value & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) << 64); - return (value >> 128) | (value << 128); // swap 16-byte long pairs - } - - function reverseUint128(uint128 value) internal pure returns (uint256) { - value = // swap bytes - ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | - ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); - value = // swap 2-byte long pairs - ((value & 0xFFFF0000FFFF0000FFFF0000FFFF0000) >> 16) | - ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF) << 16); - value = // swap 4-byte long pairs - ((value & 0xFFFFFFFF00000000FFFFFFFF00000000) >> 32) | - ((value & 0x00000000FFFFFFFF00000000FFFFFFFF) << 32); - return (value >> 64) | (value << 64); // swap 8-byte long pairs - } - - function reverseUint64(uint64 value) internal pure returns (uint256) { - value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes - value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs - return (value >> 32) | (value << 32); // swap 4-byte long pairs - } - - function reverseUint32(uint64 value) internal pure returns (uint256) { - value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes - return (value >> 16) | (value << 16); // swap 2-byte long pairs - } - - function reverseUint16(uint64 value) internal pure returns (uint256) { - return (value >> 8) | (value << 8); - } -} diff --git a/contracts/utils/math/Math.sol b/contracts/utils/math/Math.sol index f0d608a2dea..12546593ffe 100644 --- a/contracts/utils/math/Math.sol +++ b/contracts/utils/math/Math.sol @@ -740,6 +740,58 @@ library Math { } } + /** + * @dev Reverses the byte order of a uint256 value, converting between little-endian and big-endian. + * Inspired in https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel[Reverse Parallel] + */ + function reverseBitsUint256(uint256 value) internal pure returns (uint256) { + value = // swap bytes + ((value >> 8) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) | + ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) << 8); + value = // swap 2-byte long pairs + ((value >> 16) & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) | + ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) << 16); + value = // swap 4-byte long pairs + ((value >> 32) & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) | + ((value & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) << 32); + value = // swap 8-byte long pairs + ((value >> 64) & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) | + ((value & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) << 64); + return (value >> 128) | (value << 128); // swap 16-byte long pairs + } + + /// @dev Same as {reverseBitsUint256} but optimized for 128-bit values. + function reverseBitsUint128(uint128 value) internal pure returns (uint256) { + value = // swap bytes + ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | + ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); + value = // swap 2-byte long pairs + ((value & 0xFFFF0000FFFF0000FFFF0000FFFF0000) >> 16) | + ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF) << 16); + value = // swap 4-byte long pairs + ((value & 0xFFFFFFFF00000000FFFFFFFF00000000) >> 32) | + ((value & 0x00000000FFFFFFFF00000000FFFFFFFF) << 32); + return (value >> 64) | (value << 64); // swap 8-byte long pairs + } + + /// @dev Same as {reverseBitsUint256} but optimized for 64-bit values. + function reverseBitsUint64(uint64 value) internal pure returns (uint256) { + value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes + value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs + return (value >> 32) | (value << 32); // swap 4-byte long pairs + } + + /// @dev Same as {reverseBitsUint256} but optimized for 32-bit values. + function reverseBitsUint32(uint32 value) internal pure returns (uint256) { + value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes + return (value >> 16) | (value << 16); // swap 2-byte long pairs + } + + /// @dev Same as {reverseBitsUint256} but optimized for 16-bit values. + function reverseBits16(uint16 value) internal pure returns (uint256) { + return (value >> 8) | (value << 8); + } + /** * @dev Returns whether a provided rounding mode is considered rounding up for unsigned integers. */ From 163f27ceaace88630ea0d4fb28f6e39212770ee5 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 24 May 2025 09:19:41 -0600 Subject: [PATCH 013/114] Workaround stack too deep --- contracts/utils/TrieProof.sol | 57 +++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/contracts/utils/TrieProof.sol b/contracts/utils/TrieProof.sol index ac0e5060bba..c8e121ec68b 100644 --- a/contracts/utils/TrieProof.sol +++ b/contracts/utils/TrieProof.sol @@ -126,26 +126,7 @@ library TrieProof { uint8 branchKey = uint8(key[keyIndex]); (nodeId, keyIndex) = (_id(node.decoded[branchKey]), keyIndex + 1); } else if (nodeLength == LEAF_OR_EXTENSION_NODE_LENGTH) { - bytes memory path = _path(node); - uint8 prefix = uint8(path[0]); - uint8 offset = 2 - (prefix % 2); // Calculate offset based on even/odd path length - bytes memory pathRemainder = Bytes.slice(path, offset); // Path after the prefix - bytes memory keyRemainder = Bytes.slice(key, keyIndex); // Remaining key to match - uint256 sharedNibbleLength = _sharedNibbleLength(pathRemainder, keyRemainder); - - // Path must match at least partially with our key - if (sharedNibbleLength == 0) return ("", ProofError.INVALID_PATH_REMAINDER); - if (prefix > uint8(type(Prefix).max)) return ("", ProofError.UNKNOWN_NODE_PREFIX); - - // Leaf node (terminal) - return its value if key matches completely - if (Prefix(prefix) == Prefix.LEAF_EVEN || Prefix(prefix) == Prefix.LEAF_ODD) { - if (keyRemainder.length == 0) return ("", ProofError.INVALID_KEY_REMAINDER); - return _validateLastItem(node.decoded[1], trieProof, i); - } - - // Extension node (non-terminal) - continue to next node - // Increment keyIndex by the number of nibbles consumed - (nodeId, keyIndex) = (_id(node.decoded[1]), keyIndex + sharedNibbleLength); + return _processLeafOrExtension(node, trieProof, key, nodeId, keyIndex, i); } } @@ -166,6 +147,42 @@ library TrieProof { return ProofError.NO_ERROR; // No error } + /** + * @dev Processes a leaf or extension node in the trie proof. + * + * For leaf nodes, validates that the key matches completely and returns the value. + * For extension nodes, continues traversal by updating the node ID and key index. + */ + function _processLeafOrExtension( + Node memory node, + Node[] memory trieProof, + bytes memory key, + bytes memory nodeId, + uint256 keyIndex, + uint256 i + ) private pure returns (bytes memory value, ProofError err) { + bytes memory path = _path(node); + uint8 prefix = uint8(path[0]); + uint8 offset = 2 - (prefix % 2); // Calculate offset based on even/odd path length + bytes memory pathRemainder = Bytes.slice(path, offset); // Path after the prefix + bytes memory keyRemainder = Bytes.slice(key, keyIndex); // Remaining key to match + uint256 sharedNibbleLength = _sharedNibbleLength(pathRemainder, keyRemainder); + + // Path must match at least partially with our key + if (sharedNibbleLength == 0) return ("", ProofError.INVALID_PATH_REMAINDER); + if (prefix > uint8(type(Prefix).max)) return ("", ProofError.UNKNOWN_NODE_PREFIX); + + // Leaf node (terminal) - return its value if key matches completely + if (Prefix(prefix) == Prefix.LEAF_EVEN || Prefix(prefix) == Prefix.LEAF_ODD) { + if (keyRemainder.length == 0) return ("", ProofError.INVALID_KEY_REMAINDER); + return _validateLastItem(node.decoded[1], trieProof, i); + } + + // Extension node (non-terminal) - continue to next node + // Increment keyIndex by the number of nibbles consumed + (nodeId, keyIndex) = (_id(node.decoded[1]), keyIndex + sharedNibbleLength); + } + /** * @dev Validates that we've reached a valid leaf value and this is the last proof element. * Ensures the value is not empty and no extra proof elements exist. From c48428956dbaca384e797f4af15a0baf44afe1dd Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 29 May 2025 09:08:32 -0600 Subject: [PATCH 014/114] Add Changesets --- .changeset/lovely-cooks-add.md | 5 +++++ .changeset/shaky-phones-mix.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/lovely-cooks-add.md create mode 100644 .changeset/shaky-phones-mix.md diff --git a/.changeset/lovely-cooks-add.md b/.changeset/lovely-cooks-add.md new file mode 100644 index 00000000000..6637c92478d --- /dev/null +++ b/.changeset/lovely-cooks-add.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`RLP`: Add library for Ethereum's Recursive Length Prefix encoding/decoding. diff --git a/.changeset/shaky-phones-mix.md b/.changeset/shaky-phones-mix.md new file mode 100644 index 00000000000..410af473108 --- /dev/null +++ b/.changeset/shaky-phones-mix.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`TrieProof`: Add library for verifying Ethereum Merkle-Patricia trie inclusion proofs. From e0d4790fd4c1e1b1723625604d1023b745bae7da Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 12:59:27 -0600 Subject: [PATCH 015/114] Add more changesets --- .changeset/khaki-hats-leave.md | 5 +++++ .changeset/major-feet-write.md | 5 +++++ .changeset/ten-steaks-try.md | 5 +++++ .changeset/whole-cats-find.md | 5 +++++ 4 files changed, 20 insertions(+) create mode 100644 .changeset/khaki-hats-leave.md create mode 100644 .changeset/major-feet-write.md create mode 100644 .changeset/ten-steaks-try.md create mode 100644 .changeset/whole-cats-find.md diff --git a/.changeset/khaki-hats-leave.md b/.changeset/khaki-hats-leave.md new file mode 100644 index 00000000000..021df0ff083 --- /dev/null +++ b/.changeset/khaki-hats-leave.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Bytes`: Add a `nibbles` function to split each byte into two nibbles. diff --git a/.changeset/major-feet-write.md b/.changeset/major-feet-write.md new file mode 100644 index 00000000000..da2966f00cd --- /dev/null +++ b/.changeset/major-feet-write.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Math`: Add `reverseBitsUint256`, `reverseBitsUint128`, `reverseBitsUint64`, `reverseBitsUint32`, and `reverseBits16` functions to reverse byte order for converting between little-endian and big-endian representations. diff --git a/.changeset/ten-steaks-try.md b/.changeset/ten-steaks-try.md new file mode 100644 index 00000000000..a734f5fdb45 --- /dev/null +++ b/.changeset/ten-steaks-try.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Bytes`: Add an `equal` function to compare byte buffers. diff --git a/.changeset/whole-cats-find.md b/.changeset/whole-cats-find.md new file mode 100644 index 00000000000..e170da3dc63 --- /dev/null +++ b/.changeset/whole-cats-find.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Bytes`: Add a `countLeadingZeroes` function to count the leading zero bytes in a `uint256` value. From a6f9053a7b99a863edb929bf42467b544528c6ba Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 14:54:33 -0600 Subject: [PATCH 016/114] Add FV and fuzz tests --- test/utils/Bytes.t.sol | 72 ++++++++++++++++++++++++++++++++++++++ test/utils/math/Math.t.sol | 21 +++++++++++ 2 files changed, 93 insertions(+) create mode 100644 test/utils/Bytes.t.sol diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol new file mode 100644 index 00000000000..90657e0974b --- /dev/null +++ b/test/utils/Bytes.t.sol @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +import {Test} from "forge-std/Test.sol"; +import {Math} from "@openzeppelin/contracts/utils/math/Math.sol"; +import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; + +contract BytesTest is Test { + function testIndexOf(bytes memory buffer, bytes1 s) public pure { + testIndexOf(buffer, s, 0); + } + + function testIndexOf(bytes memory buffer, bytes1 s, uint256 pos) public pure { + uint256 result = Bytes.indexOf(buffer, s, pos); + + // Should not be found before result + for (uint256 i = pos; result != type(uint256).max && i < result; i++) assertNotEq(buffer[i], s); + if (result != type(uint256).max) assertEq(buffer[result], s); + } + + function testLastIndexOf(bytes memory buffer, bytes1 s) public pure { + testLastIndexOf(buffer, s, 0); + } + + function testLastIndexOf(bytes memory buffer, bytes1 s, uint256 pos) public pure { + pos = bound(pos, 0, buffer.length); + uint256 result = Bytes.lastIndexOf(buffer, s, pos); + + // Should not be found before result + for (uint256 i = pos; result != type(uint256).max && i < result; i++) assertNotEq(buffer[i], s); + if (result != type(uint256).max) assertEq(buffer[result], s); + } + + function testSlice(bytes memory buffer, uint256 start) public pure { + testSlice(buffer, start, buffer.length); + } + + function testSlice(bytes memory buffer, uint256 start, uint256 end) public pure { + bytes memory result = Bytes.slice(buffer, start, end); + uint256 sanitizedEnd = Math.min(end, buffer.length); + uint256 sanitizedStart = Math.min(start, sanitizedEnd); + assertEq(result.length, sanitizedEnd - sanitizedStart); + for (uint256 i = 0; i < result.length; i++) assertEq(result[i], buffer[sanitizedStart + i]); + } + + function testNibbles(bytes memory value) public pure { + bytes memory result = Bytes.nibbles(value); + assertEq(result.length, value.length * 2); + for (uint256 i = 0; i < value.length; i++) { + bytes1 originalByte = value[i]; + bytes1 highNibble = result[i * 2]; + bytes1 lowNibble = result[i * 2 + 1]; + + assertEq(highNibble, originalByte & 0xf0); + assertEq(lowNibble, originalByte & 0x0f); + } + } + + function testSymbolicEqual(bytes memory a, bytes memory b) public pure { + assertEq(Bytes.equal(a, b), Bytes.equal(a, b)); + } + + function testSymbolicCountLeadingZeroes(uint256 x) public pure { + uint256 result = Bytes.countLeadingZeroes(x); + assertLe(result, 31); + uint256 firstNonZeroByte = 31 - result; + uint256 byteValue = (x >> (firstNonZeroByte * 8)) & 0xff; + assertTrue(byteValue > 0 || x == 0); + for (uint256 i = 0; i < result; i++) assertEq((x >> ((31 - i) * 8)) & 0xff, 0); + } +} diff --git a/test/utils/math/Math.t.sol b/test/utils/math/Math.t.sol index 3c83febe9df..9f501b0e367 100644 --- a/test/utils/math/Math.t.sol +++ b/test/utils/math/Math.t.sol @@ -308,6 +308,27 @@ contract MathTest is Test { } } + // REVERSE BITS + function testSymbolicReverseBitsUint256(uint256 value) public pure { + assertEq(Math.reverseBitsUint256(Math.reverseBitsUint256(value)), value); + } + + function testSymbolicReverseBitsUint128(uint128 value) public pure { + assertEq(Math.reverseBitsUint128(uint128(Math.reverseBitsUint128(value))), value); + } + + function testSymbolicReverseBitsUint64(uint64 value) public pure { + assertEq(Math.reverseBitsUint64(uint64(Math.reverseBitsUint64(value))), value); + } + + function testSymbolicReverseBitsUint32(uint32 value) public pure { + assertEq(Math.reverseBitsUint32(uint32(Math.reverseBitsUint32(value))), value); + } + + function testSymbolicReverseBits16(uint16 value) public pure { + assertEq(Math.reverseBits16(uint16(Math.reverseBits16(value))), value); + } + // Helpers function _asRounding(uint8 r) private pure returns (Math.Rounding) { vm.assume(r < uint8(type(Math.Rounding).max)); From 203d1a204b22cb09f3757ea006f88bf29e407014 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 15:00:17 -0600 Subject: [PATCH 017/114] up --- CHANGELOG.md | 2 +- contracts/utils/Strings.sol | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fec1725f61d..3506605408d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Breaking changes -- Update minimum pragma to 0.8.24 in `SignatureChecker`, `Governor` and Governor's extensions. ([#5716](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5716)). +- Update minimum pragma to 0.8.24 in `SignatureChecker`, `Governor`, Governor's extensions, and Strings. ([#5716](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5716)), ([#5680](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5680)) ## 5.3.0 (2025-04-09) diff --git a/contracts/utils/Strings.sol b/contracts/utils/Strings.sol index a865bfbc785..1b779f4aae5 100644 --- a/contracts/utils/Strings.sol +++ b/contracts/utils/Strings.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (utils/Strings.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {Math} from "./math/Math.sol"; import {SafeCast} from "./math/SafeCast.sol"; From 48eabc17eed90bdaa3589adcb21a5aea2e0bc8dd Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 15:02:53 -0600 Subject: [PATCH 018/114] docs --- contracts/utils/README.adoc | 5 ++++- contracts/utils/cryptography/README.adoc | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/contracts/utils/README.adoc b/contracts/utils/README.adoc index 231bccd9738..4834613834f 100644 --- a/contracts/utils/README.adoc +++ b/contracts/utils/README.adoc @@ -40,7 +40,8 @@ Miscellaneous contracts and libraries containing utility functions you can use t * {CAIP2}, {CAIP10}: Libraries for formatting and parsing CAIP-2 and CAIP-10 identifiers. * {Blockhash}: A library for accessing historical block hashes beyond the standard 256 block limit utilizing EIP-2935's historical blockhash functionality. * {Time}: A library that provides helpers for manipulating time-related objects, including a `Delay` type. - + * {RLP}: Library for encoding and decoding data in Ethereum's Recursive Length Prefix format. + [NOTE] ==== Because Solidity does not support generic types, {EnumerableMap} and {EnumerableSet} are specialized to a limited number of key-value types. @@ -137,3 +138,5 @@ Ethereum contracts have no native concept of an interface, so applications must {{Blockhash}} {{Time}} + +{{RLP}} diff --git a/contracts/utils/cryptography/README.adoc b/contracts/utils/cryptography/README.adoc index 79b10437322..fd822f7bbfd 100644 --- a/contracts/utils/cryptography/README.adoc +++ b/contracts/utils/cryptography/README.adoc @@ -11,6 +11,7 @@ A collection of contracts and libraries that implement various signature validat * {SignatureChecker}: A library helper to support regular ECDSA from EOAs as well as ERC-1271 signatures for smart contracts. * {Hashes}: Commonly used hash functions. * {MerkleProof}: Functions for verifying https://en.wikipedia.org/wiki/Merkle_tree[Merkle Tree] proofs. + * {TrieProof}: Library for verifying Ethereum Merkle-Patricia trie inclusion proofs. * {EIP712}: Contract with functions to allow processing signed typed structure data according to https://eips.ethereum.org/EIPS/eip-712[EIP-712]. * {ERC7739Utils}: Utilities library that implements a defensive rehashing mechanism to prevent replayability of smart contract signatures based on ERC-7739. * {AbstractSigner}: Abstract contract for internal signature validation in smart contracts. @@ -36,6 +37,8 @@ A collection of contracts and libraries that implement various signature validat {{MerkleProof}} +{{TrieProof}} + {{EIP712}} {{ERC7739Utils}} From 63ced951ebe43f5c6c5f32fba6509664ca4c9d37 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 15:10:13 -0600 Subject: [PATCH 019/114] up pragma --- CHANGELOG.md | 2 +- contracts/mocks/docs/utilities/Base64NFT.sol | 2 +- contracts/token/ERC1155/extensions/ERC1155URIStorage.sol | 2 +- contracts/token/ERC721/ERC721.sol | 2 +- contracts/token/ERC721/extensions/ERC721URIStorage.sol | 2 +- contracts/utils/cryptography/MessageHashUtils.sol | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3506605408d..e3907a82796 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Breaking changes -- Update minimum pragma to 0.8.24 in `SignatureChecker`, `Governor`, Governor's extensions, and Strings. ([#5716](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5716)), ([#5680](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5680)) +- Update minimum pragma to 0.8.24 in `SignatureChecker`, `Governor`, Governor's extensions, `Strings`, `ERC1155URIStorage`, `ERC721URIStorage`, `MessageHashUtils`. ([#5716](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5716)), ([#5680](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5680)) ## 5.3.0 (2025-04-09) diff --git a/contracts/mocks/docs/utilities/Base64NFT.sol b/contracts/mocks/docs/utilities/Base64NFT.sol index 1fb66234310..057e93a4cef 100644 --- a/contracts/mocks/docs/utilities/Base64NFT.sol +++ b/contracts/mocks/docs/utilities/Base64NFT.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../../../token/ERC721/ERC721.sol"; import {Strings} from "../../../utils/Strings.sol"; diff --git a/contracts/token/ERC1155/extensions/ERC1155URIStorage.sol b/contracts/token/ERC1155/extensions/ERC1155URIStorage.sol index 5abf319d327..85143ef099c 100644 --- a/contracts/token/ERC1155/extensions/ERC1155URIStorage.sol +++ b/contracts/token/ERC1155/extensions/ERC1155URIStorage.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC1155/extensions/ERC1155URIStorage.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {Strings} from "../../../utils/Strings.sol"; import {ERC1155} from "../ERC1155.sol"; diff --git a/contracts/token/ERC721/ERC721.sol b/contracts/token/ERC721/ERC721.sol index a757e9b05c4..9e58bbe6ed3 100644 --- a/contracts/token/ERC721/ERC721.sol +++ b/contracts/token/ERC721/ERC721.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/ERC721.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {IERC721} from "./IERC721.sol"; import {IERC721Metadata} from "./extensions/IERC721Metadata.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721URIStorage.sol b/contracts/token/ERC721/extensions/ERC721URIStorage.sol index 432fec71d77..5a42b4e774f 100644 --- a/contracts/token/ERC721/extensions/ERC721URIStorage.sol +++ b/contracts/token/ERC721/extensions/ERC721URIStorage.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (token/ERC721/extensions/ERC721URIStorage.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../ERC721.sol"; import {IERC721Metadata} from "./IERC721Metadata.sol"; diff --git a/contracts/utils/cryptography/MessageHashUtils.sol b/contracts/utils/cryptography/MessageHashUtils.sol index 37e92395f89..232090f31db 100644 --- a/contracts/utils/cryptography/MessageHashUtils.sol +++ b/contracts/utils/cryptography/MessageHashUtils.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (utils/cryptography/MessageHashUtils.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {Strings} from "../Strings.sol"; From f3427561649444d006e48cc572b98c22b3810f2f Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 15:14:43 -0600 Subject: [PATCH 020/114] Add missing Bytes test --- test/utils/Bytes.test.js | 94 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/test/utils/Bytes.test.js b/test/utils/Bytes.test.js index 52a1ae95e77..2d09059371a 100644 --- a/test/utils/Bytes.test.js +++ b/test/utils/Bytes.test.js @@ -85,4 +85,98 @@ describe('Bytes', function () { } }); }); + + describe('nibbles', function () { + it('converts single byte', async function () { + await expect(this.mock.$nibbles('0xab')).to.eventually.equal('0xa00b'); + }); + + it('converts multiple bytes', async function () { + await expect(this.mock.$nibbles('0x1234')).to.eventually.equal('0x10023004'); + }); + + it('handles empty bytes', async function () { + await expect(this.mock.$nibbles('0x')).to.eventually.equal('0x'); + }); + + it('converts lorem text', async function () { + const result = await this.mock.$nibbles(lorem); + expect(ethers.dataLength(result)).to.equal(lorem.length * 2); + + // Check nibble extraction for first few bytes + for (let i = 0; i < Math.min(lorem.length, 5); i++) { + const originalByte = lorem[i]; + const highNibble = ethers.dataSlice(result, i * 2, i * 2 + 1); + const lowNibble = ethers.dataSlice(result, i * 2 + 1, i * 2 + 2); + + expect(highNibble).to.equal(ethers.toBeHex(originalByte & 0xf0, 1)); + expect(lowNibble).to.equal(ethers.toBeHex(originalByte & 0x0f, 1)); + } + }); + }); + + describe('equal', function () { + it('identical arrays', async function () { + await expect(this.mock.$equal(lorem, lorem)).to.eventually.be.true; + }); + + it('same content', async function () { + const copy = new Uint8Array(lorem); + await expect(this.mock.$equal(lorem, copy)).to.eventually.be.true; + }); + + it('different content', async function () { + const different = ethers.toUtf8Bytes('Different content'); + await expect(this.mock.$equal(lorem, different)).to.eventually.be.false; + }); + + it('different lengths', async function () { + const shorter = lorem.slice(0, 10); + await expect(this.mock.$equal(lorem, shorter)).to.eventually.be.false; + }); + + it('empty arrays', async function () { + const empty1 = new Uint8Array(0); + const empty2 = new Uint8Array(0); + await expect(this.mock.$equal(empty1, empty2)).to.eventually.be.true; + }); + + it('one empty one not', async function () { + const empty = new Uint8Array(0); + await expect(this.mock.$equal(lorem, empty)).to.eventually.be.false; + }); + }); + + describe('countLeadingZeroes', function () { + it('zero value', async function () { + await expect(this.mock.$countLeadingZeroes(0)).to.eventually.equal(31); + }); + + it('small values', async function () { + await expect(this.mock.$countLeadingZeroes(1)).to.eventually.equal(31); + await expect(this.mock.$countLeadingZeroes(255)).to.eventually.equal(31); + }); + + it('larger values', async function () { + await expect(this.mock.$countLeadingZeroes(256)).to.eventually.equal(30); + await expect(this.mock.$countLeadingZeroes(0xff00)).to.eventually.equal(30); + await expect(this.mock.$countLeadingZeroes(0x10000)).to.eventually.equal(29); + }); + + it('max value', async function () { + await expect(this.mock.$countLeadingZeroes(ethers.MaxUint256)).to.eventually.equal(0); + }); + + it('specific patterns', async function () { + await expect( + this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000000000100'), + ).to.eventually.equal(30); + await expect( + this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000000010000'), + ).to.eventually.equal(29); + await expect( + this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000001000000'), + ).to.eventually.equal(28); + }); + }); }); From 23dba376ca2c04bb83fe807a9b4a6bbcb239d073 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 15:22:43 -0600 Subject: [PATCH 021/114] Add unit tests --- test/helpers/constants.js | 2 + test/utils/math/Math.test.js | 115 +++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/test/helpers/constants.js b/test/helpers/constants.js index eb9b43e5549..d08c3ec0455 100644 --- a/test/helpers/constants.js +++ b/test/helpers/constants.js @@ -1,5 +1,7 @@ module.exports = { + MAX_UINT16: 2n ** 16n - 1n, MAX_UINT32: 2n ** 32n - 1n, MAX_UINT48: 2n ** 48n - 1n, MAX_UINT64: 2n ** 64n - 1n, + MAX_UINT128: 2n ** 128n - 1n, }; diff --git a/test/utils/math/Math.test.js b/test/utils/math/Math.test.js index 6a09938148a..ce1abdd8a09 100644 --- a/test/utils/math/Math.test.js +++ b/test/utils/math/Math.test.js @@ -7,6 +7,7 @@ const { Rounding } = require('../../helpers/enums'); const { min, max, modExp } = require('../../helpers/math'); const { generators } = require('../../helpers/random'); const { product, range } = require('../../helpers/iterate'); +const { MAX_UINT128, MAX_UINT64, MAX_UINT32, MAX_UINT16 } = require('../../helpers/constants'); const RoundingDown = [Rounding.Floor, Rounding.Trunc]; const RoundingUp = [Rounding.Ceil, Rounding.Expand]; @@ -710,4 +711,118 @@ describe('Math', function () { }); }); }); + + describe('reverseBits', function () { + describe('reverseBitsUint256', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBitsUint256(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint256(ethers.MaxUint256)).to.eventually.equal(ethers.MaxUint256); + + // Test simple pattern + await expect( + this.mock.$reverseBitsUint256('0x0000000000000000000000000000000000000000000000000000000000000001'), + ).to.eventually.equal('0x0100000000000000000000000000000000000000000000000000000000000000'); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, ethers.MaxUint256]; + for (const value of values) { + const reversed = await this.mock.$reverseBitsUint256(value); + await expect(this.mock.$reverseBitsUint256(reversed)).to.eventually.equal(value); + } + }); + }); + + describe('reverseBitsUint128', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBitsUint128(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint128(MAX_UINT128)).to.eventually.equal(MAX_UINT128); + + // Test simple pattern + await expect(this.mock.$reverseBitsUint128('0x00000000000000000000000000000001')).to.eventually.equal( + '0x01000000000000000000000000000000', + ); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT128]; + for (const value of values) { + const reversed = await this.mock.$reverseBitsUint128(value); + // Cast back to uint128 for comparison since function returns uint256 + await expect(this.mock.$reverseBitsUint128(reversed & MAX_UINT128)).to.eventually.equal(value); + } + }); + }); + + describe('reverseBitsUint64', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBitsUint64(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint64(MAX_UINT64)).to.eventually.equal(MAX_UINT64); + + // Test known pattern: 0x123456789ABCDEF0 -> 0xF0DEBC9A78563412 + await expect(this.mock.$reverseBitsUint64('0x123456789ABCDEF0')).to.eventually.equal('0xF0DEBC9A78563412'); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT64]; + for (const value of values) { + const reversed = await this.mock.$reverseBitsUint64(value); + // Cast back to uint64 for comparison since function returns uint256 + await expect(this.mock.$reverseBitsUint64(reversed & MAX_UINT64)).to.eventually.equal(value); + } + }); + }); + + describe('reverseBitsUint32', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBitsUint32(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint32(MAX_UINT32)).to.eventually.equal(MAX_UINT32); + + // Test known pattern: 0x12345678 -> 0x78563412 + await expect(this.mock.$reverseBitsUint32(0x12345678)).to.eventually.equal(0x78563412); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT32]; + for (const value of values) { + const reversed = await this.mock.$reverseBitsUint32(value); + // Cast back to uint32 for comparison since function returns uint256 + await expect(this.mock.$reverseBitsUint32(reversed & MAX_UINT32)).to.eventually.equal(value); + } + }); + }); + + describe('reverseBits16', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBits16(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBits16(MAX_UINT16)).to.eventually.equal(MAX_UINT16); + + // Test known pattern: 0x1234 -> 0x3412 + await expect(this.mock.$reverseBits16(0x1234)).to.eventually.equal(0x3412); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x1234n, MAX_UINT16]; + for (const value of values) { + const reversed = await this.mock.$reverseBits16(value); + // Cast back to uint16 for comparison since function returns uint256 + await expect(this.mock.$reverseBits16(reversed & MAX_UINT16)).to.eventually.equal(value); + } + }); + }); + + describe('edge cases', function () { + it('handles single byte values', async function () { + await expect(this.mock.$reverseBits16(0x00ff)).to.eventually.equal(0xff00); + await expect(this.mock.$reverseBitsUint32(0x000000ff)).to.eventually.equal(0xff000000); + }); + + it('handles alternating patterns', async function () { + await expect(this.mock.$reverseBits16(0xaaaa)).to.eventually.equal(0xaaaa); + await expect(this.mock.$reverseBits16(0x5555)).to.eventually.equal(0x5555); + await expect(this.mock.$reverseBitsUint32(0xaaaaaaaa)).to.eventually.equal(0xaaaaaaaa); + await expect(this.mock.$reverseBitsUint32(0x55555555)).to.eventually.equal(0x55555555); + }); + }); + }); }); From 0cacca21ea410cd0841c898bf680e79439bd3e19 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 15:27:21 -0600 Subject: [PATCH 022/114] up pragma --- CHANGELOG.md | 2 +- contracts/mocks/docs/MyNFT.sol | 2 +- contracts/mocks/docs/token/ERC721/GameItem.sol | 2 +- contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol | 2 +- contracts/mocks/token/ERC721ConsecutiveMock.sol | 2 +- contracts/mocks/token/ERC721URIStorageMock.sol | 2 +- contracts/token/ERC721/extensions/ERC721Burnable.sol | 2 +- contracts/token/ERC721/extensions/ERC721Consecutive.sol | 2 +- contracts/token/ERC721/extensions/ERC721Enumerable.sol | 2 +- contracts/token/ERC721/extensions/ERC721Pausable.sol | 2 +- contracts/token/ERC721/extensions/ERC721Votes.sol | 2 +- contracts/token/ERC721/extensions/ERC721Wrapper.sol | 2 +- contracts/utils/cryptography/EIP712.sol | 2 +- contracts/utils/cryptography/signers/ERC7739.sol | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3907a82796..2b6a1e2a76d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Breaking changes -- Update minimum pragma to 0.8.24 in `SignatureChecker`, `Governor`, Governor's extensions, `Strings`, `ERC1155URIStorage`, `ERC721URIStorage`, `MessageHashUtils`. ([#5716](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5716)), ([#5680](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5680)) +- Update minimum pragma to 0.8.24 in `SignatureChecker`, `Governor`, Governor's extensions, `Strings`, `ERC1155URIStorage`, `MessageHashUtils`, `ERC721URIStorage`, `ERC721Votes`, `ERC721Wrapper`, `ERC721Burnable`, `ERC721Consecutive`, `ERC721Enumerable`, `ERC721Pausable`, `EIP712` and `ERC7739`. ([#5716](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5716)), ([#5680](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5680)) ## 5.3.0 (2025-04-09) diff --git a/contracts/mocks/docs/MyNFT.sol b/contracts/mocks/docs/MyNFT.sol index 1a442fa0aad..b6d982eee69 100644 --- a/contracts/mocks/docs/MyNFT.sol +++ b/contracts/mocks/docs/MyNFT.sol @@ -1,6 +1,6 @@ // contracts/MyNFT.sol // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../../token/ERC721/ERC721.sol"; diff --git a/contracts/mocks/docs/token/ERC721/GameItem.sol b/contracts/mocks/docs/token/ERC721/GameItem.sol index b7f576f1000..182e1919eea 100644 --- a/contracts/mocks/docs/token/ERC721/GameItem.sol +++ b/contracts/mocks/docs/token/ERC721/GameItem.sol @@ -1,6 +1,6 @@ // contracts/GameItem.sol // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721URIStorage, ERC721} from "../../../../token/ERC721/extensions/ERC721URIStorage.sol"; diff --git a/contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol b/contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol index 7732ae4a5d7..77fd8f66b13 100644 --- a/contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol +++ b/contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../../token/ERC721/ERC721.sol"; import {ERC721Consecutive} from "../../token/ERC721/extensions/ERC721Consecutive.sol"; diff --git a/contracts/mocks/token/ERC721ConsecutiveMock.sol b/contracts/mocks/token/ERC721ConsecutiveMock.sol index 10986471893..005a5ad6097 100644 --- a/contracts/mocks/token/ERC721ConsecutiveMock.sol +++ b/contracts/mocks/token/ERC721ConsecutiveMock.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../../token/ERC721/ERC721.sol"; import {ERC721Consecutive} from "../../token/ERC721/extensions/ERC721Consecutive.sol"; diff --git a/contracts/mocks/token/ERC721URIStorageMock.sol b/contracts/mocks/token/ERC721URIStorageMock.sol index 254435e07a8..e574fe12e02 100644 --- a/contracts/mocks/token/ERC721URIStorageMock.sol +++ b/contracts/mocks/token/ERC721URIStorageMock.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721URIStorage} from "../../token/ERC721/extensions/ERC721URIStorage.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Burnable.sol b/contracts/token/ERC721/extensions/ERC721Burnable.sol index c6d22455761..06babdcf259 100644 --- a/contracts/token/ERC721/extensions/ERC721Burnable.sol +++ b/contracts/token/ERC721/extensions/ERC721Burnable.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Burnable.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../ERC721.sol"; import {Context} from "../../../utils/Context.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Consecutive.sol b/contracts/token/ERC721/extensions/ERC721Consecutive.sol index 0f3267364f2..ec3abe02e54 100644 --- a/contracts/token/ERC721/extensions/ERC721Consecutive.sol +++ b/contracts/token/ERC721/extensions/ERC721Consecutive.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (token/ERC721/extensions/ERC721Consecutive.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../ERC721.sol"; import {IERC2309} from "../../../interfaces/IERC2309.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Enumerable.sol b/contracts/token/ERC721/extensions/ERC721Enumerable.sol index 6d699429db4..4f76f97bc60 100644 --- a/contracts/token/ERC721/extensions/ERC721Enumerable.sol +++ b/contracts/token/ERC721/extensions/ERC721Enumerable.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Enumerable.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../ERC721.sol"; import {IERC721Enumerable} from "./IERC721Enumerable.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Pausable.sol b/contracts/token/ERC721/extensions/ERC721Pausable.sol index 9a75623c62a..0404488664f 100644 --- a/contracts/token/ERC721/extensions/ERC721Pausable.sol +++ b/contracts/token/ERC721/extensions/ERC721Pausable.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Pausable.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../ERC721.sol"; import {Pausable} from "../../../utils/Pausable.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Votes.sol b/contracts/token/ERC721/extensions/ERC721Votes.sol index f71195ce7c8..4b2ddd61284 100644 --- a/contracts/token/ERC721/extensions/ERC721Votes.sol +++ b/contracts/token/ERC721/extensions/ERC721Votes.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Votes.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {ERC721} from "../ERC721.sol"; import {Votes} from "../../../governance/utils/Votes.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Wrapper.sol b/contracts/token/ERC721/extensions/ERC721Wrapper.sol index 111136bbe77..f7b9c6cc2cb 100644 --- a/contracts/token/ERC721/extensions/ERC721Wrapper.sol +++ b/contracts/token/ERC721/extensions/ERC721Wrapper.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Wrapper.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {IERC721, ERC721} from "../ERC721.sol"; import {IERC721Receiver} from "../IERC721Receiver.sol"; diff --git a/contracts/utils/cryptography/EIP712.sol b/contracts/utils/cryptography/EIP712.sol index c39954e35dd..116ba839ebb 100644 --- a/contracts/utils/cryptography/EIP712.sol +++ b/contracts/utils/cryptography/EIP712.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (utils/cryptography/EIP712.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {MessageHashUtils} from "./MessageHashUtils.sol"; import {ShortStrings, ShortString} from "../ShortStrings.sol"; diff --git a/contracts/utils/cryptography/signers/ERC7739.sol b/contracts/utils/cryptography/signers/ERC7739.sol index ff0bd4e3818..9ac79fb9a50 100644 --- a/contracts/utils/cryptography/signers/ERC7739.sol +++ b/contracts/utils/cryptography/signers/ERC7739.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {AbstractSigner} from "./AbstractSigner.sol"; import {EIP712} from "../EIP712.sol"; From 831e8ab7f582dfc296eef7b634f161c5c5292370 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sat, 7 Jun 2025 15:30:11 -0600 Subject: [PATCH 023/114] Move TrieProof --- contracts/utils/{ => cryptography}/TrieProof.sol | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename contracts/utils/{ => cryptography}/TrieProof.sol (98%) diff --git a/contracts/utils/TrieProof.sol b/contracts/utils/cryptography/TrieProof.sol similarity index 98% rename from contracts/utils/TrieProof.sol rename to contracts/utils/cryptography/TrieProof.sol index c8e121ec68b..9ee44f708f0 100644 --- a/contracts/utils/TrieProof.sol +++ b/contracts/utils/cryptography/TrieProof.sol @@ -1,9 +1,9 @@ // SPDX-License-Identifier: MIT pragma solidity ^0.8.27; -import {Bytes} from "./Bytes.sol"; -import {RLP} from "./RLP.sol"; -import {Math} from "./math/Math.sol"; +import {Bytes} from "../Bytes.sol"; +import {RLP} from "../RLP.sol"; +import {Math} from "../math/Math.sol"; /** * @dev Library for verifying Ethereum Merkle-Patricia trie inclusion proofs. From 5da111fae1ca5a08cfb94e21b4effabf370aff68 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 10:01:26 -0600 Subject: [PATCH 024/114] Fix countLeadingZeroes --- contracts/utils/Bytes.sol | 1 + test/utils/Bytes.t.sol | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 3391f31753d..a7ff88a4982 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -118,6 +118,7 @@ library Bytes { /// @dev Counts the number of leading zero bytes in a uint256. function countLeadingZeroes(uint256 x) internal pure returns (uint256) { + if (x == 0) return 32; // All 32 bytes are zero uint256 r = 0; if (x > 0xffffffffffffffffffffffffffffffff) r = 128; // Upper 128 bits if ((x >> r) > 0xffffffffffffffff) r |= 64; // Next 64 bits diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index 90657e0974b..c89856bacda 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -63,10 +63,17 @@ contract BytesTest is Test { function testSymbolicCountLeadingZeroes(uint256 x) public pure { uint256 result = Bytes.countLeadingZeroes(x); - assertLe(result, 31); - uint256 firstNonZeroByte = 31 - result; - uint256 byteValue = (x >> (firstNonZeroByte * 8)) & 0xff; - assertTrue(byteValue > 0 || x == 0); - for (uint256 i = 0; i < result; i++) assertEq((x >> ((31 - i) * 8)) & 0xff, 0); + assertLe(result, 32); // [0, 32] + + if (x != 0) { + uint256 firstNonZeroBytePos = 32 - result - 1; + uint256 byteValue = (x >> (firstNonZeroBytePos * 8)) & 0xff; + assertNotEq(byteValue, 0); + + // x != 0 implies result < 32 + // most significant byte should be non-zero + uint256 msbValue = (x >> (248 - result * 8)) & 0xff; + assertNotEq(msbValue, 0); + } } } From ba2293e6387cf0260f916c79bd61c8b434553521 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 10:12:48 -0600 Subject: [PATCH 025/114] nits --- contracts/utils/RLP.sol | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index abc671f37cb..971f97624ed 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -126,11 +126,10 @@ library RLP { require(itemType == ItemType.DATA_ITEM); require(item.length == itemOffset + itemLength); - uint256 start = itemOffset; bytes32 itemPtr = item.ptr; bytes memory result = new bytes(itemLength); assembly ("memory-safe") { - mcopy(add(result, 0x20), add(itemPtr, start), itemLength) + mcopy(add(result, 0x20), add(itemPtr, itemOffset), itemLength) } return result; @@ -155,7 +154,7 @@ library RLP { /// @dev Checks if a buffer is a single byte below 128 (0x80). Encoded as-is in RLP. function _isSingleByte(bytes memory buffer) private pure returns (bool) { - return buffer.length == 1 && uint8(buffer[0]) <= SHORT_OFFSET - 1; + return buffer.length == 1 && uint8(buffer[0]) < SHORT_OFFSET; } /** @@ -298,10 +297,4 @@ library RLP { v := mload(ptr) } } - - function _buffer(bytes32 ptr) private pure returns (bytes memory buffer) { - assembly ("memory-safe") { - buffer := ptr - } - } } From 9409bc6d9b3fb676101d6dfdcfad4dd5cb72b23a Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 10:52:39 -0600 Subject: [PATCH 026/114] Improve --- contracts/utils/RLP.sol | 59 ++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 971f97624ed..628bb5b44ea 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -13,6 +13,18 @@ library RLP { using Math for uint256; using Bytes for *; + /// @dev Items with length 0 are not RLP items. + error RLPEmptyItem(); + + /// @dev The `item` is not of the `expected` type. + error RLPUnexpectedType(ItemType expected, ItemType actual); + + /// @dev The item is not long enough to contain the data. + error RLPInvalidDataRemainder(uint256 minLength, uint256 actualLength); + + /// @dev The content length does not match the expected length. + error RLPContentLengthMismatch(uint256 expectedLength, uint256 actualLength); + struct Item { uint256 length; // Total length of the item in bytes bytes32 ptr; // Memory pointer to the start of the item @@ -77,25 +89,40 @@ library RLP { /** * @dev Convenience method to encode a boolean as RLP. - * Boolean `true` is encoded as single byte 0x01, false as an empty string (0x80). + * + * Boolean `true` is encoded as 0x01, `false` as 0x80 (equivalent to encoding integers 1 and 0). + * This follows the de facto ecosystem standard where booleans are treated as 0/1 integers. + * + * NOTE: Both this and {encodeStrict} produce identical encoded bytes at the output level. + * Use this for ecosystem compatibility; use {encodeStrict} for strict RLP spec compliance. */ function encode(bool value) internal pure returns (bytes memory) { - bytes memory encoded = new bytes(1); - encoded[0] = value ? bytes1(0x01) : bytes1(SHORT_OFFSET); // false is encoded as an empty string - return encoded; + return encode(value ? uint256(1) : uint256(0)); + } + + /** + * @dev Strict RLP encoding of a boolean following literal spec interpretation. + * Boolean `true` is encoded as 0x01, `false` as empty bytes (0x80). + * + * NOTE: This is the strict RLP spec interpretation where false represents "empty". + * Use this for strict RLP spec compliance; use {encode} for ecosystem compatibility. + */ + function encodeStrict(bool value) internal pure returns (bytes memory) { + return value ? abi.encodePacked(bytes1(0x01)) : encode(new bytes(0)); } /// @dev Creates an RLP Item from a bytes array. function toItem(bytes memory value) internal pure returns (Item memory) { - require(value.length != 0); // Empty arrays are not RLP items. + require(value.length != 0, RLPEmptyItem()); // Empty arrays are not RLP items. return Item(value.length, _skippedLengthPtr(value)); } /// @dev Decodes an RLP encoded list into an array of RLP Items. See {_decodeLength} function readList(Item memory item) internal pure returns (Item[] memory) { (uint256 listOffset, uint256 listLength, ItemType itemType) = _decodeLength(item); - require(itemType == ItemType.LIST_ITEM); - require(listOffset + listLength == item.length); + require(itemType == ItemType.LIST_ITEM, RLPUnexpectedType(ItemType.LIST_ITEM, itemType)); + uint256 expectedLength = listOffset + listLength; + require(expectedLength == item.length, RLPContentLengthMismatch(expectedLength, item.length)); Item[] memory items = new Item[](32); uint256 itemCount = item.length; @@ -123,8 +150,9 @@ library RLP { /// @dev Decodes an RLP encoded item. function readBytes(Item memory item) internal pure returns (bytes memory) { (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); - require(itemType == ItemType.DATA_ITEM); - require(item.length == itemOffset + itemLength); + require(itemType == ItemType.DATA_ITEM, RLPUnexpectedType(ItemType.DATA_ITEM, itemType)); + uint256 expectedLength = itemOffset + itemLength; + require(expectedLength == item.length, RLPContentLengthMismatch(expectedLength, item.length)); bytes32 itemPtr = item.ptr; bytes memory result = new bytes(itemLength); @@ -217,7 +245,7 @@ library RLP { * Returns the offset, length, and type of the RLP item based on the encoding rules. */ function _decodeLength(Item memory item) private pure returns (uint256 offset, uint256 length, ItemType) { - require(item.length != 0); + require(item.length != 0, RLPEmptyItem()); bytes32 ptr = item.ptr; uint256 prefix = uint8(_extractMemoryByte(ptr)); @@ -246,7 +274,7 @@ library RLP { uint256 strLength, Item memory item ) private pure returns (uint256 offset, uint256 length, ItemType) { - require(item.length > strLength); + require(item.length > strLength, RLPInvalidDataRemainder(strLength, item.length)); require(strLength != 1 || _extractMemoryByte(bytes32(uint256(item.ptr) + 1)) >= bytes1(SHORT_OFFSET)); return (1, strLength, ItemType.DATA_ITEM); } @@ -256,20 +284,21 @@ library RLP { uint256 listLength, Item memory item ) private pure returns (uint256 offset, uint256 length, ItemType) { - require(item.length > listLength); + require(item.length > listLength, RLPInvalidDataRemainder(listLength, item.length)); return (1, listLength, ItemType.LIST_ITEM); } /// @dev Decodes a long string or list (>55 bytes). The first byte indicates the length of the length, followed by the length itself. function _decodeLong(uint256 lengthLength, Item memory item) private pure returns (uint256 offset, uint256 length) { lengthLength += 1; // 1 byte for the length itself - require(item.length > lengthLength); + require(item.length > lengthLength, RLPInvalidDataRemainder(lengthLength, item.length)); require(_extractMemoryByte(item.ptr) != 0x00); // Extract the length value from the next bytes uint256 len = _extractMemoryWord(bytes32(uint256(item.ptr) + 1)) >> (256 - 8 * lengthLength); - require(len > SHORT_OFFSET); - require(item.length <= lengthLength + len); + require(len > SHORT_OFFSET, RLPInvalidDataRemainder(SHORT_OFFSET, len)); + uint256 expectedLength = lengthLength + len; + require(item.length <= expectedLength, RLPContentLengthMismatch(expectedLength, item.length)); return (lengthLength + 1, len); } From e740dac6f8758a8be6667a9de2a37422df31823d Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 11:04:13 -0600 Subject: [PATCH 027/114] Fix --- contracts/utils/RLP.sol | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 628bb5b44ea..aa991ee0add 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -125,14 +125,14 @@ library RLP { require(expectedLength == item.length, RLPContentLengthMismatch(expectedLength, item.length)); Item[] memory items = new Item[](32); - uint256 itemCount = item.length; + uint256 itemCount; - for (uint256 i; listOffset < itemCount; i++) { + for (uint256 currentOffset = listOffset; currentOffset < item.length; ++itemCount) { (uint256 itemOffset, uint256 itemLength, ) = _decodeLength( - Item(itemCount - listOffset, bytes32(uint256(item.ptr) + listOffset)) + Item(item.length - currentOffset, bytes32(uint256(item.ptr) + currentOffset)) ); - items[i] = Item(itemLength + itemOffset, bytes32(uint256(item.ptr) + listOffset)); - listOffset += itemOffset + itemLength; + items[itemCount] = Item(itemLength + itemOffset, bytes32(uint256(item.ptr) + currentOffset)); + currentOffset += itemOffset + itemLength; } // Decrease the array size to match the actual item count. @@ -296,7 +296,7 @@ library RLP { // Extract the length value from the next bytes uint256 len = _extractMemoryWord(bytes32(uint256(item.ptr) + 1)) >> (256 - 8 * lengthLength); - require(len > SHORT_OFFSET, RLPInvalidDataRemainder(SHORT_OFFSET, len)); + require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); uint256 expectedLength = lengthLength + len; require(item.length <= expectedLength, RLPContentLengthMismatch(expectedLength, item.length)); return (lengthLength + 1, len); From 0332ffe69e737a0aaef5d611463a6de0d6306f10 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 11:38:34 -0600 Subject: [PATCH 028/114] Add Memory.sol library --- contracts/utils/Memory.sol | 86 ++++++++++++++++++++++++++++++++++++++ contracts/utils/RLP.sol | 60 +++++++------------------- 2 files changed, 101 insertions(+), 45 deletions(-) create mode 100644 contracts/utils/Memory.sol diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol new file mode 100644 index 00000000000..5435e9ff5fe --- /dev/null +++ b/contracts/utils/Memory.sol @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +/** + * @dev Utilities to manipulate memory. + * + * Memory is a contiguous and dynamic byte array in which Solidity stores non-primitive types. + * This library provides functions to manipulate pointers to this dynamic array. + * + * WARNING: When manipulating memory, make sure to follow the Solidity documentation + * guidelines for https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[Memory Safety]. + */ +library Memory { + type Pointer is bytes32; + + /// @dev Returns a memory pointer to the current free memory pointer. + function getFreePointer() internal pure returns (Pointer ptr) { + assembly ("memory-safe") { + ptr := mload(0x40) + } + } + + /// @dev Sets the free memory pointer to a specific value. + /// + /// WARNING: Everything after the pointer may be overwritten. + function setFreePointer(Pointer ptr) internal pure { + assembly ("memory-safe") { + mstore(0x40, ptr) + } + } + + /// @dev Returns a memory pointer to the content of a buffer. Skips the length word. + function contentPointer(bytes memory buffer) internal pure returns (Pointer) { + bytes32 ptr; + assembly ("memory-safe") { + ptr := add(buffer, 32) + } + return asPointer(ptr); + } + + /// @dev Copies `length` bytes from `srcPtr` to `destPtr`. + function copy(Pointer destPtr, Pointer srcPtr, uint256 length) internal pure { + assembly ("memory-safe") { + mcopy(destPtr, srcPtr, length) + } + } + + /// @dev Extracts a byte from a memory pointer. + function extractByte(Pointer ptr) internal pure returns (bytes1 v) { + assembly ("memory-safe") { + v := byte(0, mload(ptr)) + } + } + + /// @dev Extracts a word from a memory pointer. + function extractWord(Pointer ptr) internal pure returns (uint256 v) { + assembly ("memory-safe") { + v := mload(ptr) + } + } + + /// @dev Adds an offset to a memory pointer. + function addOffset(Pointer ptr, uint256 offset) internal pure returns (Pointer) { + return asPointer(bytes32(uint256(asBytes32(ptr)) + offset)); + } + + /// @dev Pointer to `bytes32`. + function asBytes32(Pointer ptr) internal pure returns (bytes32) { + return Pointer.unwrap(ptr); + } + + /// @dev `bytes32` to pointer. + function asPointer(bytes32 value) internal pure returns (Pointer) { + return Pointer.wrap(value); + } + + /// @dev `bytes` to pointer. + function asPointer(bytes memory value) internal pure returns (Pointer) { + bytes32 ptr; + assembly ("memory-safe") { + ptr := value + } + return asPointer(ptr); + } +} diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index aa991ee0add..1aedf6adcb3 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -3,6 +3,7 @@ pragma solidity ^0.8.27; import {Math} from "./math/Math.sol"; import {Bytes} from "./Bytes.sol"; +import {Memory} from "./Memory.sol"; /** * @dev Library for encoding and decoding data in RLP format. @@ -12,6 +13,7 @@ import {Bytes} from "./Bytes.sol"; library RLP { using Math for uint256; using Bytes for *; + using Memory for *; /// @dev Items with length 0 are not RLP items. error RLPEmptyItem(); @@ -27,7 +29,7 @@ library RLP { struct Item { uint256 length; // Total length of the item in bytes - bytes32 ptr; // Memory pointer to the start of the item + Memory.Pointer ptr; // Memory pointer to the start of the item } enum ItemType { @@ -114,7 +116,7 @@ library RLP { /// @dev Creates an RLP Item from a bytes array. function toItem(bytes memory value) internal pure returns (Item memory) { require(value.length != 0, RLPEmptyItem()); // Empty arrays are not RLP items. - return Item(value.length, _skippedLengthPtr(value)); + return Item(value.length, value.contentPointer()); } /// @dev Decodes an RLP encoded list into an array of RLP Items. See {_decodeLength} @@ -129,9 +131,9 @@ library RLP { for (uint256 currentOffset = listOffset; currentOffset < item.length; ++itemCount) { (uint256 itemOffset, uint256 itemLength, ) = _decodeLength( - Item(item.length - currentOffset, bytes32(uint256(item.ptr) + currentOffset)) + Item(item.length - currentOffset, item.ptr.addOffset(currentOffset)) ); - items[itemCount] = Item(itemLength + itemOffset, bytes32(uint256(item.ptr) + currentOffset)); + items[itemCount] = Item(itemLength + itemOffset, item.ptr.addOffset(currentOffset)); currentOffset += itemOffset + itemLength; } @@ -154,11 +156,8 @@ library RLP { uint256 expectedLength = itemOffset + itemLength; require(expectedLength == item.length, RLPContentLengthMismatch(expectedLength, item.length)); - bytes32 itemPtr = item.ptr; bytes memory result = new bytes(itemLength); - assembly ("memory-safe") { - mcopy(add(result, 0x20), add(itemPtr, itemOffset), itemLength) - } + result.contentPointer().copy(item.ptr.addOffset(itemOffset), itemLength); return result; } @@ -170,12 +169,9 @@ library RLP { /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. function readRawBytes(Item memory item) internal pure returns (bytes memory) { - bytes32 itemPtr = item.ptr; uint256 itemLength = item.length; bytes memory result = new bytes(itemLength); - assembly ("memory-safe") { - mcopy(add(result, 0x20), itemPtr, itemLength) - } + result.contentPointer().copy(item.ptr, itemLength); return result; } @@ -220,12 +216,12 @@ library RLP { function _flatten(bytes[] memory list) private pure returns (bytes memory) { // TODO: Move to Arrays.sol bytes memory flattened = new bytes(_totalLength(list)); - bytes32 dataPtr = _skippedLengthPtr(flattened); + Memory.Pointer dataPtr = flattened.contentPointer(); for (uint256 i = 0; i < list.length; i++) { bytes memory item = list[i]; uint256 length = item.length; - _copy(dataPtr, _skippedLengthPtr(item), length); - dataPtr = bytes32(uint256(dataPtr) + length); + dataPtr.copy(item.contentPointer(), length); + dataPtr = dataPtr.addOffset(length); } return flattened; } @@ -246,8 +242,7 @@ library RLP { */ function _decodeLength(Item memory item) private pure returns (uint256 offset, uint256 length, ItemType) { require(item.length != 0, RLPEmptyItem()); - bytes32 ptr = item.ptr; - uint256 prefix = uint8(_extractMemoryByte(ptr)); + uint256 prefix = uint8(item.ptr.extractByte()); // Single byte below 128 if (prefix < SHORT_OFFSET) return (0, 1, ItemType.DATA_ITEM); @@ -275,7 +270,7 @@ library RLP { Item memory item ) private pure returns (uint256 offset, uint256 length, ItemType) { require(item.length > strLength, RLPInvalidDataRemainder(strLength, item.length)); - require(strLength != 1 || _extractMemoryByte(bytes32(uint256(item.ptr) + 1)) >= bytes1(SHORT_OFFSET)); + require(strLength != 1 || item.ptr.addOffset(1).extractByte() >= bytes1(SHORT_OFFSET)); return (1, strLength, ItemType.DATA_ITEM); } @@ -292,38 +287,13 @@ library RLP { function _decodeLong(uint256 lengthLength, Item memory item) private pure returns (uint256 offset, uint256 length) { lengthLength += 1; // 1 byte for the length itself require(item.length > lengthLength, RLPInvalidDataRemainder(lengthLength, item.length)); - require(_extractMemoryByte(item.ptr) != 0x00); + require(item.ptr.extractByte() != 0x00); // Extract the length value from the next bytes - uint256 len = _extractMemoryWord(bytes32(uint256(item.ptr) + 1)) >> (256 - 8 * lengthLength); + uint256 len = item.ptr.addOffset(1).extractWord() >> (256 - 8 * lengthLength); require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); uint256 expectedLength = lengthLength + len; require(item.length <= expectedLength, RLPContentLengthMismatch(expectedLength, item.length)); return (lengthLength + 1, len); } - - function _copy(bytes32 destPtr, bytes32 srcPtr, uint256 length) private pure returns (bytes memory src) { - assembly ("memory-safe") { - mcopy(destPtr, srcPtr, length) - src := mload(src) - } - } - - function _skippedLengthPtr(bytes memory buffer) private pure returns (bytes32 ptr) { - assembly ("memory-safe") { - ptr := add(buffer, 32) - } - } - - function _extractMemoryByte(bytes32 ptr) private pure returns (bytes1 v) { - assembly ("memory-safe") { - v := byte(0, mload(ptr)) - } - } - - function _extractMemoryWord(bytes32 ptr) private pure returns (uint256 v) { - assembly ("memory-safe") { - v := mload(ptr) - } - } } From ac92bb41b6ccb71120b40601e63bb9f490d89ead Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 13:12:54 -0600 Subject: [PATCH 029/114] up --- contracts/access/manager/AuthorityUtils.sol | 4 ---- contracts/token/ERC20/extensions/ERC4626.sol | 3 --- contracts/token/ERC20/utils/SafeERC20.sol | 7 ------- contracts/utils/cryptography/SignatureChecker.sol | 6 +----- 4 files changed, 1 insertion(+), 19 deletions(-) diff --git a/contracts/access/manager/AuthorityUtils.sol b/contracts/access/manager/AuthorityUtils.sol index 5aeed4f6285..8b0470968b9 100644 --- a/contracts/access/manager/AuthorityUtils.sol +++ b/contracts/access/manager/AuthorityUtils.sol @@ -4,7 +4,6 @@ pragma solidity ^0.8.20; import {IAuthority} from "./IAuthority.sol"; -import {Memory} from "../../utils/Memory.sol"; library AuthorityUtils { /** @@ -18,7 +17,6 @@ library AuthorityUtils { address target, bytes4 selector ) internal view returns (bool immediate, uint32 delay) { - Memory.Pointer ptr = Memory.getFreePointer(); bytes memory data = abi.encodeCall(IAuthority.canCall, (caller, target, selector)); assembly ("memory-safe") { @@ -34,7 +32,5 @@ library AuthorityUtils { delay := mul(delay, iszero(shr(32, delay))) } } - - Memory.setFreePointer(ptr); } } diff --git a/contracts/token/ERC20/extensions/ERC4626.sol b/contracts/token/ERC20/extensions/ERC4626.sol index d5b8bcb9888..6e6a57c305d 100644 --- a/contracts/token/ERC20/extensions/ERC4626.sol +++ b/contracts/token/ERC20/extensions/ERC4626.sol @@ -7,7 +7,6 @@ import {IERC20, IERC20Metadata, ERC20} from "../ERC20.sol"; import {SafeERC20} from "../utils/SafeERC20.sol"; import {IERC4626} from "../../../interfaces/IERC4626.sol"; import {Math} from "../../../utils/math/Math.sol"; -import {Memory} from "../../../utils/Memory.sol"; /** * @dev Implementation of the ERC-4626 "Tokenized Vault Standard" as defined in @@ -85,7 +84,6 @@ abstract contract ERC4626 is ERC20, IERC4626 { * @dev Attempts to fetch the asset decimals. A return value of false indicates that the attempt failed in some way. */ function _tryGetAssetDecimals(IERC20 asset_) private view returns (bool ok, uint8 assetDecimals) { - Memory.Pointer ptr = Memory.getFreePointer(); (bool success, bytes memory encodedDecimals) = address(asset_).staticcall( abi.encodeCall(IERC20Metadata.decimals, ()) ); @@ -95,7 +93,6 @@ abstract contract ERC4626 is ERC20, IERC4626 { return (true, uint8(returnedDecimals)); } } - Memory.setFreePointer(ptr); return (false, 0); } diff --git a/contracts/token/ERC20/utils/SafeERC20.sol b/contracts/token/ERC20/utils/SafeERC20.sol index bcd17bc0111..883e8d30c97 100644 --- a/contracts/token/ERC20/utils/SafeERC20.sol +++ b/contracts/token/ERC20/utils/SafeERC20.sol @@ -5,7 +5,6 @@ pragma solidity ^0.8.20; import {IERC20} from "../IERC20.sol"; import {IERC1363} from "../../../interfaces/IERC1363.sol"; -import {Memory} from "../../../utils/Memory.sol"; /** * @title SafeERC20 @@ -32,9 +31,7 @@ library SafeERC20 { * non-reverting calls are assumed to be successful. */ function safeTransfer(IERC20 token, address to, uint256 value) internal { - Memory.Pointer ptr = Memory.getFreePointer(); _callOptionalReturn(token, abi.encodeCall(token.transfer, (to, value))); - Memory.setFreePointer(ptr); } /** @@ -42,9 +39,7 @@ library SafeERC20 { * calling contract. If `token` returns no value, non-reverting calls are assumed to be successful. */ function safeTransferFrom(IERC20 token, address from, address to, uint256 value) internal { - Memory.Pointer ptr = Memory.getFreePointer(); _callOptionalReturn(token, abi.encodeCall(token.transferFrom, (from, to, value))); - Memory.setFreePointer(ptr); } /** @@ -104,14 +99,12 @@ library SafeERC20 { * set here. */ function forceApprove(IERC20 token, address spender, uint256 value) internal { - Memory.Pointer ptr = Memory.getFreePointer(); bytes memory approvalCall = abi.encodeCall(token.approve, (spender, value)); if (!_callOptionalReturnBool(token, approvalCall)) { _callOptionalReturn(token, abi.encodeCall(token.approve, (spender, 0))); _callOptionalReturn(token, approvalCall); } - Memory.setFreePointer(ptr); } /** diff --git a/contracts/utils/cryptography/SignatureChecker.sol b/contracts/utils/cryptography/SignatureChecker.sol index 1e8991a6bb9..261372f0c3d 100644 --- a/contracts/utils/cryptography/SignatureChecker.sol +++ b/contracts/utils/cryptography/SignatureChecker.sol @@ -5,7 +5,6 @@ pragma solidity ^0.8.24; import {ECDSA} from "./ECDSA.sol"; import {IERC1271} from "../../interfaces/IERC1271.sol"; -import {Memory} from "../Memory.sol"; import {IERC7913SignatureVerifier} from "../../interfaces/IERC7913.sol"; import {Bytes} from "../../utils/Bytes.sol"; @@ -51,15 +50,12 @@ library SignatureChecker { bytes32 hash, bytes memory signature ) internal view returns (bool) { - Memory.Pointer ptr = Memory.getFreePointer(); (bool success, bytes memory result) = signer.staticcall( abi.encodeCall(IERC1271.isValidSignature, (hash, signature)) ); - bool valid = (success && + return (success && result.length >= 32 && abi.decode(result, (bytes32)) == bytes32(IERC1271.isValidSignature.selector)); - Memory.setFreePointer(ptr); - return valid; } /** From 6bb96d5fcb850b240e4ae8a05473db943c379890 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 15:35:11 -0600 Subject: [PATCH 030/114] WIP: Add more Memory functions --- contracts/utils/Bytes.sol | 7 ++++ contracts/utils/Memory.sol | 66 ++++++++++++++++++++++++++++++++++--- contracts/utils/Strings.sol | 3 +- test/utils/Bytes.t.sol | 12 +++++++ test/utils/Memory.t.sol | 43 +++++++++++++++++++++--- 5 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 test/utils/Bytes.t.sol diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 1234b845513..f8c3fb2ebfa 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -99,6 +99,13 @@ library Bytes { return result; } + /** + * @dev Returns true if the two byte buffers are equal. + */ + function equal(bytes memory a, bytes memory b) internal pure returns (bool) { + return a.length == b.length && keccak256(a) == keccak256(b); + } + /** * @dev Reads a bytes32 from a bytes array without bounds checking. * diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index e5cc0e06cc8..2d8d76e85ce 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -14,14 +14,14 @@ pragma solidity ^0.8.20; library Memory { type Pointer is bytes32; - /// @dev Returns a memory pointer to the current free memory pointer. + /// @dev Returns a `Pointer` to the current free `Pointer`. function getFreePointer() internal pure returns (Pointer ptr) { assembly ("memory-safe") { ptr := mload(0x40) } } - /// @dev Sets the free memory pointer to a specific value. + /// @dev Sets the free `Pointer` to a specific value. /// /// WARNING: Everything after the pointer may be overwritten. function setFreePointer(Pointer ptr) internal pure { @@ -30,13 +30,71 @@ library Memory { } } - /// @dev Pointer to `bytes32`. + /// @dev Returns a `Pointer` to the content of a `bytes` buffer. Skips the length word. + function contentPointer(bytes memory buffer) internal pure returns (Pointer) { + return addOffset(asPointer(buffer), 32); + } + + /** + * @dev Copies `length` bytes from `srcPtr` to `destPtr`. Equivalent to https://www.evm.codes/?fork=cancun#5e[`mcopy`]. + * + * WARNING: Reading or writing beyond the allocated memory bounds of either pointer + * will result in undefined behavior and potential memory corruption. + */ + function copy(Pointer destPtr, Pointer srcPtr, uint256 length) internal pure { + assembly ("memory-safe") { + mcopy(destPtr, srcPtr, length) + } + } + + /// @dev Extracts a `bytes1` from a `Pointer`. `offset` starts from the most significant byte. + function extractByte(Pointer ptr, uint256 offset) internal pure returns (bytes1 v) { + bytes32 word = extractWord(ptr); + assembly ("memory-safe") { + v := byte(offset, word) + } + } + + /// @dev Extracts a `bytes32` from a `Pointer`. + function extractWord(Pointer ptr) internal pure returns (bytes32 v) { + assembly ("memory-safe") { + v := mload(ptr) + } + } + + /// @dev Adds an offset to a `Pointer`. + function addOffset(Pointer ptr, uint256 offset) internal pure returns (Pointer) { + return asPointer(bytes32(asUint256(ptr) + offset)); + } + + /// @dev `Pointer` to `bytes32`. function asBytes32(Pointer ptr) internal pure returns (bytes32) { return Pointer.unwrap(ptr); } - /// @dev `bytes32` to pointer. + /// @dev `Pointer` to `uint256`. + function asUint256(Pointer ptr) internal pure returns (uint256) { + return uint256(asBytes32(ptr)); + } + + /// @dev `bytes32` to `Pointer`. function asPointer(bytes32 value) internal pure returns (Pointer) { return Pointer.wrap(value); } + + /// @dev `bytes` to `Pointer`. + function asPointer(bytes memory value) internal pure returns (Pointer) { + bytes32 ptr; + assembly ("memory-safe") { + ptr := value + } + return asPointer(ptr); + } + + /// @dev `Pointer` to `bytes`. + function asBytes(Pointer ptr) internal pure returns (bytes memory b) { + assembly ("memory-safe") { + b := ptr + } + } } diff --git a/contracts/utils/Strings.sol b/contracts/utils/Strings.sol index 4cc597646f2..a865bfbc785 100644 --- a/contracts/utils/Strings.sol +++ b/contracts/utils/Strings.sol @@ -6,6 +6,7 @@ pragma solidity ^0.8.20; import {Math} from "./math/Math.sol"; import {SafeCast} from "./math/SafeCast.sol"; import {SignedMath} from "./math/SignedMath.sol"; +import {Bytes} from "./Bytes.sol"; /** * @dev String operations. @@ -132,7 +133,7 @@ library Strings { * @dev Returns true if the two strings are equal. */ function equal(string memory a, string memory b) internal pure returns (bool) { - return bytes(a).length == bytes(b).length && keccak256(bytes(a)) == keccak256(bytes(b)); + return Bytes.equal(bytes(a), bytes(b)); } /** diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol new file mode 100644 index 00000000000..6b2d7b5cad3 --- /dev/null +++ b/test/utils/Bytes.t.sol @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +import {Test} from "forge-std/Test.sol"; +import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; + +contract BytesTest is Test { + function testSymbolicEqual(bytes memory a, bytes memory b) public pure { + assertEq(Bytes.equal(a, b), Bytes.equal(a, b)); + } +} diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol index 0affe3234c4..8964c164523 100644 --- a/test/utils/Memory.t.sol +++ b/test/utils/Memory.t.sol @@ -4,16 +4,49 @@ pragma solidity ^0.8.20; import {Test} from "forge-std/Test.sol"; import {Memory} from "@openzeppelin/contracts/utils/Memory.sol"; +import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; contract MemoryTest is Test { using Memory for *; - function testSymbolicGetSetFreePointer(uint256 seed) public pure { - // - first 0x80 bytes are reserved (scratch + FMP + zero) - // - moving the free memory pointer to far causes OOG errors - bytes32 ptr = bytes32(bound(seed, 0x80, type(uint24).max)); + // - first 0x80 bytes are reserved (scratch + FMP + zero) + uint256 constant START_PTR = 0x80; + // - moving the free memory pointer to far causes OOG errors + uint256 constant END_PTR = type(uint24).max; - Memory.setFreePointer(ptr.asPointer()); + function testGetSetFreePointer(uint256 seed) public pure { + bytes32 ptr = bytes32(bound(seed, START_PTR, END_PTR)); + ptr.asPointer().setFreePointer(); assertEq(Memory.getFreePointer().asBytes32(), ptr); } + + function testSymbolicContentPointer(uint256 seed) public pure { + Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); + assertEq(ptr.asBytes().contentPointer().asBytes32(), ptr.addOffset(32).asBytes32()); + } + + // function testCopy(bytes memory data, uint256 destSeed) public pure { + // uint256 upperPtr = data.asPointer().asUint256() + data.length; + // Memory.Pointer destPtr = bytes32(bound(destSeed, upperPtr, upperPtr + 100)).asPointer(); + // Memory.copy(data.asPointer(), destPtr, data.length + 32); + // for (uint256 i = 0; i < data.length; i++) { + // assertEq(data[i], destPtr.asBytes()[i]); + // } + // } + + function testExtractByte(uint256 seed, uint256 index) public pure { + index = bound(index, 0, 31); + Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); + assertEq(ptr.extractByte(index), bytes1(ptr.asBytes32() >> (256 - index * 8))); + } + + // function testExtractWord(uint256 seed) public pure { + // Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); + // assertEq(ptr.extractWord(), ptr.asBytes32()); + // } + + // function testAddOffset(uint256 seed, uint256 offset) public pure { + // Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); + // assertEq(ptr.addOffset(offset).asUint256(), ptr.asUint256() + offset); + // } } From 860e5a819701d49f4cbb8f13ca6306e1853e6939 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 16:50:42 -0600 Subject: [PATCH 031/114] up --- contracts/utils/Memory.sol | 16 +++++--- test/utils/Memory.test.js | 78 +++++++++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 2d8d76e85ce..891754f94d1 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -47,7 +47,11 @@ library Memory { } } - /// @dev Extracts a `bytes1` from a `Pointer`. `offset` starts from the most significant byte. + /** + * @dev Extracts a `bytes1` from a `Pointer`. `offset` starts from the most significant byte. + * + * NOTE: Will return `0x00` if `offset` is larger or equal to `32`. + */ function extractByte(Pointer ptr, uint256 offset) internal pure returns (bytes1 v) { bytes32 word = extractWord(ptr); assembly ("memory-safe") { @@ -67,22 +71,22 @@ library Memory { return asPointer(bytes32(asUint256(ptr) + offset)); } - /// @dev `Pointer` to `bytes32`. + /// @dev `Pointer` to `bytes32`. Expects a pointer to a properly ABI-encoded `bytes` object. function asBytes32(Pointer ptr) internal pure returns (bytes32) { return Pointer.unwrap(ptr); } - /// @dev `Pointer` to `uint256`. + /// @dev `Pointer` to `uint256`. Expects a pointer to a properly ABI-encoded `bytes` object. function asUint256(Pointer ptr) internal pure returns (uint256) { return uint256(asBytes32(ptr)); } - /// @dev `bytes32` to `Pointer`. + /// @dev `bytes32` to `Pointer`. Expects a pointer to a properly ABI-encoded `bytes` object. function asPointer(bytes32 value) internal pure returns (Pointer) { return Pointer.wrap(value); } - /// @dev `bytes` to `Pointer`. + /// @dev Returns a `Pointer` to the `value`'s header (i.e. includes the length word). function asPointer(bytes memory value) internal pure returns (Pointer) { bytes32 ptr; assembly ("memory-safe") { @@ -91,7 +95,7 @@ library Memory { return asPointer(ptr); } - /// @dev `Pointer` to `bytes`. + /// @dev `Pointer` to `bytes`. Expects a pointer to a properly ABI-encoded `bytes` object. function asBytes(Pointer ptr) internal pure returns (bytes memory b) { assembly ("memory-safe") { b := ptr diff --git a/test/utils/Memory.test.js b/test/utils/Memory.test.js index 5698728dcfd..c6ae6ba2d76 100644 --- a/test/utils/Memory.test.js +++ b/test/utils/Memory.test.js @@ -14,28 +14,78 @@ describe('Memory', function () { }); describe('free pointer', function () { - it('sets memory pointer', async function () { - const ptr = '0x00000000000000000000000000000000000000000000000000000000000000a0'; - expect(await this.mock.$setFreePointer(ptr)).to.not.be.reverted; + it('sets free memory pointer', async function () { + const ptr = ethers.toBeHex(0xa0, 32); + await expect(this.mock.$setFreePointer(ptr)).to.not.be.reverted; }); - it('gets memory pointer', async function () { - expect(await this.mock.$getFreePointer()).to.equal( - // Default pointer - '0x0000000000000000000000000000000000000000000000000000000000000080', + it('gets free memory pointer', async function () { + await expect(this.mock.$getFreePointer()).to.eventually.equal( + ethers.toBeHex(0x80, 32), // Default pointer ); }); + }); - it('asBytes32', async function () { - const ptr = ethers.toBeHex('0x1234', 32); - await this.mock.$setFreePointer(ptr); - expect(await this.mock.$asBytes32(ptr)).to.equal(ptr); + it('extractWord extracts a word', async function () { + const ptr = await this.mock.$getFreePointer(); + await expect(this.mock.$extractWord(ptr)).to.eventually.equal(ethers.toBeHex(0, 32)); + }); + + it('extractByte extracts a byte', async function () { + const ptr = await this.mock.$getFreePointer(); + await expect(this.mock.$extractByte(ptr, 0)).to.eventually.equal(ethers.toBeHex(0, 1)); + }); + + it('contentPointer', async function () { + const data = ethers.toUtf8Bytes('hello world'); + const result = await this.mock.$contentPointer(data); + expect(result).to.equal(ethers.toBeHex(0xa0, 32)); // 0x80 is the default free pointer (length) + }); + + describe('addOffset', function () { + it('addOffset', async function () { + const basePtr = ethers.toBeHex(0x80, 32); + const offset = 32; + const expectedPtr = ethers.toBeHex(0xa0, 32); + + await expect(this.mock.$addOffset(basePtr, offset)).to.eventually.equal(expectedPtr); }); - it('asPointer', async function () { + it('addOffsetwraps around', async function () { + const basePtr = ethers.toBeHex(0x80, 32); + const offset = 256; + const expectedPtr = ethers.toBeHex(0x180, 32); + await expect(this.mock.$addOffset(basePtr, offset)).to.eventually.equal(expectedPtr); + }); + }); + + describe('pointer conversions', function () { + it('asBytes32 / asPointer', async function () { const ptr = ethers.toBeHex('0x1234', 32); - await this.mock.$setFreePointer(ptr); - expect(await this.mock.$asPointer(ptr)).to.equal(ptr); + await expect(this.mock.$asBytes32(ptr)).to.eventually.equal(ptr); + await expect(this.mock.$asPointer(ethers.Typed.bytes32(ptr))).to.eventually.equal(ptr); + }); + + it('asBytes / asPointer', async function () { + const ptr = await this.mock.$asPointer(ethers.Typed.bytes(ethers.toUtf8Bytes('hello world'))); + expect(ptr).to.equal(ethers.toBeHex(0x80, 32)); // Default free pointer + await expect(this.mock.$asBytes(ptr)).to.eventually.equal(ethers.toBeHex(0x20, 32)); + }); + + it('asUint256', async function () { + const value = 0x1234; + const ptr = ethers.toBeHex(value, 32); + await expect(this.mock.$asUint256(ptr)).to.eventually.equal(value); + }); + }); + + describe('memory operations', function () { + it('copy', async function () { + await expect(this.mock.$copy(ethers.toBeHex(0x80, 32), ethers.toBeHex(0xc0, 32), 32)).to.not.be.reverted; + }); + + it('copy with zero length', async function () { + await expect(this.mock.$copy(ethers.toBeHex(0x80, 32), ethers.toBeHex(0xc0, 32), 0)).to.not.be.reverted; }); }); }); From ecdb768fc852cc37608f4f9842cd1c9703bfa19a Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 17:01:02 -0600 Subject: [PATCH 032/114] revert --- contracts/utils/Bytes.sol | 7 ------- contracts/utils/Strings.sol | 3 +-- test/utils/Bytes.t.sol | 12 ------------ 3 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 test/utils/Bytes.t.sol diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index f8c3fb2ebfa..1234b845513 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -99,13 +99,6 @@ library Bytes { return result; } - /** - * @dev Returns true if the two byte buffers are equal. - */ - function equal(bytes memory a, bytes memory b) internal pure returns (bool) { - return a.length == b.length && keccak256(a) == keccak256(b); - } - /** * @dev Reads a bytes32 from a bytes array without bounds checking. * diff --git a/contracts/utils/Strings.sol b/contracts/utils/Strings.sol index a865bfbc785..4cc597646f2 100644 --- a/contracts/utils/Strings.sol +++ b/contracts/utils/Strings.sol @@ -6,7 +6,6 @@ pragma solidity ^0.8.20; import {Math} from "./math/Math.sol"; import {SafeCast} from "./math/SafeCast.sol"; import {SignedMath} from "./math/SignedMath.sol"; -import {Bytes} from "./Bytes.sol"; /** * @dev String operations. @@ -133,7 +132,7 @@ library Strings { * @dev Returns true if the two strings are equal. */ function equal(string memory a, string memory b) internal pure returns (bool) { - return Bytes.equal(bytes(a), bytes(b)); + return bytes(a).length == bytes(b).length && keccak256(bytes(a)) == keccak256(bytes(b)); } /** diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol deleted file mode 100644 index 6b2d7b5cad3..00000000000 --- a/test/utils/Bytes.t.sol +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: MIT - -pragma solidity ^0.8.20; - -import {Test} from "forge-std/Test.sol"; -import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; - -contract BytesTest is Test { - function testSymbolicEqual(bytes memory a, bytes memory b) public pure { - assertEq(Bytes.equal(a, b), Bytes.equal(a, b)); - } -} From 95907aa286018d536920104790000c5f8ac6b478 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 17:11:23 -0600 Subject: [PATCH 033/114] Update docs --- docs/modules/ROOT/pages/utilities.adoc | 49 +++++++++++++++++--------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/docs/modules/ROOT/pages/utilities.adoc b/docs/modules/ROOT/pages/utilities.adoc index 24c79276bc8..68358f8ec2a 100644 --- a/docs/modules/ROOT/pages/utilities.adoc +++ b/docs/modules/ROOT/pages/utilities.adoc @@ -463,37 +463,52 @@ await instance.multicall([ === Memory -The `Memory` library provides functions for advanced use cases that require granular memory management. A common use case is to avoid unnecessary memory expansion costs when iterating over a section of the code that allocates new memory. Consider the following example: +The `Memory` library provides functions for advanced use cases that require granular memory management. A common use case is to avoid unnecessary memory expansion costs when performing repeated operations that allocate memory in a loop. Consider the following example: [source,solidity] ---- -function callFoo(address target) internal { - bytes memory callData = abi.encodeWithSelector( - bytes4(keccak256("foo()")) - ) - (bool success, /* bytes memory returndata */) = target.call(callData); - require(success); +function processMultipleItems(uint256[] memory items) internal { + for (uint256 i = 0; i < items.length; i++) { + bytes memory tempData = abi.encode(items[i], block.timestamp); + // Process tempData... + } } ---- -Note the function allocates memory for both the `callData` argument and for the returndata even if it's ignored. As such, it may be desirable to reset the free memory pointer after the end of the function. +Note that each iteration allocates new memory for `tempData`, causing the memory to expand continuously. This can be optimized by resetting the memory pointer between iterations: [source,solidity] ---- -function callFoo(address target) internal { +function processMultipleItems(uint256[] memory items) internal { Memory.Pointer ptr = Memory.getFreePointer(); // Cache pointer - bytes memory callData = abi.encodeWithSelector( - bytes4(keccak256("foo()")) - ) - (bool success, /* bytes memory returndata */) = target.call(callData); - require(success); - Memory.setFreePointer(ptr); // Reset pointer + for (uint256 i = 0; i < items.length; i++) { + bytes memory tempData = abi.encode(items[i], block.timestamp); + // Process tempData... + Memory.setFreePointer(ptr); // Reset pointer for reuse + } } ---- -This way, memory is allocated to accommodate the `callData`, and the `returndata` is freed. This allows other memory operations to reuse that space, thus reducing the memory expansion costs of these operations. In particular, this allows many `callFoo` to be performed in a loop with limited memory expansion costs. +This way, memory allocated for `tempData` in each iteration is reused, significantly reducing memory expansion costs when processing many items. -IMPORTANT: By default, Solidity handles memory safely. Using this library without understanding how memory works may be dangerous. Consider thoroughly reading the Solidity documentation about the https://docs.soliditylang.org/en/v0.8.20/internals/layout_in_memory.html[memory layout] and how the language defines https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[memory safety]. +==== Copying memory buffers + +The `Memory` library provides a `copy` function that allows copying data between memory locations. This is useful when you need to extract a segment of data from a larger buffer or when you want to avoid unnecessary memory allocations. The following example demonstrates how to copy a segment of data from a source buffer: + +[source,solidity] +---- +function copyDataSegment(bytes memory source, uint256 offset, uint256 length) + internal pure returns (bytes memory result) { + + result = new bytes(length); + Memory.Pointer srcPtr = Memory.addOffset(Memory.contentPointer(source), offset); + Memory.Pointer destPtr = Memory.contentPointer(result); + + Memory.copy(destPtr, srcPtr, length); +} +---- + +IMPORTANT: Manual memory management increases gas costs and prevents compiler optimizations. Only use these functions after profiling confirms they're necessary. By default, Solidity handles memory safely - using this library without understanding memory layout and safety may be dangerous. See the https://docs.soliditylang.org/en/v0.8.20/internals/layout_in_memory.html[memory layout] and https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[memory safety] documentation for details. === Historical Block Hashes From 124cceee184dc01c1d50301e7ef46a686696d988 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 17:12:00 -0600 Subject: [PATCH 034/114] Nit --- docs/modules/ROOT/pages/utilities.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/utilities.adoc b/docs/modules/ROOT/pages/utilities.adoc index 68358f8ec2a..6d42ddc914d 100644 --- a/docs/modules/ROOT/pages/utilities.adoc +++ b/docs/modules/ROOT/pages/utilities.adoc @@ -463,7 +463,7 @@ await instance.multicall([ === Memory -The `Memory` library provides functions for advanced use cases that require granular memory management. A common use case is to avoid unnecessary memory expansion costs when performing repeated operations that allocate memory in a loop. Consider the following example: +The xref:api:utils.adoc#Memory[`Memory`] library provides functions for advanced use cases that require granular memory management. A common use case is to avoid unnecessary memory expansion costs when performing repeated operations that allocate memory in a loop. Consider the following example: [source,solidity] ---- From c3237dfbaa2cbb9855179d7fc689dbec4cbaa080 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 18:07:37 -0600 Subject: [PATCH 035/114] Finish fuzz tests and FV --- test/utils/Memory.t.sol | 53 ++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol index 8964c164523..dcdc015ea28 100644 --- a/test/utils/Memory.t.sol +++ b/test/utils/Memory.t.sol @@ -4,7 +4,6 @@ pragma solidity ^0.8.20; import {Test} from "forge-std/Test.sol"; import {Memory} from "@openzeppelin/contracts/utils/Memory.sol"; -import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; contract MemoryTest is Test { using Memory for *; @@ -25,28 +24,44 @@ contract MemoryTest is Test { assertEq(ptr.asBytes().contentPointer().asBytes32(), ptr.addOffset(32).asBytes32()); } - // function testCopy(bytes memory data, uint256 destSeed) public pure { - // uint256 upperPtr = data.asPointer().asUint256() + data.length; - // Memory.Pointer destPtr = bytes32(bound(destSeed, upperPtr, upperPtr + 100)).asPointer(); - // Memory.copy(data.asPointer(), destPtr, data.length + 32); - // for (uint256 i = 0; i < data.length; i++) { - // assertEq(data[i], destPtr.asBytes()[i]); - // } - // } + function testCopy(bytes memory data, uint256 destSeed) public pure { + uint256 minDestPtr = Memory.getFreePointer().asUint256(); + Memory.Pointer destPtr = bytes32(bound(destSeed, minDestPtr, minDestPtr + END_PTR)).asPointer(); + destPtr.addOffset(data.length + 32).setFreePointer(); + destPtr.copy(data.asPointer(), data.length + 32); + bytes memory copiedData = destPtr.asBytes(); + assertEq(data.length, copiedData.length); + for (uint256 i = 0; i < data.length; i++) { + assertEq(data[i], copiedData[i]); + } + } - function testExtractByte(uint256 seed, uint256 index) public pure { + function testExtractByte(uint256 seed, uint256 index, bytes32 value) public pure { index = bound(index, 0, 31); Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); - assertEq(ptr.extractByte(index), bytes1(ptr.asBytes32() >> (256 - index * 8))); + + assembly ("memory-safe") { + mstore(ptr, value) + } + + bytes1 expected; + assembly ("memory-safe") { + expected := byte(index, value) + } + assertEq(ptr.extractByte(index), expected); } - // function testExtractWord(uint256 seed) public pure { - // Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); - // assertEq(ptr.extractWord(), ptr.asBytes32()); - // } + function testExtractWord(uint256 seed, bytes32 value) public pure { + Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); + assembly ("memory-safe") { + mstore(ptr, value) + } + assertEq(ptr.extractWord(), value); + } - // function testAddOffset(uint256 seed, uint256 offset) public pure { - // Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); - // assertEq(ptr.addOffset(offset).asUint256(), ptr.asUint256() + offset); - // } + function testSymbolicAddOffset(uint256 seed, uint256 offset) public pure { + offset = bound(offset, 0, type(uint256).max - END_PTR); + Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); + assertEq(ptr.addOffset(offset).asUint256(), ptr.asUint256() + offset); + } } From 27f0a9b2926df3170c208914a0c027abe5ef936d Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 18:31:49 -0600 Subject: [PATCH 036/114] up --- contracts/utils/Memory.sol | 6 +++--- test/utils/Memory.t.sol | 8 ++++---- test/utils/Memory.test.js | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 891754f94d1..84071f4d16b 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -52,15 +52,15 @@ library Memory { * * NOTE: Will return `0x00` if `offset` is larger or equal to `32`. */ - function extractByte(Pointer ptr, uint256 offset) internal pure returns (bytes1 v) { - bytes32 word = extractWord(ptr); + function loadByte(Pointer ptr, uint256 offset) internal pure returns (bytes1 v) { + bytes32 word = load(ptr); assembly ("memory-safe") { v := byte(offset, word) } } /// @dev Extracts a `bytes32` from a `Pointer`. - function extractWord(Pointer ptr) internal pure returns (bytes32 v) { + function load(Pointer ptr) internal pure returns (bytes32 v) { assembly ("memory-safe") { v := mload(ptr) } diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol index dcdc015ea28..3a663d2c95d 100644 --- a/test/utils/Memory.t.sol +++ b/test/utils/Memory.t.sol @@ -36,7 +36,7 @@ contract MemoryTest is Test { } } - function testExtractByte(uint256 seed, uint256 index, bytes32 value) public pure { + function testLoadByte(uint256 seed, uint256 index, bytes32 value) public pure { index = bound(index, 0, 31); Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); @@ -48,15 +48,15 @@ contract MemoryTest is Test { assembly ("memory-safe") { expected := byte(index, value) } - assertEq(ptr.extractByte(index), expected); + assertEq(ptr.loadByte(index), expected); } - function testExtractWord(uint256 seed, bytes32 value) public pure { + function testLoad(uint256 seed, bytes32 value) public pure { Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); assembly ("memory-safe") { mstore(ptr, value) } - assertEq(ptr.extractWord(), value); + assertEq(ptr.load(), value); } function testSymbolicAddOffset(uint256 seed, uint256 offset) public pure { diff --git a/test/utils/Memory.test.js b/test/utils/Memory.test.js index c6ae6ba2d76..7b675d40672 100644 --- a/test/utils/Memory.test.js +++ b/test/utils/Memory.test.js @@ -26,14 +26,14 @@ describe('Memory', function () { }); }); - it('extractWord extracts a word', async function () { + it('load extracts a word', async function () { const ptr = await this.mock.$getFreePointer(); - await expect(this.mock.$extractWord(ptr)).to.eventually.equal(ethers.toBeHex(0, 32)); + await expect(this.mock.$load(ptr)).to.eventually.equal(ethers.toBeHex(0, 32)); }); - it('extractByte extracts a byte', async function () { + it('loadByte extracts a byte', async function () { const ptr = await this.mock.$getFreePointer(); - await expect(this.mock.$extractByte(ptr, 0)).to.eventually.equal(ethers.toBeHex(0, 1)); + await expect(this.mock.$loadByte(ptr, 0)).to.eventually.equal(ethers.toBeHex(0, 1)); }); it('contentPointer', async function () { From 282ce39e6c65c7bc72f3dcef438dc6b8d68e5529 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 18:57:34 -0600 Subject: [PATCH 037/114] up --- test/utils/Bytes.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils/Bytes.test.js b/test/utils/Bytes.test.js index 2d09059371a..a496d0518f7 100644 --- a/test/utils/Bytes.test.js +++ b/test/utils/Bytes.test.js @@ -149,7 +149,7 @@ describe('Bytes', function () { describe('countLeadingZeroes', function () { it('zero value', async function () { - await expect(this.mock.$countLeadingZeroes(0)).to.eventually.equal(31); + await expect(this.mock.$countLeadingZeroes(0)).to.eventually.equal(32); }); it('small values', async function () { From bdd2cf116eda32365aa1679c147331599dbb4967 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 19:02:49 -0600 Subject: [PATCH 038/114] Add operations to Math.sol --- .changeset/major-feet-write.md | 5 ++ contracts/utils/math/Math.sol | 52 +++++++++++++++ test/helpers/constants.js | 2 + test/utils/math/Math.t.sol | 21 ++++++ test/utils/math/Math.test.js | 115 +++++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+) create mode 100644 .changeset/major-feet-write.md diff --git a/.changeset/major-feet-write.md b/.changeset/major-feet-write.md new file mode 100644 index 00000000000..da2966f00cd --- /dev/null +++ b/.changeset/major-feet-write.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Math`: Add `reverseBitsUint256`, `reverseBitsUint128`, `reverseBitsUint64`, `reverseBitsUint32`, and `reverseBits16` functions to reverse byte order for converting between little-endian and big-endian representations. diff --git a/contracts/utils/math/Math.sol b/contracts/utils/math/Math.sol index f0d608a2dea..12546593ffe 100644 --- a/contracts/utils/math/Math.sol +++ b/contracts/utils/math/Math.sol @@ -740,6 +740,58 @@ library Math { } } + /** + * @dev Reverses the byte order of a uint256 value, converting between little-endian and big-endian. + * Inspired in https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel[Reverse Parallel] + */ + function reverseBitsUint256(uint256 value) internal pure returns (uint256) { + value = // swap bytes + ((value >> 8) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) | + ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) << 8); + value = // swap 2-byte long pairs + ((value >> 16) & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) | + ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) << 16); + value = // swap 4-byte long pairs + ((value >> 32) & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) | + ((value & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) << 32); + value = // swap 8-byte long pairs + ((value >> 64) & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) | + ((value & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) << 64); + return (value >> 128) | (value << 128); // swap 16-byte long pairs + } + + /// @dev Same as {reverseBitsUint256} but optimized for 128-bit values. + function reverseBitsUint128(uint128 value) internal pure returns (uint256) { + value = // swap bytes + ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | + ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); + value = // swap 2-byte long pairs + ((value & 0xFFFF0000FFFF0000FFFF0000FFFF0000) >> 16) | + ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF) << 16); + value = // swap 4-byte long pairs + ((value & 0xFFFFFFFF00000000FFFFFFFF00000000) >> 32) | + ((value & 0x00000000FFFFFFFF00000000FFFFFFFF) << 32); + return (value >> 64) | (value << 64); // swap 8-byte long pairs + } + + /// @dev Same as {reverseBitsUint256} but optimized for 64-bit values. + function reverseBitsUint64(uint64 value) internal pure returns (uint256) { + value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes + value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs + return (value >> 32) | (value << 32); // swap 4-byte long pairs + } + + /// @dev Same as {reverseBitsUint256} but optimized for 32-bit values. + function reverseBitsUint32(uint32 value) internal pure returns (uint256) { + value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes + return (value >> 16) | (value << 16); // swap 2-byte long pairs + } + + /// @dev Same as {reverseBitsUint256} but optimized for 16-bit values. + function reverseBits16(uint16 value) internal pure returns (uint256) { + return (value >> 8) | (value << 8); + } + /** * @dev Returns whether a provided rounding mode is considered rounding up for unsigned integers. */ diff --git a/test/helpers/constants.js b/test/helpers/constants.js index eb9b43e5549..d08c3ec0455 100644 --- a/test/helpers/constants.js +++ b/test/helpers/constants.js @@ -1,5 +1,7 @@ module.exports = { + MAX_UINT16: 2n ** 16n - 1n, MAX_UINT32: 2n ** 32n - 1n, MAX_UINT48: 2n ** 48n - 1n, MAX_UINT64: 2n ** 64n - 1n, + MAX_UINT128: 2n ** 128n - 1n, }; diff --git a/test/utils/math/Math.t.sol b/test/utils/math/Math.t.sol index 3c83febe9df..9f501b0e367 100644 --- a/test/utils/math/Math.t.sol +++ b/test/utils/math/Math.t.sol @@ -308,6 +308,27 @@ contract MathTest is Test { } } + // REVERSE BITS + function testSymbolicReverseBitsUint256(uint256 value) public pure { + assertEq(Math.reverseBitsUint256(Math.reverseBitsUint256(value)), value); + } + + function testSymbolicReverseBitsUint128(uint128 value) public pure { + assertEq(Math.reverseBitsUint128(uint128(Math.reverseBitsUint128(value))), value); + } + + function testSymbolicReverseBitsUint64(uint64 value) public pure { + assertEq(Math.reverseBitsUint64(uint64(Math.reverseBitsUint64(value))), value); + } + + function testSymbolicReverseBitsUint32(uint32 value) public pure { + assertEq(Math.reverseBitsUint32(uint32(Math.reverseBitsUint32(value))), value); + } + + function testSymbolicReverseBits16(uint16 value) public pure { + assertEq(Math.reverseBits16(uint16(Math.reverseBits16(value))), value); + } + // Helpers function _asRounding(uint8 r) private pure returns (Math.Rounding) { vm.assume(r < uint8(type(Math.Rounding).max)); diff --git a/test/utils/math/Math.test.js b/test/utils/math/Math.test.js index 6a09938148a..ce1abdd8a09 100644 --- a/test/utils/math/Math.test.js +++ b/test/utils/math/Math.test.js @@ -7,6 +7,7 @@ const { Rounding } = require('../../helpers/enums'); const { min, max, modExp } = require('../../helpers/math'); const { generators } = require('../../helpers/random'); const { product, range } = require('../../helpers/iterate'); +const { MAX_UINT128, MAX_UINT64, MAX_UINT32, MAX_UINT16 } = require('../../helpers/constants'); const RoundingDown = [Rounding.Floor, Rounding.Trunc]; const RoundingUp = [Rounding.Ceil, Rounding.Expand]; @@ -710,4 +711,118 @@ describe('Math', function () { }); }); }); + + describe('reverseBits', function () { + describe('reverseBitsUint256', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBitsUint256(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint256(ethers.MaxUint256)).to.eventually.equal(ethers.MaxUint256); + + // Test simple pattern + await expect( + this.mock.$reverseBitsUint256('0x0000000000000000000000000000000000000000000000000000000000000001'), + ).to.eventually.equal('0x0100000000000000000000000000000000000000000000000000000000000000'); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, ethers.MaxUint256]; + for (const value of values) { + const reversed = await this.mock.$reverseBitsUint256(value); + await expect(this.mock.$reverseBitsUint256(reversed)).to.eventually.equal(value); + } + }); + }); + + describe('reverseBitsUint128', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBitsUint128(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint128(MAX_UINT128)).to.eventually.equal(MAX_UINT128); + + // Test simple pattern + await expect(this.mock.$reverseBitsUint128('0x00000000000000000000000000000001')).to.eventually.equal( + '0x01000000000000000000000000000000', + ); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT128]; + for (const value of values) { + const reversed = await this.mock.$reverseBitsUint128(value); + // Cast back to uint128 for comparison since function returns uint256 + await expect(this.mock.$reverseBitsUint128(reversed & MAX_UINT128)).to.eventually.equal(value); + } + }); + }); + + describe('reverseBitsUint64', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBitsUint64(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint64(MAX_UINT64)).to.eventually.equal(MAX_UINT64); + + // Test known pattern: 0x123456789ABCDEF0 -> 0xF0DEBC9A78563412 + await expect(this.mock.$reverseBitsUint64('0x123456789ABCDEF0')).to.eventually.equal('0xF0DEBC9A78563412'); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT64]; + for (const value of values) { + const reversed = await this.mock.$reverseBitsUint64(value); + // Cast back to uint64 for comparison since function returns uint256 + await expect(this.mock.$reverseBitsUint64(reversed & MAX_UINT64)).to.eventually.equal(value); + } + }); + }); + + describe('reverseBitsUint32', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBitsUint32(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint32(MAX_UINT32)).to.eventually.equal(MAX_UINT32); + + // Test known pattern: 0x12345678 -> 0x78563412 + await expect(this.mock.$reverseBitsUint32(0x12345678)).to.eventually.equal(0x78563412); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT32]; + for (const value of values) { + const reversed = await this.mock.$reverseBitsUint32(value); + // Cast back to uint32 for comparison since function returns uint256 + await expect(this.mock.$reverseBitsUint32(reversed & MAX_UINT32)).to.eventually.equal(value); + } + }); + }); + + describe('reverseBits16', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBits16(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBits16(MAX_UINT16)).to.eventually.equal(MAX_UINT16); + + // Test known pattern: 0x1234 -> 0x3412 + await expect(this.mock.$reverseBits16(0x1234)).to.eventually.equal(0x3412); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x1234n, MAX_UINT16]; + for (const value of values) { + const reversed = await this.mock.$reverseBits16(value); + // Cast back to uint16 for comparison since function returns uint256 + await expect(this.mock.$reverseBits16(reversed & MAX_UINT16)).to.eventually.equal(value); + } + }); + }); + + describe('edge cases', function () { + it('handles single byte values', async function () { + await expect(this.mock.$reverseBits16(0x00ff)).to.eventually.equal(0xff00); + await expect(this.mock.$reverseBitsUint32(0x000000ff)).to.eventually.equal(0xff000000); + }); + + it('handles alternating patterns', async function () { + await expect(this.mock.$reverseBits16(0xaaaa)).to.eventually.equal(0xaaaa); + await expect(this.mock.$reverseBits16(0x5555)).to.eventually.equal(0x5555); + await expect(this.mock.$reverseBitsUint32(0xaaaaaaaa)).to.eventually.equal(0xaaaaaaaa); + await expect(this.mock.$reverseBitsUint32(0x55555555)).to.eventually.equal(0x55555555); + }); + }); + }); }); From 42c79f1416f29a76c2c55fc7335a65d382d0b8b6 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 19:07:31 -0600 Subject: [PATCH 039/114] Add new equal, nibbles and countLeadingZeroes functions --- .changeset/khaki-hats-leave.md | 5 ++ .changeset/ten-steaks-try.md | 5 ++ .changeset/whole-cats-find.md | 5 ++ contracts/utils/Bytes.sol | 29 +++++++++++ contracts/utils/Strings.sol | 3 +- test/utils/Bytes.t.sol | 77 ++++++++++++++++++++++++++++ test/utils/Bytes.test.js | 94 ++++++++++++++++++++++++++++++++++ 7 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 .changeset/khaki-hats-leave.md create mode 100644 .changeset/ten-steaks-try.md create mode 100644 .changeset/whole-cats-find.md create mode 100644 test/utils/Bytes.t.sol diff --git a/.changeset/khaki-hats-leave.md b/.changeset/khaki-hats-leave.md new file mode 100644 index 00000000000..021df0ff083 --- /dev/null +++ b/.changeset/khaki-hats-leave.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Bytes`: Add a `nibbles` function to split each byte into two nibbles. diff --git a/.changeset/ten-steaks-try.md b/.changeset/ten-steaks-try.md new file mode 100644 index 00000000000..a734f5fdb45 --- /dev/null +++ b/.changeset/ten-steaks-try.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Bytes`: Add an `equal` function to compare byte buffers. diff --git a/.changeset/whole-cats-find.md b/.changeset/whole-cats-find.md new file mode 100644 index 00000000000..e170da3dc63 --- /dev/null +++ b/.changeset/whole-cats-find.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Bytes`: Add a `countLeadingZeroes` function to count the leading zero bytes in a `uint256` value. diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 1234b845513..a7ff88a4982 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -99,6 +99,35 @@ library Bytes { return result; } + /// @dev Split each byte in `value` into two nibbles (4 bits each). + function nibbles(bytes memory value) internal pure returns (bytes memory) { + uint256 length = value.length; + bytes memory nibbles_ = new bytes(length * 2); + for (uint256 i = 0; i < length; i++) { + (nibbles_[i * 2], nibbles_[i * 2 + 1]) = (value[i] & 0xf0, value[i] & 0x0f); + } + return nibbles_; + } + + /** + * @dev Returns true if the two byte buffers are equal. + */ + function equal(bytes memory a, bytes memory b) internal pure returns (bool) { + return a.length == b.length && keccak256(a) == keccak256(b); + } + + /// @dev Counts the number of leading zero bytes in a uint256. + function countLeadingZeroes(uint256 x) internal pure returns (uint256) { + if (x == 0) return 32; // All 32 bytes are zero + uint256 r = 0; + if (x > 0xffffffffffffffffffffffffffffffff) r = 128; // Upper 128 bits + if ((x >> r) > 0xffffffffffffffff) r |= 64; // Next 64 bits + if ((x >> r) > 0xffffffff) r |= 32; // Next 32 bits + if ((x >> r) > 0xffff) r |= 16; // Next 16 bits + if ((x >> r) > 0xff) r |= 8; // Next 8 bits + return 31 ^ (r >> 3); // Convert to leading zero bytes count + } + /** * @dev Reads a bytes32 from a bytes array without bounds checking. * diff --git a/contracts/utils/Strings.sol b/contracts/utils/Strings.sol index 4cc597646f2..a865bfbc785 100644 --- a/contracts/utils/Strings.sol +++ b/contracts/utils/Strings.sol @@ -6,6 +6,7 @@ pragma solidity ^0.8.20; import {Math} from "./math/Math.sol"; import {SafeCast} from "./math/SafeCast.sol"; import {SignedMath} from "./math/SignedMath.sol"; +import {Bytes} from "./Bytes.sol"; /** * @dev String operations. @@ -132,7 +133,7 @@ library Strings { * @dev Returns true if the two strings are equal. */ function equal(string memory a, string memory b) internal pure returns (bool) { - return bytes(a).length == bytes(b).length && keccak256(bytes(a)) == keccak256(bytes(b)); + return Bytes.equal(bytes(a), bytes(b)); } /** diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol new file mode 100644 index 00000000000..e473ec4ff6a --- /dev/null +++ b/test/utils/Bytes.t.sol @@ -0,0 +1,77 @@ +pragma solidity ^0.8.20; + +import {Test} from "forge-std/Test.sol"; +import {Math} from "@openzeppelin/contracts/utils/math/Math.sol"; +import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; + +contract BytesTest is Test { + function testIndexOf(bytes memory buffer, bytes1 s) public pure { + testIndexOf(buffer, s, 0); + } + + function testIndexOf(bytes memory buffer, bytes1 s, uint256 pos) public pure { + uint256 result = Bytes.indexOf(buffer, s, pos); + + // Should not be found before result + for (uint256 i = pos; result != type(uint256).max && i < result; i++) assertNotEq(buffer[i], s); + if (result != type(uint256).max) assertEq(buffer[result], s); + } + + function testLastIndexOf(bytes memory buffer, bytes1 s) public pure { + testLastIndexOf(buffer, s, 0); + } + + function testLastIndexOf(bytes memory buffer, bytes1 s, uint256 pos) public pure { + pos = bound(pos, 0, buffer.length); + uint256 result = Bytes.lastIndexOf(buffer, s, pos); + + // Should not be found before result + for (uint256 i = pos; result != type(uint256).max && i < result; i++) assertNotEq(buffer[i], s); + if (result != type(uint256).max) assertEq(buffer[result], s); + } + + function testSlice(bytes memory buffer, uint256 start) public pure { + testSlice(buffer, start, buffer.length); + } + + function testSlice(bytes memory buffer, uint256 start, uint256 end) public pure { + bytes memory result = Bytes.slice(buffer, start, end); + uint256 sanitizedEnd = Math.min(end, buffer.length); + uint256 sanitizedStart = Math.min(start, sanitizedEnd); + assertEq(result.length, sanitizedEnd - sanitizedStart); + for (uint256 i = 0; i < result.length; i++) assertEq(result[i], buffer[sanitizedStart + i]); + } + + function testNibbles(bytes memory value) public pure { + bytes memory result = Bytes.nibbles(value); + assertEq(result.length, value.length * 2); + for (uint256 i = 0; i < value.length; i++) { + bytes1 originalByte = value[i]; + bytes1 highNibble = result[i * 2]; + bytes1 lowNibble = result[i * 2 + 1]; + + assertEq(highNibble, originalByte & 0xf0); + assertEq(lowNibble, originalByte & 0x0f); + } + } + + function testSymbolicEqual(bytes memory a, bytes memory b) public pure { + assertEq(Bytes.equal(a, b), Bytes.equal(a, b)); + } + + function testSymbolicCountLeadingZeroes(uint256 x) public pure { + uint256 result = Bytes.countLeadingZeroes(x); + assertLe(result, 32); // [0, 32] + + if (x != 0) { + uint256 firstNonZeroBytePos = 32 - result - 1; + uint256 byteValue = (x >> (firstNonZeroBytePos * 8)) & 0xff; + assertNotEq(byteValue, 0); + + // x != 0 implies result < 32 + // most significant byte should be non-zero + uint256 msbValue = (x >> (248 - result * 8)) & 0xff; + assertNotEq(msbValue, 0); + } + } +} diff --git a/test/utils/Bytes.test.js b/test/utils/Bytes.test.js index 52a1ae95e77..a496d0518f7 100644 --- a/test/utils/Bytes.test.js +++ b/test/utils/Bytes.test.js @@ -85,4 +85,98 @@ describe('Bytes', function () { } }); }); + + describe('nibbles', function () { + it('converts single byte', async function () { + await expect(this.mock.$nibbles('0xab')).to.eventually.equal('0xa00b'); + }); + + it('converts multiple bytes', async function () { + await expect(this.mock.$nibbles('0x1234')).to.eventually.equal('0x10023004'); + }); + + it('handles empty bytes', async function () { + await expect(this.mock.$nibbles('0x')).to.eventually.equal('0x'); + }); + + it('converts lorem text', async function () { + const result = await this.mock.$nibbles(lorem); + expect(ethers.dataLength(result)).to.equal(lorem.length * 2); + + // Check nibble extraction for first few bytes + for (let i = 0; i < Math.min(lorem.length, 5); i++) { + const originalByte = lorem[i]; + const highNibble = ethers.dataSlice(result, i * 2, i * 2 + 1); + const lowNibble = ethers.dataSlice(result, i * 2 + 1, i * 2 + 2); + + expect(highNibble).to.equal(ethers.toBeHex(originalByte & 0xf0, 1)); + expect(lowNibble).to.equal(ethers.toBeHex(originalByte & 0x0f, 1)); + } + }); + }); + + describe('equal', function () { + it('identical arrays', async function () { + await expect(this.mock.$equal(lorem, lorem)).to.eventually.be.true; + }); + + it('same content', async function () { + const copy = new Uint8Array(lorem); + await expect(this.mock.$equal(lorem, copy)).to.eventually.be.true; + }); + + it('different content', async function () { + const different = ethers.toUtf8Bytes('Different content'); + await expect(this.mock.$equal(lorem, different)).to.eventually.be.false; + }); + + it('different lengths', async function () { + const shorter = lorem.slice(0, 10); + await expect(this.mock.$equal(lorem, shorter)).to.eventually.be.false; + }); + + it('empty arrays', async function () { + const empty1 = new Uint8Array(0); + const empty2 = new Uint8Array(0); + await expect(this.mock.$equal(empty1, empty2)).to.eventually.be.true; + }); + + it('one empty one not', async function () { + const empty = new Uint8Array(0); + await expect(this.mock.$equal(lorem, empty)).to.eventually.be.false; + }); + }); + + describe('countLeadingZeroes', function () { + it('zero value', async function () { + await expect(this.mock.$countLeadingZeroes(0)).to.eventually.equal(32); + }); + + it('small values', async function () { + await expect(this.mock.$countLeadingZeroes(1)).to.eventually.equal(31); + await expect(this.mock.$countLeadingZeroes(255)).to.eventually.equal(31); + }); + + it('larger values', async function () { + await expect(this.mock.$countLeadingZeroes(256)).to.eventually.equal(30); + await expect(this.mock.$countLeadingZeroes(0xff00)).to.eventually.equal(30); + await expect(this.mock.$countLeadingZeroes(0x10000)).to.eventually.equal(29); + }); + + it('max value', async function () { + await expect(this.mock.$countLeadingZeroes(ethers.MaxUint256)).to.eventually.equal(0); + }); + + it('specific patterns', async function () { + await expect( + this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000000000100'), + ).to.eventually.equal(30); + await expect( + this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000000010000'), + ).to.eventually.equal(29); + await expect( + this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000001000000'), + ).to.eventually.equal(28); + }); + }); }); From 5754ab890369b0667293263b404ecd780808bca9 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 19:08:02 -0600 Subject: [PATCH 040/114] Rename countLeadingZeroes to clz --- .changeset/whole-cats-find.md | 2 +- contracts/utils/Bytes.sol | 2 +- test/utils/Bytes.t.sol | 2 +- test/utils/Bytes.test.js | 22 +++++++++++----------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.changeset/whole-cats-find.md b/.changeset/whole-cats-find.md index e170da3dc63..e5ba8df6e5d 100644 --- a/.changeset/whole-cats-find.md +++ b/.changeset/whole-cats-find.md @@ -2,4 +2,4 @@ 'openzeppelin-solidity': minor --- -`Bytes`: Add a `countLeadingZeroes` function to count the leading zero bytes in a `uint256` value. +`Bytes`: Add a `clz` function to count the leading zero bytes in a `uint256` value. diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index a7ff88a4982..633a9cc913b 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -117,7 +117,7 @@ library Bytes { } /// @dev Counts the number of leading zero bytes in a uint256. - function countLeadingZeroes(uint256 x) internal pure returns (uint256) { + function clz(uint256 x) internal pure returns (uint256) { if (x == 0) return 32; // All 32 bytes are zero uint256 r = 0; if (x > 0xffffffffffffffffffffffffffffffff) r = 128; // Upper 128 bits diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index e473ec4ff6a..73c63b70bb3 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -60,7 +60,7 @@ contract BytesTest is Test { } function testSymbolicCountLeadingZeroes(uint256 x) public pure { - uint256 result = Bytes.countLeadingZeroes(x); + uint256 result = Bytes.clz(x); assertLe(result, 32); // [0, 32] if (x != 0) { diff --git a/test/utils/Bytes.test.js b/test/utils/Bytes.test.js index a496d0518f7..05ba530d94d 100644 --- a/test/utils/Bytes.test.js +++ b/test/utils/Bytes.test.js @@ -147,35 +147,35 @@ describe('Bytes', function () { }); }); - describe('countLeadingZeroes', function () { + describe('clz', function () { it('zero value', async function () { - await expect(this.mock.$countLeadingZeroes(0)).to.eventually.equal(32); + await expect(this.mock.$clz(0)).to.eventually.equal(32); }); it('small values', async function () { - await expect(this.mock.$countLeadingZeroes(1)).to.eventually.equal(31); - await expect(this.mock.$countLeadingZeroes(255)).to.eventually.equal(31); + await expect(this.mock.$clz(1)).to.eventually.equal(31); + await expect(this.mock.$clz(255)).to.eventually.equal(31); }); it('larger values', async function () { - await expect(this.mock.$countLeadingZeroes(256)).to.eventually.equal(30); - await expect(this.mock.$countLeadingZeroes(0xff00)).to.eventually.equal(30); - await expect(this.mock.$countLeadingZeroes(0x10000)).to.eventually.equal(29); + await expect(this.mock.$clz(256)).to.eventually.equal(30); + await expect(this.mock.$clz(0xff00)).to.eventually.equal(30); + await expect(this.mock.$clz(0x10000)).to.eventually.equal(29); }); it('max value', async function () { - await expect(this.mock.$countLeadingZeroes(ethers.MaxUint256)).to.eventually.equal(0); + await expect(this.mock.$clz(ethers.MaxUint256)).to.eventually.equal(0); }); it('specific patterns', async function () { await expect( - this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000000000100'), + this.mock.$clz('0x0000000000000000000000000000000000000000000000000000000000000100'), ).to.eventually.equal(30); await expect( - this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000000010000'), + this.mock.$clz('0x0000000000000000000000000000000000000000000000000000000000010000'), ).to.eventually.equal(29); await expect( - this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000001000000'), + this.mock.$clz('0x0000000000000000000000000000000000000000000000000000000001000000'), ).to.eventually.equal(28); }); }); From 44f0e14036e66d3fa73eab585f31315b012c116d Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 19:25:08 -0600 Subject: [PATCH 041/114] up --- .changeset/whole-cats-find.md | 2 +- contracts/utils/Bytes.sol | 2 +- contracts/utils/RLP.sol | 2 +- test/utils/Bytes.t.sol | 2 +- test/utils/Bytes.test.js | 22 +++++++++++----------- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.changeset/whole-cats-find.md b/.changeset/whole-cats-find.md index e170da3dc63..e5ba8df6e5d 100644 --- a/.changeset/whole-cats-find.md +++ b/.changeset/whole-cats-find.md @@ -2,4 +2,4 @@ 'openzeppelin-solidity': minor --- -`Bytes`: Add a `countLeadingZeroes` function to count the leading zero bytes in a `uint256` value. +`Bytes`: Add a `clz` function to count the leading zero bytes in a `uint256` value. diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index a7ff88a4982..633a9cc913b 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -117,7 +117,7 @@ library Bytes { } /// @dev Counts the number of leading zero bytes in a uint256. - function countLeadingZeroes(uint256 x) internal pure returns (uint256) { + function clz(uint256 x) internal pure returns (uint256) { if (x == 0) return 32; // All 32 bytes are zero uint256 r = 0; if (x > 0xffffffffffffffffffffffffffffffff) r = 128; // Upper 128 bits diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 1aedf6adcb3..e5203a24062 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -209,7 +209,7 @@ library RLP { /// @dev Converts a uint256 to minimal binary representation, removing leading zeros. function _binaryBuffer(uint256 value) private pure returns (bytes memory) { - return abi.encodePacked(value).slice(value.countLeadingZeroes()); + return abi.encodePacked(value).slice(value.clz()); } /// @dev Concatenates all byte arrays in the `list` sequentially. Returns a flattened buffer. diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index c89856bacda..cec57caafd3 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -62,7 +62,7 @@ contract BytesTest is Test { } function testSymbolicCountLeadingZeroes(uint256 x) public pure { - uint256 result = Bytes.countLeadingZeroes(x); + uint256 result = Bytes.clz(x); assertLe(result, 32); // [0, 32] if (x != 0) { diff --git a/test/utils/Bytes.test.js b/test/utils/Bytes.test.js index a496d0518f7..05ba530d94d 100644 --- a/test/utils/Bytes.test.js +++ b/test/utils/Bytes.test.js @@ -147,35 +147,35 @@ describe('Bytes', function () { }); }); - describe('countLeadingZeroes', function () { + describe('clz', function () { it('zero value', async function () { - await expect(this.mock.$countLeadingZeroes(0)).to.eventually.equal(32); + await expect(this.mock.$clz(0)).to.eventually.equal(32); }); it('small values', async function () { - await expect(this.mock.$countLeadingZeroes(1)).to.eventually.equal(31); - await expect(this.mock.$countLeadingZeroes(255)).to.eventually.equal(31); + await expect(this.mock.$clz(1)).to.eventually.equal(31); + await expect(this.mock.$clz(255)).to.eventually.equal(31); }); it('larger values', async function () { - await expect(this.mock.$countLeadingZeroes(256)).to.eventually.equal(30); - await expect(this.mock.$countLeadingZeroes(0xff00)).to.eventually.equal(30); - await expect(this.mock.$countLeadingZeroes(0x10000)).to.eventually.equal(29); + await expect(this.mock.$clz(256)).to.eventually.equal(30); + await expect(this.mock.$clz(0xff00)).to.eventually.equal(30); + await expect(this.mock.$clz(0x10000)).to.eventually.equal(29); }); it('max value', async function () { - await expect(this.mock.$countLeadingZeroes(ethers.MaxUint256)).to.eventually.equal(0); + await expect(this.mock.$clz(ethers.MaxUint256)).to.eventually.equal(0); }); it('specific patterns', async function () { await expect( - this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000000000100'), + this.mock.$clz('0x0000000000000000000000000000000000000000000000000000000000000100'), ).to.eventually.equal(30); await expect( - this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000000010000'), + this.mock.$clz('0x0000000000000000000000000000000000000000000000000000000000010000'), ).to.eventually.equal(29); await expect( - this.mock.$countLeadingZeroes('0x0000000000000000000000000000000000000000000000000000000001000000'), + this.mock.$clz('0x0000000000000000000000000000000000000000000000000000000001000000'), ).to.eventually.equal(28); }); }); From 05c73bdc3d7bef3c6725c41bddc4c03f50f7dcba Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 19:27:55 -0600 Subject: [PATCH 042/114] Pragma changes --- .changeset/ten-steaks-try.md | 5 ----- contracts/mocks/docs/MyNFT.sol | 2 +- contracts/mocks/docs/token/ERC721/GameItem.sol | 2 +- contracts/mocks/docs/utilities/Base64NFT.sol | 2 +- contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol | 2 +- contracts/mocks/token/ERC721ConsecutiveMock.sol | 2 +- contracts/mocks/token/ERC721URIStorageMock.sol | 2 +- contracts/token/ERC1155/extensions/ERC1155URIStorage.sol | 2 +- contracts/token/ERC721/ERC721.sol | 2 +- contracts/token/ERC721/extensions/ERC721Burnable.sol | 2 +- contracts/token/ERC721/extensions/ERC721Consecutive.sol | 2 +- contracts/token/ERC721/extensions/ERC721Enumerable.sol | 2 +- contracts/token/ERC721/extensions/ERC721Pausable.sol | 2 +- contracts/token/ERC721/extensions/ERC721URIStorage.sol | 2 +- contracts/token/ERC721/extensions/ERC721Votes.sol | 2 +- contracts/token/ERC721/extensions/ERC721Wrapper.sol | 2 +- contracts/utils/Bytes.sol | 7 ------- contracts/utils/Strings.sol | 3 +-- contracts/utils/cryptography/EIP712.sol | 2 +- contracts/utils/cryptography/MessageHashUtils.sol | 2 +- contracts/utils/cryptography/signers/ERC7739.sol | 2 +- 21 files changed, 19 insertions(+), 32 deletions(-) delete mode 100644 .changeset/ten-steaks-try.md diff --git a/.changeset/ten-steaks-try.md b/.changeset/ten-steaks-try.md deleted file mode 100644 index a734f5fdb45..00000000000 --- a/.changeset/ten-steaks-try.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'openzeppelin-solidity': minor ---- - -`Bytes`: Add an `equal` function to compare byte buffers. diff --git a/contracts/mocks/docs/MyNFT.sol b/contracts/mocks/docs/MyNFT.sol index b6d982eee69..1a442fa0aad 100644 --- a/contracts/mocks/docs/MyNFT.sol +++ b/contracts/mocks/docs/MyNFT.sol @@ -1,6 +1,6 @@ // contracts/MyNFT.sol // SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../../token/ERC721/ERC721.sol"; diff --git a/contracts/mocks/docs/token/ERC721/GameItem.sol b/contracts/mocks/docs/token/ERC721/GameItem.sol index 182e1919eea..b7f576f1000 100644 --- a/contracts/mocks/docs/token/ERC721/GameItem.sol +++ b/contracts/mocks/docs/token/ERC721/GameItem.sol @@ -1,6 +1,6 @@ // contracts/GameItem.sol // SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721URIStorage, ERC721} from "../../../../token/ERC721/extensions/ERC721URIStorage.sol"; diff --git a/contracts/mocks/docs/utilities/Base64NFT.sol b/contracts/mocks/docs/utilities/Base64NFT.sol index 057e93a4cef..1fb66234310 100644 --- a/contracts/mocks/docs/utilities/Base64NFT.sol +++ b/contracts/mocks/docs/utilities/Base64NFT.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../../../token/ERC721/ERC721.sol"; import {Strings} from "../../../utils/Strings.sol"; diff --git a/contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol b/contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol index 77fd8f66b13..7732ae4a5d7 100644 --- a/contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol +++ b/contracts/mocks/token/ERC721ConsecutiveEnumerableMock.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../../token/ERC721/ERC721.sol"; import {ERC721Consecutive} from "../../token/ERC721/extensions/ERC721Consecutive.sol"; diff --git a/contracts/mocks/token/ERC721ConsecutiveMock.sol b/contracts/mocks/token/ERC721ConsecutiveMock.sol index 005a5ad6097..10986471893 100644 --- a/contracts/mocks/token/ERC721ConsecutiveMock.sol +++ b/contracts/mocks/token/ERC721ConsecutiveMock.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../../token/ERC721/ERC721.sol"; import {ERC721Consecutive} from "../../token/ERC721/extensions/ERC721Consecutive.sol"; diff --git a/contracts/mocks/token/ERC721URIStorageMock.sol b/contracts/mocks/token/ERC721URIStorageMock.sol index e574fe12e02..254435e07a8 100644 --- a/contracts/mocks/token/ERC721URIStorageMock.sol +++ b/contracts/mocks/token/ERC721URIStorageMock.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721URIStorage} from "../../token/ERC721/extensions/ERC721URIStorage.sol"; diff --git a/contracts/token/ERC1155/extensions/ERC1155URIStorage.sol b/contracts/token/ERC1155/extensions/ERC1155URIStorage.sol index 85143ef099c..5abf319d327 100644 --- a/contracts/token/ERC1155/extensions/ERC1155URIStorage.sol +++ b/contracts/token/ERC1155/extensions/ERC1155URIStorage.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC1155/extensions/ERC1155URIStorage.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {Strings} from "../../../utils/Strings.sol"; import {ERC1155} from "../ERC1155.sol"; diff --git a/contracts/token/ERC721/ERC721.sol b/contracts/token/ERC721/ERC721.sol index 9e58bbe6ed3..a757e9b05c4 100644 --- a/contracts/token/ERC721/ERC721.sol +++ b/contracts/token/ERC721/ERC721.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/ERC721.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {IERC721} from "./IERC721.sol"; import {IERC721Metadata} from "./extensions/IERC721Metadata.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Burnable.sol b/contracts/token/ERC721/extensions/ERC721Burnable.sol index 06babdcf259..c6d22455761 100644 --- a/contracts/token/ERC721/extensions/ERC721Burnable.sol +++ b/contracts/token/ERC721/extensions/ERC721Burnable.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Burnable.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../ERC721.sol"; import {Context} from "../../../utils/Context.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Consecutive.sol b/contracts/token/ERC721/extensions/ERC721Consecutive.sol index ec3abe02e54..0f3267364f2 100644 --- a/contracts/token/ERC721/extensions/ERC721Consecutive.sol +++ b/contracts/token/ERC721/extensions/ERC721Consecutive.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (token/ERC721/extensions/ERC721Consecutive.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../ERC721.sol"; import {IERC2309} from "../../../interfaces/IERC2309.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Enumerable.sol b/contracts/token/ERC721/extensions/ERC721Enumerable.sol index 4f76f97bc60..6d699429db4 100644 --- a/contracts/token/ERC721/extensions/ERC721Enumerable.sol +++ b/contracts/token/ERC721/extensions/ERC721Enumerable.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Enumerable.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../ERC721.sol"; import {IERC721Enumerable} from "./IERC721Enumerable.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Pausable.sol b/contracts/token/ERC721/extensions/ERC721Pausable.sol index 0404488664f..9a75623c62a 100644 --- a/contracts/token/ERC721/extensions/ERC721Pausable.sol +++ b/contracts/token/ERC721/extensions/ERC721Pausable.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Pausable.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../ERC721.sol"; import {Pausable} from "../../../utils/Pausable.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721URIStorage.sol b/contracts/token/ERC721/extensions/ERC721URIStorage.sol index 5a42b4e774f..432fec71d77 100644 --- a/contracts/token/ERC721/extensions/ERC721URIStorage.sol +++ b/contracts/token/ERC721/extensions/ERC721URIStorage.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (token/ERC721/extensions/ERC721URIStorage.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../ERC721.sol"; import {IERC721Metadata} from "./IERC721Metadata.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Votes.sol b/contracts/token/ERC721/extensions/ERC721Votes.sol index 4b2ddd61284..f71195ce7c8 100644 --- a/contracts/token/ERC721/extensions/ERC721Votes.sol +++ b/contracts/token/ERC721/extensions/ERC721Votes.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Votes.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {ERC721} from "../ERC721.sol"; import {Votes} from "../../../governance/utils/Votes.sol"; diff --git a/contracts/token/ERC721/extensions/ERC721Wrapper.sol b/contracts/token/ERC721/extensions/ERC721Wrapper.sol index f7b9c6cc2cb..111136bbe77 100644 --- a/contracts/token/ERC721/extensions/ERC721Wrapper.sol +++ b/contracts/token/ERC721/extensions/ERC721Wrapper.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.1.0) (token/ERC721/extensions/ERC721Wrapper.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {IERC721, ERC721} from "../ERC721.sol"; import {IERC721Receiver} from "../IERC721Receiver.sol"; diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 633a9cc913b..b6829f666f4 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -109,13 +109,6 @@ library Bytes { return nibbles_; } - /** - * @dev Returns true if the two byte buffers are equal. - */ - function equal(bytes memory a, bytes memory b) internal pure returns (bool) { - return a.length == b.length && keccak256(a) == keccak256(b); - } - /// @dev Counts the number of leading zero bytes in a uint256. function clz(uint256 x) internal pure returns (uint256) { if (x == 0) return 32; // All 32 bytes are zero diff --git a/contracts/utils/Strings.sol b/contracts/utils/Strings.sol index 1b779f4aae5..65e349f034e 100644 --- a/contracts/utils/Strings.sol +++ b/contracts/utils/Strings.sol @@ -6,7 +6,6 @@ pragma solidity ^0.8.24; import {Math} from "./math/Math.sol"; import {SafeCast} from "./math/SafeCast.sol"; import {SignedMath} from "./math/SignedMath.sol"; -import {Bytes} from "./Bytes.sol"; /** * @dev String operations. @@ -133,7 +132,7 @@ library Strings { * @dev Returns true if the two strings are equal. */ function equal(string memory a, string memory b) internal pure returns (bool) { - return Bytes.equal(bytes(a), bytes(b)); + return a.length == b.length && keccak256(bytes(a)) == keccak256(bytes(b)); } /** diff --git a/contracts/utils/cryptography/EIP712.sol b/contracts/utils/cryptography/EIP712.sol index 116ba839ebb..c39954e35dd 100644 --- a/contracts/utils/cryptography/EIP712.sol +++ b/contracts/utils/cryptography/EIP712.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (utils/cryptography/EIP712.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {MessageHashUtils} from "./MessageHashUtils.sol"; import {ShortStrings, ShortString} from "../ShortStrings.sol"; diff --git a/contracts/utils/cryptography/MessageHashUtils.sol b/contracts/utils/cryptography/MessageHashUtils.sol index 232090f31db..37e92395f89 100644 --- a/contracts/utils/cryptography/MessageHashUtils.sol +++ b/contracts/utils/cryptography/MessageHashUtils.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.3.0) (utils/cryptography/MessageHashUtils.sol) -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {Strings} from "../Strings.sol"; diff --git a/contracts/utils/cryptography/signers/ERC7739.sol b/contracts/utils/cryptography/signers/ERC7739.sol index 9ac79fb9a50..ff0bd4e3818 100644 --- a/contracts/utils/cryptography/signers/ERC7739.sol +++ b/contracts/utils/cryptography/signers/ERC7739.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; +pragma solidity ^0.8.20; import {AbstractSigner} from "./AbstractSigner.sol"; import {EIP712} from "../EIP712.sol"; From 3a6fbf639a0b3e237d029912ccb272ee3a2cf844 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Sun, 8 Jun 2025 19:28:34 -0600 Subject: [PATCH 043/114] up --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b6a1e2a76d..fec1725f61d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Breaking changes -- Update minimum pragma to 0.8.24 in `SignatureChecker`, `Governor`, Governor's extensions, `Strings`, `ERC1155URIStorage`, `MessageHashUtils`, `ERC721URIStorage`, `ERC721Votes`, `ERC721Wrapper`, `ERC721Burnable`, `ERC721Consecutive`, `ERC721Enumerable`, `ERC721Pausable`, `EIP712` and `ERC7739`. ([#5716](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5716)), ([#5680](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5680)) +- Update minimum pragma to 0.8.24 in `SignatureChecker`, `Governor` and Governor's extensions. ([#5716](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5716)). ## 5.3.0 (2025-04-09) From e67e8b4fd020806f3c77ea68dab22020c0459e58 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Fri, 4 Jul 2025 15:59:36 -0600 Subject: [PATCH 044/114] up --- contracts/utils/Memory.sol | 4 ++-- docs/modules/ROOT/pages/utilities.adoc | 4 ++-- test/utils/Memory.t.sol | 10 +++++----- test/utils/Memory.test.js | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 84071f4d16b..d787608dc63 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -15,7 +15,7 @@ library Memory { type Pointer is bytes32; /// @dev Returns a `Pointer` to the current free `Pointer`. - function getFreePointer() internal pure returns (Pointer ptr) { + function getFreeMemoryPointer() internal pure returns (Pointer ptr) { assembly ("memory-safe") { ptr := mload(0x40) } @@ -24,7 +24,7 @@ library Memory { /// @dev Sets the free `Pointer` to a specific value. /// /// WARNING: Everything after the pointer may be overwritten. - function setFreePointer(Pointer ptr) internal pure { + function setFreeMemoryPointer(Pointer ptr) internal pure { assembly ("memory-safe") { mstore(0x40, ptr) } diff --git a/docs/modules/ROOT/pages/utilities.adoc b/docs/modules/ROOT/pages/utilities.adoc index 6d42ddc914d..ee34c0c4c03 100644 --- a/docs/modules/ROOT/pages/utilities.adoc +++ b/docs/modules/ROOT/pages/utilities.adoc @@ -480,11 +480,11 @@ Note that each iteration allocates new memory for `tempData`, causing the memory [source,solidity] ---- function processMultipleItems(uint256[] memory items) internal { - Memory.Pointer ptr = Memory.getFreePointer(); // Cache pointer + Memory.Pointer ptr = Memory.getFreeMemoryPointer(); // Cache pointer for (uint256 i = 0; i < items.length; i++) { bytes memory tempData = abi.encode(items[i], block.timestamp); // Process tempData... - Memory.setFreePointer(ptr); // Reset pointer for reuse + Memory.setFreeMemoryPointer(ptr); // Reset pointer for reuse } } ---- diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol index 3a663d2c95d..016f328c41b 100644 --- a/test/utils/Memory.t.sol +++ b/test/utils/Memory.t.sol @@ -13,10 +13,10 @@ contract MemoryTest is Test { // - moving the free memory pointer to far causes OOG errors uint256 constant END_PTR = type(uint24).max; - function testGetSetFreePointer(uint256 seed) public pure { + function testGetsetFreeMemoryPointer(uint256 seed) public pure { bytes32 ptr = bytes32(bound(seed, START_PTR, END_PTR)); - ptr.asPointer().setFreePointer(); - assertEq(Memory.getFreePointer().asBytes32(), ptr); + ptr.asPointer().setFreeMemoryPointer(); + assertEq(Memory.getFreeMemoryPointer().asBytes32(), ptr); } function testSymbolicContentPointer(uint256 seed) public pure { @@ -25,9 +25,9 @@ contract MemoryTest is Test { } function testCopy(bytes memory data, uint256 destSeed) public pure { - uint256 minDestPtr = Memory.getFreePointer().asUint256(); + uint256 minDestPtr = Memory.getFreeMemoryPointer().asUint256(); Memory.Pointer destPtr = bytes32(bound(destSeed, minDestPtr, minDestPtr + END_PTR)).asPointer(); - destPtr.addOffset(data.length + 32).setFreePointer(); + destPtr.addOffset(data.length + 32).setFreeMemoryPointer(); destPtr.copy(data.asPointer(), data.length + 32); bytes memory copiedData = destPtr.asBytes(); assertEq(data.length, copiedData.length); diff --git a/test/utils/Memory.test.js b/test/utils/Memory.test.js index 7b675d40672..cd687e2f37c 100644 --- a/test/utils/Memory.test.js +++ b/test/utils/Memory.test.js @@ -16,23 +16,23 @@ describe('Memory', function () { describe('free pointer', function () { it('sets free memory pointer', async function () { const ptr = ethers.toBeHex(0xa0, 32); - await expect(this.mock.$setFreePointer(ptr)).to.not.be.reverted; + await expect(this.mock.$setFreeMemoryPointer(ptr)).to.not.be.reverted; }); it('gets free memory pointer', async function () { - await expect(this.mock.$getFreePointer()).to.eventually.equal( + await expect(this.mock.$getFreeMemoryPointer()).to.eventually.equal( ethers.toBeHex(0x80, 32), // Default pointer ); }); }); it('load extracts a word', async function () { - const ptr = await this.mock.$getFreePointer(); + const ptr = await this.mock.$getFreeMemoryPointer(); await expect(this.mock.$load(ptr)).to.eventually.equal(ethers.toBeHex(0, 32)); }); it('loadByte extracts a byte', async function () { - const ptr = await this.mock.$getFreePointer(); + const ptr = await this.mock.$getFreeMemoryPointer(); await expect(this.mock.$loadByte(ptr, 0)).to.eventually.equal(ethers.toBeHex(0, 1)); }); From 33857183a08d86d23dc8b2a961b260f5a8232896 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:04:04 -0600 Subject: [PATCH 045/114] Rename to in Math library and update corresponding tests for consistency --- contracts/utils/math/Math.sol | 2 +- test/utils/math/Math.t.sol | 4 ++-- test/utils/math/Math.test.js | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contracts/utils/math/Math.sol b/contracts/utils/math/Math.sol index 12546593ffe..d328509f31f 100644 --- a/contracts/utils/math/Math.sol +++ b/contracts/utils/math/Math.sol @@ -788,7 +788,7 @@ library Math { } /// @dev Same as {reverseBitsUint256} but optimized for 16-bit values. - function reverseBits16(uint16 value) internal pure returns (uint256) { + function reverseBitsUint16(uint16 value) internal pure returns (uint256) { return (value >> 8) | (value << 8); } diff --git a/test/utils/math/Math.t.sol b/test/utils/math/Math.t.sol index 9f501b0e367..cea4b901dee 100644 --- a/test/utils/math/Math.t.sol +++ b/test/utils/math/Math.t.sol @@ -325,8 +325,8 @@ contract MathTest is Test { assertEq(Math.reverseBitsUint32(uint32(Math.reverseBitsUint32(value))), value); } - function testSymbolicReverseBits16(uint16 value) public pure { - assertEq(Math.reverseBits16(uint16(Math.reverseBits16(value))), value); + function testSymbolicreverseBitsUint16(uint16 value) public pure { + assertEq(Math.reverseBitsUint16(uint16(Math.reverseBitsUint16(value))), value); } // Helpers diff --git a/test/utils/math/Math.test.js b/test/utils/math/Math.test.js index ce1abdd8a09..80166c59dc7 100644 --- a/test/utils/math/Math.test.js +++ b/test/utils/math/Math.test.js @@ -794,32 +794,32 @@ describe('Math', function () { describe('reverseBits16', function () { it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBits16(0)).to.eventually.equal(0n); - await expect(this.mock.$reverseBits16(MAX_UINT16)).to.eventually.equal(MAX_UINT16); + await expect(this.mock.$reverseBitsUint16(0)).to.eventually.equal(0n); + await expect(this.mock.$reverseBitsUint16(MAX_UINT16)).to.eventually.equal(MAX_UINT16); // Test known pattern: 0x1234 -> 0x3412 - await expect(this.mock.$reverseBits16(0x1234)).to.eventually.equal(0x3412); + await expect(this.mock.$reverseBitsUint16(0x1234)).to.eventually.equal(0x3412); }); it('double reverse returns original', async function () { const values = [0n, 1n, 0x1234n, MAX_UINT16]; for (const value of values) { - const reversed = await this.mock.$reverseBits16(value); + const reversed = await this.mock.$reverseBitsUint16(value); // Cast back to uint16 for comparison since function returns uint256 - await expect(this.mock.$reverseBits16(reversed & MAX_UINT16)).to.eventually.equal(value); + await expect(this.mock.$reverseBitsUint16(reversed & MAX_UINT16)).to.eventually.equal(value); } }); }); describe('edge cases', function () { it('handles single byte values', async function () { - await expect(this.mock.$reverseBits16(0x00ff)).to.eventually.equal(0xff00); + await expect(this.mock.$reverseBitsUint16(0x00ff)).to.eventually.equal(0xff00); await expect(this.mock.$reverseBitsUint32(0x000000ff)).to.eventually.equal(0xff000000); }); it('handles alternating patterns', async function () { - await expect(this.mock.$reverseBits16(0xaaaa)).to.eventually.equal(0xaaaa); - await expect(this.mock.$reverseBits16(0x5555)).to.eventually.equal(0x5555); + await expect(this.mock.$reverseBitsUint16(0xaaaa)).to.eventually.equal(0xaaaa); + await expect(this.mock.$reverseBitsUint16(0x5555)).to.eventually.equal(0x5555); await expect(this.mock.$reverseBitsUint32(0xaaaaaaaa)).to.eventually.equal(0xaaaaaaaa); await expect(this.mock.$reverseBitsUint32(0x55555555)).to.eventually.equal(0x55555555); }); From 40d7922684b065c544e6432301b53bfd4054373f Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:08:05 -0600 Subject: [PATCH 046/114] Update return types of reverseBits functions to match their respective bit sizes --- contracts/utils/math/Math.sol | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contracts/utils/math/Math.sol b/contracts/utils/math/Math.sol index d328509f31f..5c25022d917 100644 --- a/contracts/utils/math/Math.sol +++ b/contracts/utils/math/Math.sol @@ -761,7 +761,7 @@ library Math { } /// @dev Same as {reverseBitsUint256} but optimized for 128-bit values. - function reverseBitsUint128(uint128 value) internal pure returns (uint256) { + function reverseBitsUint128(uint128 value) internal pure returns (uint128) { value = // swap bytes ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); @@ -775,20 +775,20 @@ library Math { } /// @dev Same as {reverseBitsUint256} but optimized for 64-bit values. - function reverseBitsUint64(uint64 value) internal pure returns (uint256) { + function reverseBitsUint64(uint64 value) internal pure returns (uint64) { value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs return (value >> 32) | (value << 32); // swap 4-byte long pairs } /// @dev Same as {reverseBitsUint256} but optimized for 32-bit values. - function reverseBitsUint32(uint32 value) internal pure returns (uint256) { + function reverseBitsUint32(uint32 value) internal pure returns (uint32) { value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes return (value >> 16) | (value << 16); // swap 2-byte long pairs } /// @dev Same as {reverseBitsUint256} but optimized for 16-bit values. - function reverseBitsUint16(uint16 value) internal pure returns (uint256) { + function reverseBitsUint16(uint16 value) internal pure returns (uint16) { return (value >> 8) | (value << 8); } From 89860bc87f0618eac29109103f801a4880ec7085 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:21:40 -0600 Subject: [PATCH 047/114] Refactor reverseBits functions in to use fixed-size byte types --- .changeset/major-feet-write.md | 2 +- contracts/utils/math/Math.sol | 18 ++++---- test/utils/math/Math.t.sol | 20 ++++----- test/utils/math/Math.test.js | 79 +++++++++++++++++++--------------- 4 files changed, 64 insertions(+), 55 deletions(-) diff --git a/.changeset/major-feet-write.md b/.changeset/major-feet-write.md index da2966f00cd..f175b7cc14b 100644 --- a/.changeset/major-feet-write.md +++ b/.changeset/major-feet-write.md @@ -2,4 +2,4 @@ 'openzeppelin-solidity': minor --- -`Math`: Add `reverseBitsUint256`, `reverseBitsUint128`, `reverseBitsUint64`, `reverseBitsUint32`, and `reverseBits16` functions to reverse byte order for converting between little-endian and big-endian representations. +`Math`: Add `reverseBits256`, `reverseBits128`, `reverseBits64`, `reverseBits32`, and `reverseBits16` functions to reverse byte order for converting between little-endian and big-endian representations. diff --git a/contracts/utils/math/Math.sol b/contracts/utils/math/Math.sol index 5c25022d917..3d49b50778e 100644 --- a/contracts/utils/math/Math.sol +++ b/contracts/utils/math/Math.sol @@ -744,7 +744,7 @@ library Math { * @dev Reverses the byte order of a uint256 value, converting between little-endian and big-endian. * Inspired in https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel[Reverse Parallel] */ - function reverseBitsUint256(uint256 value) internal pure returns (uint256) { + function reverseBits256(bytes32 value) internal pure returns (bytes32) { value = // swap bytes ((value >> 8) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) | ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) << 8); @@ -760,8 +760,8 @@ library Math { return (value >> 128) | (value << 128); // swap 16-byte long pairs } - /// @dev Same as {reverseBitsUint256} but optimized for 128-bit values. - function reverseBitsUint128(uint128 value) internal pure returns (uint128) { + /// @dev Same as {reverseBits256} but optimized for 128-bit values. + function reverseBits128(bytes16 value) internal pure returns (bytes16) { value = // swap bytes ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); @@ -774,21 +774,21 @@ library Math { return (value >> 64) | (value << 64); // swap 8-byte long pairs } - /// @dev Same as {reverseBitsUint256} but optimized for 64-bit values. - function reverseBitsUint64(uint64 value) internal pure returns (uint64) { + /// @dev Same as {reverseBits256} but optimized for 64-bit values. + function reverseBits64(bytes8 value) internal pure returns (bytes8) { value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs return (value >> 32) | (value << 32); // swap 4-byte long pairs } - /// @dev Same as {reverseBitsUint256} but optimized for 32-bit values. - function reverseBitsUint32(uint32 value) internal pure returns (uint32) { + /// @dev Same as {reverseBits256} but optimized for 32-bit values. + function reverseBits32(bytes4 value) internal pure returns (bytes4) { value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes return (value >> 16) | (value << 16); // swap 2-byte long pairs } - /// @dev Same as {reverseBitsUint256} but optimized for 16-bit values. - function reverseBitsUint16(uint16 value) internal pure returns (uint16) { + /// @dev Same as {reverseBits256} but optimized for 16-bit values. + function reverseBits16(bytes2 value) internal pure returns (bytes2) { return (value >> 8) | (value << 8); } diff --git a/test/utils/math/Math.t.sol b/test/utils/math/Math.t.sol index cea4b901dee..47f74930556 100644 --- a/test/utils/math/Math.t.sol +++ b/test/utils/math/Math.t.sol @@ -309,24 +309,24 @@ contract MathTest is Test { } // REVERSE BITS - function testSymbolicReverseBitsUint256(uint256 value) public pure { - assertEq(Math.reverseBitsUint256(Math.reverseBitsUint256(value)), value); + function testSymbolicReverseBits256(bytes32 value) public pure { + assertEq(Math.reverseBits256(Math.reverseBits256(value)), value); } - function testSymbolicReverseBitsUint128(uint128 value) public pure { - assertEq(Math.reverseBitsUint128(uint128(Math.reverseBitsUint128(value))), value); + function testSymbolicReverseBits128(bytes16 value) public pure { + assertEq(Math.reverseBits128(Math.reverseBits128(value)), value); } - function testSymbolicReverseBitsUint64(uint64 value) public pure { - assertEq(Math.reverseBitsUint64(uint64(Math.reverseBitsUint64(value))), value); + function testSymbolicReverseBits64(bytes8 value) public pure { + assertEq(Math.reverseBits64(Math.reverseBits64(value)), value); } - function testSymbolicReverseBitsUint32(uint32 value) public pure { - assertEq(Math.reverseBitsUint32(uint32(Math.reverseBitsUint32(value))), value); + function testSymbolicReverseBits32(bytes4 value) public pure { + assertEq(Math.reverseBits32(Math.reverseBits32(value)), value); } - function testSymbolicreverseBitsUint16(uint16 value) public pure { - assertEq(Math.reverseBitsUint16(uint16(Math.reverseBitsUint16(value))), value); + function testSymbolicReverseBits16(bytes2 value) public pure { + assertEq(Math.reverseBits16(Math.reverseBits16(value)), value); } // Helpers diff --git a/test/utils/math/Math.test.js b/test/utils/math/Math.test.js index 80166c59dc7..3283015db01 100644 --- a/test/utils/math/Math.test.js +++ b/test/utils/math/Math.test.js @@ -17,6 +17,13 @@ const uint256 = value => ethers.Typed.uint256(value); bytes.zero = '0x'; uint256.zero = 0n; +// Helper functions for fixed bytes types +const bytes32 = value => ethers.toBeHex(value, 32); +const bytes16 = value => ethers.toBeHex(value, 16); +const bytes8 = value => ethers.toBeHex(value, 8); +const bytes4 = value => ethers.toBeHex(value, 4); +const bytes2 = value => ethers.toBeHex(value, 2); + const testCommutative = (fn, lhs, rhs, expected, ...extra) => Promise.all([ expect(fn(lhs, rhs, ...extra)).to.eventually.deep.equal(expected), @@ -713,33 +720,35 @@ describe('Math', function () { }); describe('reverseBits', function () { - describe('reverseBitsUint256', function () { + describe('reverseBits256', function () { it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBitsUint256(0)).to.eventually.equal(0n); - await expect(this.mock.$reverseBitsUint256(ethers.MaxUint256)).to.eventually.equal(ethers.MaxUint256); + await expect(this.mock.$reverseBits256(bytes32(0))).to.eventually.equal(bytes32(0)); + await expect(this.mock.$reverseBits256(bytes32(ethers.MaxUint256))).to.eventually.equal( + bytes32(ethers.MaxUint256), + ); // Test simple pattern await expect( - this.mock.$reverseBitsUint256('0x0000000000000000000000000000000000000000000000000000000000000001'), + this.mock.$reverseBits256('0x0000000000000000000000000000000000000000000000000000000000000001'), ).to.eventually.equal('0x0100000000000000000000000000000000000000000000000000000000000000'); }); it('double reverse returns original', async function () { const values = [0n, 1n, 0x12345678n, ethers.MaxUint256]; for (const value of values) { - const reversed = await this.mock.$reverseBitsUint256(value); - await expect(this.mock.$reverseBitsUint256(reversed)).to.eventually.equal(value); + const reversed = await this.mock.$reverseBits256(bytes32(value)); + await expect(this.mock.$reverseBits256(reversed)).to.eventually.equal(bytes32(value)); } }); }); - describe('reverseBitsUint128', function () { + describe('reverseBits128', function () { it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBitsUint128(0)).to.eventually.equal(0n); - await expect(this.mock.$reverseBitsUint128(MAX_UINT128)).to.eventually.equal(MAX_UINT128); + await expect(this.mock.$reverseBits128(bytes16(0))).to.eventually.equal(bytes16(0)); + await expect(this.mock.$reverseBits128(bytes16(MAX_UINT128))).to.eventually.equal(bytes16(MAX_UINT128)); // Test simple pattern - await expect(this.mock.$reverseBitsUint128('0x00000000000000000000000000000001')).to.eventually.equal( + await expect(this.mock.$reverseBits128('0x00000000000000000000000000000001')).to.eventually.equal( '0x01000000000000000000000000000000', ); }); @@ -747,81 +756,81 @@ describe('Math', function () { it('double reverse returns original', async function () { const values = [0n, 1n, 0x12345678n, MAX_UINT128]; for (const value of values) { - const reversed = await this.mock.$reverseBitsUint128(value); + const reversed = await this.mock.$reverseBits128(bytes16(value)); // Cast back to uint128 for comparison since function returns uint256 - await expect(this.mock.$reverseBitsUint128(reversed & MAX_UINT128)).to.eventually.equal(value); + await expect(this.mock.$reverseBits128(reversed)).to.eventually.equal(bytes16(value & MAX_UINT128)); } }); }); - describe('reverseBitsUint64', function () { + describe('reverseBits64', function () { it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBitsUint64(0)).to.eventually.equal(0n); - await expect(this.mock.$reverseBitsUint64(MAX_UINT64)).to.eventually.equal(MAX_UINT64); + await expect(this.mock.$reverseBits64(bytes8(0))).to.eventually.equal(bytes8(0)); + await expect(this.mock.$reverseBits64(bytes8(MAX_UINT64))).to.eventually.equal(bytes8(MAX_UINT64)); // Test known pattern: 0x123456789ABCDEF0 -> 0xF0DEBC9A78563412 - await expect(this.mock.$reverseBitsUint64('0x123456789ABCDEF0')).to.eventually.equal('0xF0DEBC9A78563412'); + await expect(this.mock.$reverseBits64('0x123456789ABCDEF0')).to.eventually.equal('0xf0debc9a78563412'); }); it('double reverse returns original', async function () { const values = [0n, 1n, 0x12345678n, MAX_UINT64]; for (const value of values) { - const reversed = await this.mock.$reverseBitsUint64(value); + const reversed = await this.mock.$reverseBits64(bytes8(value)); // Cast back to uint64 for comparison since function returns uint256 - await expect(this.mock.$reverseBitsUint64(reversed & MAX_UINT64)).to.eventually.equal(value); + await expect(this.mock.$reverseBits64(reversed)).to.eventually.equal(bytes8(value & MAX_UINT64)); } }); }); - describe('reverseBitsUint32', function () { + describe('reverseBits32', function () { it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBitsUint32(0)).to.eventually.equal(0n); - await expect(this.mock.$reverseBitsUint32(MAX_UINT32)).to.eventually.equal(MAX_UINT32); + await expect(this.mock.$reverseBits32(bytes4(0))).to.eventually.equal(bytes4(0)); + await expect(this.mock.$reverseBits32(bytes4(MAX_UINT32))).to.eventually.equal(bytes4(MAX_UINT32)); // Test known pattern: 0x12345678 -> 0x78563412 - await expect(this.mock.$reverseBitsUint32(0x12345678)).to.eventually.equal(0x78563412); + await expect(this.mock.$reverseBits32(bytes4(0x12345678))).to.eventually.equal(bytes4(0x78563412)); }); it('double reverse returns original', async function () { const values = [0n, 1n, 0x12345678n, MAX_UINT32]; for (const value of values) { - const reversed = await this.mock.$reverseBitsUint32(value); + const reversed = await this.mock.$reverseBits32(bytes4(value)); // Cast back to uint32 for comparison since function returns uint256 - await expect(this.mock.$reverseBitsUint32(reversed & MAX_UINT32)).to.eventually.equal(value); + await expect(this.mock.$reverseBits32(reversed)).to.eventually.equal(bytes4(value & MAX_UINT32)); } }); }); describe('reverseBits16', function () { it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBitsUint16(0)).to.eventually.equal(0n); - await expect(this.mock.$reverseBitsUint16(MAX_UINT16)).to.eventually.equal(MAX_UINT16); + await expect(this.mock.$reverseBits16(bytes2(0))).to.eventually.equal(bytes2(0)); + await expect(this.mock.$reverseBits16(bytes2(MAX_UINT16))).to.eventually.equal(bytes2(MAX_UINT16)); // Test known pattern: 0x1234 -> 0x3412 - await expect(this.mock.$reverseBitsUint16(0x1234)).to.eventually.equal(0x3412); + await expect(this.mock.$reverseBits16(bytes2(0x1234))).to.eventually.equal(bytes2(0x3412)); }); it('double reverse returns original', async function () { const values = [0n, 1n, 0x1234n, MAX_UINT16]; for (const value of values) { - const reversed = await this.mock.$reverseBitsUint16(value); + const reversed = await this.mock.$reverseBits16(bytes2(value)); // Cast back to uint16 for comparison since function returns uint256 - await expect(this.mock.$reverseBitsUint16(reversed & MAX_UINT16)).to.eventually.equal(value); + await expect(this.mock.$reverseBits16(reversed)).to.eventually.equal(bytes2(value & MAX_UINT16)); } }); }); describe('edge cases', function () { it('handles single byte values', async function () { - await expect(this.mock.$reverseBitsUint16(0x00ff)).to.eventually.equal(0xff00); - await expect(this.mock.$reverseBitsUint32(0x000000ff)).to.eventually.equal(0xff000000); + await expect(this.mock.$reverseBits16(bytes2(0x00ff))).to.eventually.equal(bytes2(0xff00)); + await expect(this.mock.$reverseBits32(bytes4(0x000000ff))).to.eventually.equal(bytes4(0xff000000)); }); it('handles alternating patterns', async function () { - await expect(this.mock.$reverseBitsUint16(0xaaaa)).to.eventually.equal(0xaaaa); - await expect(this.mock.$reverseBitsUint16(0x5555)).to.eventually.equal(0x5555); - await expect(this.mock.$reverseBitsUint32(0xaaaaaaaa)).to.eventually.equal(0xaaaaaaaa); - await expect(this.mock.$reverseBitsUint32(0x55555555)).to.eventually.equal(0x55555555); + await expect(this.mock.$reverseBits16(bytes2(0xaaaa))).to.eventually.equal(bytes2(0xaaaa)); + await expect(this.mock.$reverseBits16(bytes2(0x5555))).to.eventually.equal(bytes2(0x5555)); + await expect(this.mock.$reverseBits32(bytes4(0xaaaaaaaa))).to.eventually.equal(bytes4(0xaaaaaaaa)); + await expect(this.mock.$reverseBits32(bytes4(0x55555555))).to.eventually.equal(bytes4(0x55555555)); }); }); }); From 9b58730aa21682f15e5d4b409f6e936116c34f6d Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:32:34 -0600 Subject: [PATCH 048/114] Test nits --- test/utils/math/Math.t.sol | 52 ++++++++++++++++++++++++++++++++++++ test/utils/math/Math.test.js | 14 +++++----- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/test/utils/math/Math.t.sol b/test/utils/math/Math.t.sol index 47f74930556..3ba58b06959 100644 --- a/test/utils/math/Math.t.sol +++ b/test/utils/math/Math.t.sol @@ -317,19 +317,71 @@ contract MathTest is Test { assertEq(Math.reverseBits128(Math.reverseBits128(value)), value); } + function testSymbolicReverseBits128Dirty(bytes16 value) public pure { + bytes16 dirty = _dirtyBytes128(value); + assertEq(Math.reverseBits128(Math.reverseBits128(dirty)), value); + } + function testSymbolicReverseBits64(bytes8 value) public pure { assertEq(Math.reverseBits64(Math.reverseBits64(value)), value); } + function testSymbolicReverseBits64Dirty(bytes8 value) public pure { + bytes8 dirty = _dirtyBytes64(value); + assertEq(Math.reverseBits64(Math.reverseBits64(dirty)), value); + } + function testSymbolicReverseBits32(bytes4 value) public pure { assertEq(Math.reverseBits32(Math.reverseBits32(value)), value); } + function testSymbolicReverseBits32Dirty(bytes4 value) public pure { + bytes4 dirty = _dirtyBytes32(value); + assertEq(Math.reverseBits32(Math.reverseBits32(dirty)), value); + } + function testSymbolicReverseBits16(bytes2 value) public pure { assertEq(Math.reverseBits16(Math.reverseBits16(value)), value); } + function testSymbolicReverseBits16Dirty(bytes2 value) public pure { + bytes2 dirty = _dirtyBytes16(value); + assertEq(Math.reverseBits16(Math.reverseBits16(dirty)), value); + } + // Helpers + function _dirtyBytes128(bytes16 value) private pure returns (bytes16) { + bytes16 dirty = value; + assembly ("memory-safe") { + dirty := or(dirty, shr(128, not(0))) + } + return dirty; + } + + function _dirtyBytes64(bytes8 value) private pure returns (bytes8) { + bytes8 dirty = value; + assembly ("memory-safe") { + dirty := or(dirty, shr(64, not(0))) + } + return dirty; + } + + function _dirtyBytes32(bytes4 value) private pure returns (bytes4) { + bytes4 dirty = value; + assembly ("memory-safe") { + dirty := or(dirty, shr(32, not(0))) + } + return dirty; + } + + function _dirtyBytes16(bytes2 value) private pure returns (bytes2) { + bytes2 dirty = value; + assembly ("memory-safe") { + dirty := or(dirty, shr(16, not(0))) + } + return dirty; + } + function _asRounding(uint8 r) private pure returns (Math.Rounding) { vm.assume(r < uint8(type(Math.Rounding).max)); return Math.Rounding(r); diff --git a/test/utils/math/Math.test.js b/test/utils/math/Math.test.js index 3283015db01..4cd0eb7a44d 100644 --- a/test/utils/math/Math.test.js +++ b/test/utils/math/Math.test.js @@ -727,10 +727,10 @@ describe('Math', function () { bytes32(ethers.MaxUint256), ); - // Test simple pattern + // Test complex pattern that clearly shows byte reversal await expect( - this.mock.$reverseBits256('0x0000000000000000000000000000000000000000000000000000000000000001'), - ).to.eventually.equal('0x0100000000000000000000000000000000000000000000000000000000000000'); + this.mock.$reverseBits256('0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'), + ).to.eventually.equal('0xefcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301'); }); it('double reverse returns original', async function () { @@ -747,9 +747,9 @@ describe('Math', function () { await expect(this.mock.$reverseBits128(bytes16(0))).to.eventually.equal(bytes16(0)); await expect(this.mock.$reverseBits128(bytes16(MAX_UINT128))).to.eventually.equal(bytes16(MAX_UINT128)); - // Test simple pattern - await expect(this.mock.$reverseBits128('0x00000000000000000000000000000001')).to.eventually.equal( - '0x01000000000000000000000000000000', + // Test complex pattern that clearly shows byte reversal + await expect(this.mock.$reverseBits128('0x0123456789abcdef0123456789abcdef')).to.eventually.equal( + '0xefcdab8967452301efcdab8967452301', ); }); @@ -769,7 +769,7 @@ describe('Math', function () { await expect(this.mock.$reverseBits64(bytes8(MAX_UINT64))).to.eventually.equal(bytes8(MAX_UINT64)); // Test known pattern: 0x123456789ABCDEF0 -> 0xF0DEBC9A78563412 - await expect(this.mock.$reverseBits64('0x123456789ABCDEF0')).to.eventually.equal('0xf0debc9a78563412'); + await expect(this.mock.$reverseBits64('0x123456789abcdef0')).to.eventually.equal('0xf0debc9a78563412'); }); it('double reverse returns original', async function () { From 77ffa8ce90c3c9796beaadea5c6300ebec2d0475 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:33:24 -0600 Subject: [PATCH 049/114] Simplify --- test/utils/math/Math.t.sol | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test/utils/math/Math.t.sol b/test/utils/math/Math.t.sol index 3ba58b06959..fdcb4111d42 100644 --- a/test/utils/math/Math.t.sol +++ b/test/utils/math/Math.t.sol @@ -318,8 +318,7 @@ contract MathTest is Test { } function testSymbolicReverseBits128Dirty(bytes16 value) public pure { - bytes16 dirty = _dirtyBytes128(value); - assertEq(Math.reverseBits128(Math.reverseBits128(dirty)), value); + assertEq(Math.reverseBits128(Math.reverseBits128(_dirtyBytes128(value))), value); } function testSymbolicReverseBits64(bytes8 value) public pure { @@ -327,8 +326,7 @@ contract MathTest is Test { } function testSymbolicReverseBits64Dirty(bytes8 value) public pure { - bytes8 dirty = _dirtyBytes64(value); - assertEq(Math.reverseBits64(Math.reverseBits64(dirty)), value); + assertEq(Math.reverseBits64(Math.reverseBits64(_dirtyBytes64(value))), value); } function testSymbolicReverseBits32(bytes4 value) public pure { @@ -336,8 +334,7 @@ contract MathTest is Test { } function testSymbolicReverseBits32Dirty(bytes4 value) public pure { - bytes4 dirty = _dirtyBytes32(value); - assertEq(Math.reverseBits32(Math.reverseBits32(dirty)), value); + assertEq(Math.reverseBits32(Math.reverseBits32(_dirtyBytes32(value))), value); } function testSymbolicReverseBits16(bytes2 value) public pure { @@ -345,8 +342,7 @@ contract MathTest is Test { } function testSymbolicReverseBits16Dirty(bytes2 value) public pure { - bytes2 dirty = _dirtyBytes16(value); - assertEq(Math.reverseBits16(Math.reverseBits16(dirty)), value); + assertEq(Math.reverseBits16(Math.reverseBits16(_dirtyBytes16(value))), value); } // Helpers From ce91c8098f99b8181c3dcdcb5975fb0f45b4e116 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:34:07 -0600 Subject: [PATCH 050/114] up --- test/utils/math/Math.t.sol | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/utils/math/Math.t.sol b/test/utils/math/Math.t.sol index fdcb4111d42..c0ac024ed05 100644 --- a/test/utils/math/Math.t.sol +++ b/test/utils/math/Math.t.sol @@ -357,7 +357,7 @@ contract MathTest is Test { function _dirtyBytes64(bytes8 value) private pure returns (bytes8) { bytes8 dirty = value; assembly ("memory-safe") { - dirty := or(dirty, shr(64, not(0))) + dirty := or(dirty, shr(192, not(0))) } return dirty; } @@ -365,7 +365,7 @@ contract MathTest is Test { function _dirtyBytes32(bytes4 value) private pure returns (bytes4) { bytes4 dirty = value; assembly ("memory-safe") { - dirty := or(dirty, shr(32, not(0))) + dirty := or(dirty, shr(224, not(0))) } return dirty; } @@ -373,7 +373,7 @@ contract MathTest is Test { function _dirtyBytes16(bytes2 value) private pure returns (bytes2) { bytes2 dirty = value; assembly ("memory-safe") { - dirty := or(dirty, shr(16, not(0))) + dirty := or(dirty, shr(240, not(0))) } return dirty; } From b3e3adde2cbaa45f9961e8f589cd8901c694f50e Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:40:08 -0600 Subject: [PATCH 051/114] Move reverse functions to Bytes.sol --- .changeset/major-feet-write.md | 2 +- contracts/utils/Bytes.sol | 52 ++++++++++++++ contracts/utils/math/Math.sol | 52 -------------- test/utils/Bytes.test.js | 124 +++++++++++++++++++++++++++++++++ test/utils/math/Bytes.t.sol | 79 +++++++++++++++++++++ test/utils/math/Math.t.sol | 69 ------------------ test/utils/math/Math.test.js | 124 --------------------------------- 7 files changed, 256 insertions(+), 246 deletions(-) create mode 100644 test/utils/math/Bytes.t.sol diff --git a/.changeset/major-feet-write.md b/.changeset/major-feet-write.md index f175b7cc14b..81897219735 100644 --- a/.changeset/major-feet-write.md +++ b/.changeset/major-feet-write.md @@ -2,4 +2,4 @@ 'openzeppelin-solidity': minor --- -`Math`: Add `reverseBits256`, `reverseBits128`, `reverseBits64`, `reverseBits32`, and `reverseBits16` functions to reverse byte order for converting between little-endian and big-endian representations. +`Bytes`: Add `reverseBits256`, `reverseBits128`, `reverseBits64`, `reverseBits32`, and `reverseBits16` functions to reverse byte order for converting between little-endian and big-endian representations. diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 1234b845513..fa024cc2f2a 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -99,6 +99,58 @@ library Bytes { return result; } + /** + * @dev Reverses the byte order of a uint256 value, converting between little-endian and big-endian. + * Inspired in https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel[Reverse Parallel] + */ + function reverseBits256(bytes32 value) internal pure returns (bytes32) { + value = // swap bytes + ((value >> 8) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) | + ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) << 8); + value = // swap 2-byte long pairs + ((value >> 16) & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) | + ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) << 16); + value = // swap 4-byte long pairs + ((value >> 32) & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) | + ((value & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) << 32); + value = // swap 8-byte long pairs + ((value >> 64) & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) | + ((value & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) << 64); + return (value >> 128) | (value << 128); // swap 16-byte long pairs + } + + /// @dev Same as {reverseBits256} but optimized for 128-bit values. + function reverseBits128(bytes16 value) internal pure returns (bytes16) { + value = // swap bytes + ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | + ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); + value = // swap 2-byte long pairs + ((value & 0xFFFF0000FFFF0000FFFF0000FFFF0000) >> 16) | + ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF) << 16); + value = // swap 4-byte long pairs + ((value & 0xFFFFFFFF00000000FFFFFFFF00000000) >> 32) | + ((value & 0x00000000FFFFFFFF00000000FFFFFFFF) << 32); + return (value >> 64) | (value << 64); // swap 8-byte long pairs + } + + /// @dev Same as {reverseBits256} but optimized for 64-bit values. + function reverseBits64(bytes8 value) internal pure returns (bytes8) { + value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes + value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs + return (value >> 32) | (value << 32); // swap 4-byte long pairs + } + + /// @dev Same as {reverseBits256} but optimized for 32-bit values. + function reverseBits32(bytes4 value) internal pure returns (bytes4) { + value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes + return (value >> 16) | (value << 16); // swap 2-byte long pairs + } + + /// @dev Same as {reverseBits256} but optimized for 16-bit values. + function reverseBits16(bytes2 value) internal pure returns (bytes2) { + return (value >> 8) | (value << 8); + } + /** * @dev Reads a bytes32 from a bytes array without bounds checking. * diff --git a/contracts/utils/math/Math.sol b/contracts/utils/math/Math.sol index 3d49b50778e..f0d608a2dea 100644 --- a/contracts/utils/math/Math.sol +++ b/contracts/utils/math/Math.sol @@ -740,58 +740,6 @@ library Math { } } - /** - * @dev Reverses the byte order of a uint256 value, converting between little-endian and big-endian. - * Inspired in https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel[Reverse Parallel] - */ - function reverseBits256(bytes32 value) internal pure returns (bytes32) { - value = // swap bytes - ((value >> 8) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) | - ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) << 8); - value = // swap 2-byte long pairs - ((value >> 16) & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) | - ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) << 16); - value = // swap 4-byte long pairs - ((value >> 32) & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) | - ((value & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) << 32); - value = // swap 8-byte long pairs - ((value >> 64) & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) | - ((value & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) << 64); - return (value >> 128) | (value << 128); // swap 16-byte long pairs - } - - /// @dev Same as {reverseBits256} but optimized for 128-bit values. - function reverseBits128(bytes16 value) internal pure returns (bytes16) { - value = // swap bytes - ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | - ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); - value = // swap 2-byte long pairs - ((value & 0xFFFF0000FFFF0000FFFF0000FFFF0000) >> 16) | - ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF) << 16); - value = // swap 4-byte long pairs - ((value & 0xFFFFFFFF00000000FFFFFFFF00000000) >> 32) | - ((value & 0x00000000FFFFFFFF00000000FFFFFFFF) << 32); - return (value >> 64) | (value << 64); // swap 8-byte long pairs - } - - /// @dev Same as {reverseBits256} but optimized for 64-bit values. - function reverseBits64(bytes8 value) internal pure returns (bytes8) { - value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes - value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs - return (value >> 32) | (value << 32); // swap 4-byte long pairs - } - - /// @dev Same as {reverseBits256} but optimized for 32-bit values. - function reverseBits32(bytes4 value) internal pure returns (bytes4) { - value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes - return (value >> 16) | (value << 16); // swap 2-byte long pairs - } - - /// @dev Same as {reverseBits256} but optimized for 16-bit values. - function reverseBits16(bytes2 value) internal pure returns (bytes2) { - return (value >> 8) | (value << 8); - } - /** * @dev Returns whether a provided rounding mode is considered rounding up for unsigned integers. */ diff --git a/test/utils/Bytes.test.js b/test/utils/Bytes.test.js index 52a1ae95e77..1728f2213bb 100644 --- a/test/utils/Bytes.test.js +++ b/test/utils/Bytes.test.js @@ -1,6 +1,14 @@ const { ethers } = require('hardhat'); const { expect } = require('chai'); const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); +const { MAX_UINT128, MAX_UINT64, MAX_UINT32, MAX_UINT16 } = require('../helpers/constants'); + +// Helper functions for fixed bytes types +const bytes32 = value => ethers.toBeHex(value, 32); +const bytes16 = value => ethers.toBeHex(value, 16); +const bytes8 = value => ethers.toBeHex(value, 8); +const bytes4 = value => ethers.toBeHex(value, 4); +const bytes2 = value => ethers.toBeHex(value, 2); async function fixture() { const mock = await ethers.deployContract('$Bytes'); @@ -85,4 +93,120 @@ describe('Bytes', function () { } }); }); + + describe('reverseBits', function () { + describe('reverseBits256', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBits256(bytes32(0))).to.eventually.equal(bytes32(0)); + await expect(this.mock.$reverseBits256(bytes32(ethers.MaxUint256))).to.eventually.equal( + bytes32(ethers.MaxUint256), + ); + + // Test complex pattern that clearly shows byte reversal + await expect( + this.mock.$reverseBits256('0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'), + ).to.eventually.equal('0xefcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301'); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, ethers.MaxUint256]; + for (const value of values) { + const reversed = await this.mock.$reverseBits256(bytes32(value)); + await expect(this.mock.$reverseBits256(reversed)).to.eventually.equal(bytes32(value)); + } + }); + }); + + describe('reverseBits128', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBits128(bytes16(0))).to.eventually.equal(bytes16(0)); + await expect(this.mock.$reverseBits128(bytes16(MAX_UINT128))).to.eventually.equal(bytes16(MAX_UINT128)); + + // Test complex pattern that clearly shows byte reversal + await expect(this.mock.$reverseBits128('0x0123456789abcdef0123456789abcdef')).to.eventually.equal( + '0xefcdab8967452301efcdab8967452301', + ); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT128]; + for (const value of values) { + const reversed = await this.mock.$reverseBits128(bytes16(value)); + // Cast back to uint128 for comparison since function returns uint256 + await expect(this.mock.$reverseBits128(reversed)).to.eventually.equal(bytes16(value & MAX_UINT128)); + } + }); + }); + + describe('reverseBits64', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBits64(bytes8(0))).to.eventually.equal(bytes8(0)); + await expect(this.mock.$reverseBits64(bytes8(MAX_UINT64))).to.eventually.equal(bytes8(MAX_UINT64)); + + // Test known pattern: 0x123456789ABCDEF0 -> 0xF0DEBC9A78563412 + await expect(this.mock.$reverseBits64('0x123456789abcdef0')).to.eventually.equal('0xf0debc9a78563412'); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT64]; + for (const value of values) { + const reversed = await this.mock.$reverseBits64(bytes8(value)); + // Cast back to uint64 for comparison since function returns uint256 + await expect(this.mock.$reverseBits64(reversed)).to.eventually.equal(bytes8(value & MAX_UINT64)); + } + }); + }); + + describe('reverseBits32', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBits32(bytes4(0))).to.eventually.equal(bytes4(0)); + await expect(this.mock.$reverseBits32(bytes4(MAX_UINT32))).to.eventually.equal(bytes4(MAX_UINT32)); + + // Test known pattern: 0x12345678 -> 0x78563412 + await expect(this.mock.$reverseBits32(bytes4(0x12345678))).to.eventually.equal(bytes4(0x78563412)); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x12345678n, MAX_UINT32]; + for (const value of values) { + const reversed = await this.mock.$reverseBits32(bytes4(value)); + // Cast back to uint32 for comparison since function returns uint256 + await expect(this.mock.$reverseBits32(reversed)).to.eventually.equal(bytes4(value & MAX_UINT32)); + } + }); + }); + + describe('reverseBits16', function () { + it('reverses bytes correctly', async function () { + await expect(this.mock.$reverseBits16(bytes2(0))).to.eventually.equal(bytes2(0)); + await expect(this.mock.$reverseBits16(bytes2(MAX_UINT16))).to.eventually.equal(bytes2(MAX_UINT16)); + + // Test known pattern: 0x1234 -> 0x3412 + await expect(this.mock.$reverseBits16(bytes2(0x1234))).to.eventually.equal(bytes2(0x3412)); + }); + + it('double reverse returns original', async function () { + const values = [0n, 1n, 0x1234n, MAX_UINT16]; + for (const value of values) { + const reversed = await this.mock.$reverseBits16(bytes2(value)); + // Cast back to uint16 for comparison since function returns uint256 + await expect(this.mock.$reverseBits16(reversed)).to.eventually.equal(bytes2(value & MAX_UINT16)); + } + }); + }); + + describe('edge cases', function () { + it('handles single byte values', async function () { + await expect(this.mock.$reverseBits16(bytes2(0x00ff))).to.eventually.equal(bytes2(0xff00)); + await expect(this.mock.$reverseBits32(bytes4(0x000000ff))).to.eventually.equal(bytes4(0xff000000)); + }); + + it('handles alternating patterns', async function () { + await expect(this.mock.$reverseBits16(bytes2(0xaaaa))).to.eventually.equal(bytes2(0xaaaa)); + await expect(this.mock.$reverseBits16(bytes2(0x5555))).to.eventually.equal(bytes2(0x5555)); + await expect(this.mock.$reverseBits32(bytes4(0xaaaaaaaa))).to.eventually.equal(bytes4(0xaaaaaaaa)); + await expect(this.mock.$reverseBits32(bytes4(0x55555555))).to.eventually.equal(bytes4(0x55555555)); + }); + }); + }); }); diff --git a/test/utils/math/Bytes.t.sol b/test/utils/math/Bytes.t.sol new file mode 100644 index 00000000000..7b4f5acd87d --- /dev/null +++ b/test/utils/math/Bytes.t.sol @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +import {Test, stdError} from "forge-std/Test.sol"; + +import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; + +contract BytesTest is Test { + // REVERSE BITS + function testSymbolicReverseBits256(bytes32 value) public pure { + assertEq(Bytes.reverseBits256(Bytes.reverseBits256(value)), value); + } + + function testSymbolicReverseBits128(bytes16 value) public pure { + assertEq(Bytes.reverseBits128(Bytes.reverseBits128(value)), value); + } + + function testSymbolicReverseBits128Dirty(bytes16 value) public pure { + assertEq(Bytes.reverseBits128(Bytes.reverseBits128(_dirtyBytes128(value))), value); + } + + function testSymbolicReverseBits64(bytes8 value) public pure { + assertEq(Bytes.reverseBits64(Bytes.reverseBits64(value)), value); + } + + function testSymbolicReverseBits64Dirty(bytes8 value) public pure { + assertEq(Bytes.reverseBits64(Bytes.reverseBits64(_dirtyBytes64(value))), value); + } + + function testSymbolicReverseBits32(bytes4 value) public pure { + assertEq(Bytes.reverseBits32(Bytes.reverseBits32(value)), value); + } + + function testSymbolicReverseBits32Dirty(bytes4 value) public pure { + assertEq(Bytes.reverseBits32(Bytes.reverseBits32(_dirtyBytes32(value))), value); + } + + function testSymbolicReverseBits16(bytes2 value) public pure { + assertEq(Bytes.reverseBits16(Bytes.reverseBits16(value)), value); + } + + function testSymbolicReverseBits16Dirty(bytes2 value) public pure { + assertEq(Bytes.reverseBits16(Bytes.reverseBits16(_dirtyBytes16(value))), value); + } + + // Helpers + function _dirtyBytes128(bytes16 value) private pure returns (bytes16) { + bytes16 dirty = value; + assembly ("memory-safe") { + dirty := or(dirty, shr(128, not(0))) + } + return dirty; + } + + function _dirtyBytes64(bytes8 value) private pure returns (bytes8) { + bytes8 dirty = value; + assembly ("memory-safe") { + dirty := or(dirty, shr(192, not(0))) + } + return dirty; + } + + function _dirtyBytes32(bytes4 value) private pure returns (bytes4) { + bytes4 dirty = value; + assembly ("memory-safe") { + dirty := or(dirty, shr(224, not(0))) + } + return dirty; + } + + function _dirtyBytes16(bytes2 value) private pure returns (bytes2) { + bytes2 dirty = value; + assembly ("memory-safe") { + dirty := or(dirty, shr(240, not(0))) + } + return dirty; + } +} diff --git a/test/utils/math/Math.t.sol b/test/utils/math/Math.t.sol index c0ac024ed05..3c83febe9df 100644 --- a/test/utils/math/Math.t.sol +++ b/test/utils/math/Math.t.sol @@ -308,76 +308,7 @@ contract MathTest is Test { } } - // REVERSE BITS - function testSymbolicReverseBits256(bytes32 value) public pure { - assertEq(Math.reverseBits256(Math.reverseBits256(value)), value); - } - - function testSymbolicReverseBits128(bytes16 value) public pure { - assertEq(Math.reverseBits128(Math.reverseBits128(value)), value); - } - - function testSymbolicReverseBits128Dirty(bytes16 value) public pure { - assertEq(Math.reverseBits128(Math.reverseBits128(_dirtyBytes128(value))), value); - } - - function testSymbolicReverseBits64(bytes8 value) public pure { - assertEq(Math.reverseBits64(Math.reverseBits64(value)), value); - } - - function testSymbolicReverseBits64Dirty(bytes8 value) public pure { - assertEq(Math.reverseBits64(Math.reverseBits64(_dirtyBytes64(value))), value); - } - - function testSymbolicReverseBits32(bytes4 value) public pure { - assertEq(Math.reverseBits32(Math.reverseBits32(value)), value); - } - - function testSymbolicReverseBits32Dirty(bytes4 value) public pure { - assertEq(Math.reverseBits32(Math.reverseBits32(_dirtyBytes32(value))), value); - } - - function testSymbolicReverseBits16(bytes2 value) public pure { - assertEq(Math.reverseBits16(Math.reverseBits16(value)), value); - } - - function testSymbolicReverseBits16Dirty(bytes2 value) public pure { - assertEq(Math.reverseBits16(Math.reverseBits16(_dirtyBytes16(value))), value); - } - // Helpers - function _dirtyBytes128(bytes16 value) private pure returns (bytes16) { - bytes16 dirty = value; - assembly ("memory-safe") { - dirty := or(dirty, shr(128, not(0))) - } - return dirty; - } - - function _dirtyBytes64(bytes8 value) private pure returns (bytes8) { - bytes8 dirty = value; - assembly ("memory-safe") { - dirty := or(dirty, shr(192, not(0))) - } - return dirty; - } - - function _dirtyBytes32(bytes4 value) private pure returns (bytes4) { - bytes4 dirty = value; - assembly ("memory-safe") { - dirty := or(dirty, shr(224, not(0))) - } - return dirty; - } - - function _dirtyBytes16(bytes2 value) private pure returns (bytes2) { - bytes2 dirty = value; - assembly ("memory-safe") { - dirty := or(dirty, shr(240, not(0))) - } - return dirty; - } - function _asRounding(uint8 r) private pure returns (Math.Rounding) { vm.assume(r < uint8(type(Math.Rounding).max)); return Math.Rounding(r); diff --git a/test/utils/math/Math.test.js b/test/utils/math/Math.test.js index 4cd0eb7a44d..6a09938148a 100644 --- a/test/utils/math/Math.test.js +++ b/test/utils/math/Math.test.js @@ -7,7 +7,6 @@ const { Rounding } = require('../../helpers/enums'); const { min, max, modExp } = require('../../helpers/math'); const { generators } = require('../../helpers/random'); const { product, range } = require('../../helpers/iterate'); -const { MAX_UINT128, MAX_UINT64, MAX_UINT32, MAX_UINT16 } = require('../../helpers/constants'); const RoundingDown = [Rounding.Floor, Rounding.Trunc]; const RoundingUp = [Rounding.Ceil, Rounding.Expand]; @@ -17,13 +16,6 @@ const uint256 = value => ethers.Typed.uint256(value); bytes.zero = '0x'; uint256.zero = 0n; -// Helper functions for fixed bytes types -const bytes32 = value => ethers.toBeHex(value, 32); -const bytes16 = value => ethers.toBeHex(value, 16); -const bytes8 = value => ethers.toBeHex(value, 8); -const bytes4 = value => ethers.toBeHex(value, 4); -const bytes2 = value => ethers.toBeHex(value, 2); - const testCommutative = (fn, lhs, rhs, expected, ...extra) => Promise.all([ expect(fn(lhs, rhs, ...extra)).to.eventually.deep.equal(expected), @@ -718,120 +710,4 @@ describe('Math', function () { }); }); }); - - describe('reverseBits', function () { - describe('reverseBits256', function () { - it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBits256(bytes32(0))).to.eventually.equal(bytes32(0)); - await expect(this.mock.$reverseBits256(bytes32(ethers.MaxUint256))).to.eventually.equal( - bytes32(ethers.MaxUint256), - ); - - // Test complex pattern that clearly shows byte reversal - await expect( - this.mock.$reverseBits256('0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'), - ).to.eventually.equal('0xefcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301'); - }); - - it('double reverse returns original', async function () { - const values = [0n, 1n, 0x12345678n, ethers.MaxUint256]; - for (const value of values) { - const reversed = await this.mock.$reverseBits256(bytes32(value)); - await expect(this.mock.$reverseBits256(reversed)).to.eventually.equal(bytes32(value)); - } - }); - }); - - describe('reverseBits128', function () { - it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBits128(bytes16(0))).to.eventually.equal(bytes16(0)); - await expect(this.mock.$reverseBits128(bytes16(MAX_UINT128))).to.eventually.equal(bytes16(MAX_UINT128)); - - // Test complex pattern that clearly shows byte reversal - await expect(this.mock.$reverseBits128('0x0123456789abcdef0123456789abcdef')).to.eventually.equal( - '0xefcdab8967452301efcdab8967452301', - ); - }); - - it('double reverse returns original', async function () { - const values = [0n, 1n, 0x12345678n, MAX_UINT128]; - for (const value of values) { - const reversed = await this.mock.$reverseBits128(bytes16(value)); - // Cast back to uint128 for comparison since function returns uint256 - await expect(this.mock.$reverseBits128(reversed)).to.eventually.equal(bytes16(value & MAX_UINT128)); - } - }); - }); - - describe('reverseBits64', function () { - it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBits64(bytes8(0))).to.eventually.equal(bytes8(0)); - await expect(this.mock.$reverseBits64(bytes8(MAX_UINT64))).to.eventually.equal(bytes8(MAX_UINT64)); - - // Test known pattern: 0x123456789ABCDEF0 -> 0xF0DEBC9A78563412 - await expect(this.mock.$reverseBits64('0x123456789abcdef0')).to.eventually.equal('0xf0debc9a78563412'); - }); - - it('double reverse returns original', async function () { - const values = [0n, 1n, 0x12345678n, MAX_UINT64]; - for (const value of values) { - const reversed = await this.mock.$reverseBits64(bytes8(value)); - // Cast back to uint64 for comparison since function returns uint256 - await expect(this.mock.$reverseBits64(reversed)).to.eventually.equal(bytes8(value & MAX_UINT64)); - } - }); - }); - - describe('reverseBits32', function () { - it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBits32(bytes4(0))).to.eventually.equal(bytes4(0)); - await expect(this.mock.$reverseBits32(bytes4(MAX_UINT32))).to.eventually.equal(bytes4(MAX_UINT32)); - - // Test known pattern: 0x12345678 -> 0x78563412 - await expect(this.mock.$reverseBits32(bytes4(0x12345678))).to.eventually.equal(bytes4(0x78563412)); - }); - - it('double reverse returns original', async function () { - const values = [0n, 1n, 0x12345678n, MAX_UINT32]; - for (const value of values) { - const reversed = await this.mock.$reverseBits32(bytes4(value)); - // Cast back to uint32 for comparison since function returns uint256 - await expect(this.mock.$reverseBits32(reversed)).to.eventually.equal(bytes4(value & MAX_UINT32)); - } - }); - }); - - describe('reverseBits16', function () { - it('reverses bytes correctly', async function () { - await expect(this.mock.$reverseBits16(bytes2(0))).to.eventually.equal(bytes2(0)); - await expect(this.mock.$reverseBits16(bytes2(MAX_UINT16))).to.eventually.equal(bytes2(MAX_UINT16)); - - // Test known pattern: 0x1234 -> 0x3412 - await expect(this.mock.$reverseBits16(bytes2(0x1234))).to.eventually.equal(bytes2(0x3412)); - }); - - it('double reverse returns original', async function () { - const values = [0n, 1n, 0x1234n, MAX_UINT16]; - for (const value of values) { - const reversed = await this.mock.$reverseBits16(bytes2(value)); - // Cast back to uint16 for comparison since function returns uint256 - await expect(this.mock.$reverseBits16(reversed)).to.eventually.equal(bytes2(value & MAX_UINT16)); - } - }); - }); - - describe('edge cases', function () { - it('handles single byte values', async function () { - await expect(this.mock.$reverseBits16(bytes2(0x00ff))).to.eventually.equal(bytes2(0xff00)); - await expect(this.mock.$reverseBits32(bytes4(0x000000ff))).to.eventually.equal(bytes4(0xff000000)); - }); - - it('handles alternating patterns', async function () { - await expect(this.mock.$reverseBits16(bytes2(0xaaaa))).to.eventually.equal(bytes2(0xaaaa)); - await expect(this.mock.$reverseBits16(bytes2(0x5555))).to.eventually.equal(bytes2(0x5555)); - await expect(this.mock.$reverseBits32(bytes4(0xaaaaaaaa))).to.eventually.equal(bytes4(0xaaaaaaaa)); - await expect(this.mock.$reverseBits32(bytes4(0x55555555))).to.eventually.equal(bytes4(0x55555555)); - }); - }); - }); }); From 2f3107cd44082b49a1811a4a3ea8b267a4ee0fc0 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:40:43 -0600 Subject: [PATCH 052/114] Move Bytes.t.sol --- test/utils/{math => }/Bytes.t.sol | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/utils/{math => }/Bytes.t.sol (100%) diff --git a/test/utils/math/Bytes.t.sol b/test/utils/Bytes.t.sol similarity index 100% rename from test/utils/math/Bytes.t.sol rename to test/utils/Bytes.t.sol From 5a44b1116541436c68a0d973be8ec3d28d3965f8 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:49:26 -0600 Subject: [PATCH 053/114] up --- contracts/utils/Bytes.sol | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 633a9cc913b..7528416eae0 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -116,16 +116,8 @@ library Bytes { return a.length == b.length && keccak256(a) == keccak256(b); } - /// @dev Counts the number of leading zero bytes in a uint256. function clz(uint256 x) internal pure returns (uint256) { - if (x == 0) return 32; // All 32 bytes are zero - uint256 r = 0; - if (x > 0xffffffffffffffffffffffffffffffff) r = 128; // Upper 128 bits - if ((x >> r) > 0xffffffffffffffff) r |= 64; // Next 64 bits - if ((x >> r) > 0xffffffff) r |= 32; // Next 32 bits - if ((x >> r) > 0xffff) r |= 16; // Next 16 bits - if ((x >> r) > 0xff) r |= 8; // Next 8 bits - return 31 ^ (r >> 3); // Convert to leading zero bytes count + return Math.ternary(x == 0, 32, 31 - Math.log256(x)); } /** From d6db2d7a47708341a10612d34b83e91bda8d687e Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 11:49:51 -0600 Subject: [PATCH 054/114] Document --- contracts/utils/Bytes.sol | 1 + 1 file changed, 1 insertion(+) diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 7528416eae0..0650dc0d42b 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -116,6 +116,7 @@ library Bytes { return a.length == b.length && keccak256(a) == keccak256(b); } + /// @dev Counts the number of leading zeros in a uint256. function clz(uint256 x) internal pure returns (uint256) { return Math.ternary(x == 0, 32, 31 - Math.log256(x)); } From 38470504c38f502bd783f7fa03621149dca2ec06 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 12:26:54 -0600 Subject: [PATCH 055/114] Remove extra functions --- contracts/utils/Memory.sol | 46 ----------------------------------- test/utils/Memory.t.sol | 46 ----------------------------------- test/utils/Memory.test.js | 49 -------------------------------------- 3 files changed, 141 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index d787608dc63..0a4d902a0b6 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -30,57 +30,11 @@ library Memory { } } - /// @dev Returns a `Pointer` to the content of a `bytes` buffer. Skips the length word. - function contentPointer(bytes memory buffer) internal pure returns (Pointer) { - return addOffset(asPointer(buffer), 32); - } - - /** - * @dev Copies `length` bytes from `srcPtr` to `destPtr`. Equivalent to https://www.evm.codes/?fork=cancun#5e[`mcopy`]. - * - * WARNING: Reading or writing beyond the allocated memory bounds of either pointer - * will result in undefined behavior and potential memory corruption. - */ - function copy(Pointer destPtr, Pointer srcPtr, uint256 length) internal pure { - assembly ("memory-safe") { - mcopy(destPtr, srcPtr, length) - } - } - - /** - * @dev Extracts a `bytes1` from a `Pointer`. `offset` starts from the most significant byte. - * - * NOTE: Will return `0x00` if `offset` is larger or equal to `32`. - */ - function loadByte(Pointer ptr, uint256 offset) internal pure returns (bytes1 v) { - bytes32 word = load(ptr); - assembly ("memory-safe") { - v := byte(offset, word) - } - } - - /// @dev Extracts a `bytes32` from a `Pointer`. - function load(Pointer ptr) internal pure returns (bytes32 v) { - assembly ("memory-safe") { - v := mload(ptr) - } - } - - /// @dev Adds an offset to a `Pointer`. - function addOffset(Pointer ptr, uint256 offset) internal pure returns (Pointer) { - return asPointer(bytes32(asUint256(ptr) + offset)); - } - /// @dev `Pointer` to `bytes32`. Expects a pointer to a properly ABI-encoded `bytes` object. function asBytes32(Pointer ptr) internal pure returns (bytes32) { return Pointer.unwrap(ptr); } - /// @dev `Pointer` to `uint256`. Expects a pointer to a properly ABI-encoded `bytes` object. - function asUint256(Pointer ptr) internal pure returns (uint256) { - return uint256(asBytes32(ptr)); - } - /// @dev `bytes32` to `Pointer`. Expects a pointer to a properly ABI-encoded `bytes` object. function asPointer(bytes32 value) internal pure returns (Pointer) { return Pointer.wrap(value); diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol index 016f328c41b..8ed9b4c43bc 100644 --- a/test/utils/Memory.t.sol +++ b/test/utils/Memory.t.sol @@ -18,50 +18,4 @@ contract MemoryTest is Test { ptr.asPointer().setFreeMemoryPointer(); assertEq(Memory.getFreeMemoryPointer().asBytes32(), ptr); } - - function testSymbolicContentPointer(uint256 seed) public pure { - Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); - assertEq(ptr.asBytes().contentPointer().asBytes32(), ptr.addOffset(32).asBytes32()); - } - - function testCopy(bytes memory data, uint256 destSeed) public pure { - uint256 minDestPtr = Memory.getFreeMemoryPointer().asUint256(); - Memory.Pointer destPtr = bytes32(bound(destSeed, minDestPtr, minDestPtr + END_PTR)).asPointer(); - destPtr.addOffset(data.length + 32).setFreeMemoryPointer(); - destPtr.copy(data.asPointer(), data.length + 32); - bytes memory copiedData = destPtr.asBytes(); - assertEq(data.length, copiedData.length); - for (uint256 i = 0; i < data.length; i++) { - assertEq(data[i], copiedData[i]); - } - } - - function testLoadByte(uint256 seed, uint256 index, bytes32 value) public pure { - index = bound(index, 0, 31); - Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); - - assembly ("memory-safe") { - mstore(ptr, value) - } - - bytes1 expected; - assembly ("memory-safe") { - expected := byte(index, value) - } - assertEq(ptr.loadByte(index), expected); - } - - function testLoad(uint256 seed, bytes32 value) public pure { - Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); - assembly ("memory-safe") { - mstore(ptr, value) - } - assertEq(ptr.load(), value); - } - - function testSymbolicAddOffset(uint256 seed, uint256 offset) public pure { - offset = bound(offset, 0, type(uint256).max - END_PTR); - Memory.Pointer ptr = bytes32(bound(seed, START_PTR, END_PTR)).asPointer(); - assertEq(ptr.addOffset(offset).asUint256(), ptr.asUint256() + offset); - } } diff --git a/test/utils/Memory.test.js b/test/utils/Memory.test.js index cd687e2f37c..6a1159c1493 100644 --- a/test/utils/Memory.test.js +++ b/test/utils/Memory.test.js @@ -26,39 +26,6 @@ describe('Memory', function () { }); }); - it('load extracts a word', async function () { - const ptr = await this.mock.$getFreeMemoryPointer(); - await expect(this.mock.$load(ptr)).to.eventually.equal(ethers.toBeHex(0, 32)); - }); - - it('loadByte extracts a byte', async function () { - const ptr = await this.mock.$getFreeMemoryPointer(); - await expect(this.mock.$loadByte(ptr, 0)).to.eventually.equal(ethers.toBeHex(0, 1)); - }); - - it('contentPointer', async function () { - const data = ethers.toUtf8Bytes('hello world'); - const result = await this.mock.$contentPointer(data); - expect(result).to.equal(ethers.toBeHex(0xa0, 32)); // 0x80 is the default free pointer (length) - }); - - describe('addOffset', function () { - it('addOffset', async function () { - const basePtr = ethers.toBeHex(0x80, 32); - const offset = 32; - const expectedPtr = ethers.toBeHex(0xa0, 32); - - await expect(this.mock.$addOffset(basePtr, offset)).to.eventually.equal(expectedPtr); - }); - - it('addOffsetwraps around', async function () { - const basePtr = ethers.toBeHex(0x80, 32); - const offset = 256; - const expectedPtr = ethers.toBeHex(0x180, 32); - await expect(this.mock.$addOffset(basePtr, offset)).to.eventually.equal(expectedPtr); - }); - }); - describe('pointer conversions', function () { it('asBytes32 / asPointer', async function () { const ptr = ethers.toBeHex('0x1234', 32); @@ -71,21 +38,5 @@ describe('Memory', function () { expect(ptr).to.equal(ethers.toBeHex(0x80, 32)); // Default free pointer await expect(this.mock.$asBytes(ptr)).to.eventually.equal(ethers.toBeHex(0x20, 32)); }); - - it('asUint256', async function () { - const value = 0x1234; - const ptr = ethers.toBeHex(value, 32); - await expect(this.mock.$asUint256(ptr)).to.eventually.equal(value); - }); - }); - - describe('memory operations', function () { - it('copy', async function () { - await expect(this.mock.$copy(ethers.toBeHex(0x80, 32), ethers.toBeHex(0xc0, 32), 32)).to.not.be.reverted; - }); - - it('copy with zero length', async function () { - await expect(this.mock.$copy(ethers.toBeHex(0x80, 32), ethers.toBeHex(0xc0, 32), 0)).to.not.be.reverted; - }); }); }); From 4fd194722ab5527a43d4ad2b8330a0fe9ac89811 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 9 Jul 2025 12:32:33 -0600 Subject: [PATCH 056/114] Update docs --- docs/modules/ROOT/pages/utilities.adoc | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docs/modules/ROOT/pages/utilities.adoc b/docs/modules/ROOT/pages/utilities.adoc index ee34c0c4c03..d04c5cd56f7 100644 --- a/docs/modules/ROOT/pages/utilities.adoc +++ b/docs/modules/ROOT/pages/utilities.adoc @@ -491,23 +491,6 @@ function processMultipleItems(uint256[] memory items) internal { This way, memory allocated for `tempData` in each iteration is reused, significantly reducing memory expansion costs when processing many items. -==== Copying memory buffers - -The `Memory` library provides a `copy` function that allows copying data between memory locations. This is useful when you need to extract a segment of data from a larger buffer or when you want to avoid unnecessary memory allocations. The following example demonstrates how to copy a segment of data from a source buffer: - -[source,solidity] ----- -function copyDataSegment(bytes memory source, uint256 offset, uint256 length) - internal pure returns (bytes memory result) { - - result = new bytes(length); - Memory.Pointer srcPtr = Memory.addOffset(Memory.contentPointer(source), offset); - Memory.Pointer destPtr = Memory.contentPointer(result); - - Memory.copy(destPtr, srcPtr, length); -} ----- - IMPORTANT: Manual memory management increases gas costs and prevents compiler optimizations. Only use these functions after profiling confirms they're necessary. By default, Solidity handles memory safely - using this library without understanding memory layout and safety may be dangerous. See the https://docs.soliditylang.org/en/v0.8.20/internals/layout_in_memory.html[memory layout] and https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[memory safety] documentation for details. === Historical Block Hashes From aa26e487aa10caa199fe77551a9b8081e222beca Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Tue, 15 Jul 2025 10:27:54 +0200 Subject: [PATCH 057/114] up --- contracts/utils/Bytes.sol | 52 --------------------------------------- contracts/utils/RLP.sol | 5 ++-- test/utils/Bytes.t.sol | 16 ++---------- 3 files changed, 4 insertions(+), 69 deletions(-) diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 04d7c96ce82..42d9402726d 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -121,58 +121,6 @@ library Bytes { return Math.ternary(x == 0, 32, 31 - Math.log256(x)); } - /** - * @dev Reverses the byte order of a uint256 value, converting between little-endian and big-endian. - * Inspired in https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel[Reverse Parallel] - */ - function reverseBits256(bytes32 value) internal pure returns (bytes32) { - value = // swap bytes - ((value >> 8) & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) | - ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF) << 8); - value = // swap 2-byte long pairs - ((value >> 16) & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) | - ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF0000FFFF) << 16); - value = // swap 4-byte long pairs - ((value >> 32) & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) | - ((value & 0x00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF00000000FFFFFFFF) << 32); - value = // swap 8-byte long pairs - ((value >> 64) & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) | - ((value & 0x0000000000000000FFFFFFFFFFFFFFFF0000000000000000FFFFFFFFFFFFFFFF) << 64); - return (value >> 128) | (value << 128); // swap 16-byte long pairs - } - - /// @dev Same as {reverseBits256} but optimized for 128-bit values. - function reverseBits128(bytes16 value) internal pure returns (bytes16) { - value = // swap bytes - ((value & 0xFF00FF00FF00FF00FF00FF00FF00FF00) >> 8) | - ((value & 0x00FF00FF00FF00FF00FF00FF00FF00FF) << 8); - value = // swap 2-byte long pairs - ((value & 0xFFFF0000FFFF0000FFFF0000FFFF0000) >> 16) | - ((value & 0x0000FFFF0000FFFF0000FFFF0000FFFF) << 16); - value = // swap 4-byte long pairs - ((value & 0xFFFFFFFF00000000FFFFFFFF00000000) >> 32) | - ((value & 0x00000000FFFFFFFF00000000FFFFFFFF) << 32); - return (value >> 64) | (value << 64); // swap 8-byte long pairs - } - - /// @dev Same as {reverseBits256} but optimized for 64-bit values. - function reverseBits64(bytes8 value) internal pure returns (bytes8) { - value = ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); // swap bytes - value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); // swap 2-byte long pairs - return (value >> 32) | (value << 32); // swap 4-byte long pairs - } - - /// @dev Same as {reverseBits256} but optimized for 32-bit values. - function reverseBits32(bytes4 value) internal pure returns (bytes4) { - value = ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); // swap bytes - return (value >> 16) | (value << 16); // swap 2-byte long pairs - } - - /// @dev Same as {reverseBits256} but optimized for 16-bit values. - function reverseBits16(bytes2 value) internal pure returns (bytes2) { - return (value >> 8) | (value << 8); - } - /** * @dev Moves the content of `buffer`, from `start` (included) to the end of `buffer` to the start of that buffer. * diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index a4c91a2a004..2ee7f7d21a1 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -203,7 +203,7 @@ library RLP { return abi.encodePacked( bytes1(uint8(bytesLength) + uint8(offset) + SHORT_THRESHOLD), - bytes32(length).reverseBits256() // to big-endian + bytes32(length).reverseBytes32() // to big-endian ); } @@ -308,9 +308,8 @@ library RLP { } function _loadByte(Memory.Pointer ptr, uint256 offset) private pure returns (bytes1 v) { - bytes32 word = _load(ptr); assembly ("memory-safe") { - v := byte(offset, word) + v := byte(offset, mload(ptr)) } } diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index 41fc7401aae..19df07fb31c 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -89,7 +89,7 @@ contract BytesTest is Test { uint256 result = Bytes.indexOf(buffer, s, pos); // Should not be found before result - for (uint256 i = pos; result != type(uint256).max && i < result; i++) assertNotEq(buffer[i], s); + for (uint256 i = pos; i < Math.min(buffer.length, result); ++i) assertNotEq(buffer[i], s); if (result != type(uint256).max) assertEq(buffer[result], s); } @@ -102,22 +102,10 @@ contract BytesTest is Test { uint256 result = Bytes.lastIndexOf(buffer, s, pos); // Should not be found before result - for (uint256 i = pos; result != type(uint256).max && i < result; i++) assertNotEq(buffer[i], s); + for (uint256 i = pos; i < Math.min(buffer.length, result); ++i) assertNotEq(buffer[i], s); if (result != type(uint256).max) assertEq(buffer[result], s); } - function testSlice(bytes memory buffer, uint256 start) public pure { - testSlice(buffer, start, buffer.length); - } - - function testSlice(bytes memory buffer, uint256 start, uint256 end) public pure { - bytes memory result = Bytes.slice(buffer, start, end); - uint256 sanitizedEnd = Math.min(end, buffer.length); - uint256 sanitizedStart = Math.min(start, sanitizedEnd); - assertEq(result.length, sanitizedEnd - sanitizedStart); - for (uint256 i = 0; i < result.length; i++) assertEq(result[i], buffer[sanitizedStart + i]); - } - function testNibbles(bytes memory value) public pure { bytes memory result = Bytes.nibbles(value); assertEq(result.length, value.length * 2); From d4bfb8ba6171f49c2ab4bd9b8411c3db1efa0c15 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 31 Jul 2025 12:31:15 -0600 Subject: [PATCH 058/114] Fix compilation --- contracts/utils/Bytes.sol | 7 ------- contracts/utils/RLP.sol | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 249ee3d8cfb..2ef21755f53 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -108,13 +108,6 @@ library Bytes { return nibbles_; } - /** - * @dev Returns true if the two byte buffers are equal. - */ - function equal(bytes memory a, bytes memory b) internal pure returns (bool) { - return a.length == b.length && keccak256(a) == keccak256(b); - } - /// @dev Counts the number of leading zeros in a uint256. function clz(uint256 x) internal pure returns (uint256) { return Math.ternary(x == 0, 32, 31 - Math.log256(x)); diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 2ee7f7d21a1..b6526714bab 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -209,7 +209,7 @@ library RLP { /// @dev Converts a uint256 to minimal binary representation, removing leading zeros. function _binaryBuffer(uint256 value) private pure returns (bytes memory) { - return abi.encodePacked(value).slice(value.clz()); + return abi.encodePacked(value).slice(Math.clz(value)); } /// @dev Concatenates all byte arrays in the `list` sequentially. Returns a flattened buffer. From 138de7f31a73c2126ad716c78556736bcd64fb33 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 31 Jul 2025 12:32:21 -0600 Subject: [PATCH 059/114] Remove dangling clz --- contracts/utils/Bytes.sol | 5 ----- contracts/utils/RLP.sol | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 2ef21755f53..4f27b7908ff 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -108,11 +108,6 @@ library Bytes { return nibbles_; } - /// @dev Counts the number of leading zeros in a uint256. - function clz(uint256 x) internal pure returns (uint256) { - return Math.ternary(x == 0, 32, 31 - Math.log256(x)); - } - /** * @dev Moves the content of `buffer`, from `start` (included) to the end of `buffer` to the start of that buffer. * diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index b6526714bab..2ee7f7d21a1 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -209,7 +209,7 @@ library RLP { /// @dev Converts a uint256 to minimal binary representation, removing leading zeros. function _binaryBuffer(uint256 value) private pure returns (bytes memory) { - return abi.encodePacked(value).slice(Math.clz(value)); + return abi.encodePacked(value).slice(value.clz()); } /// @dev Concatenates all byte arrays in the `list` sequentially. Returns a flattened buffer. From 5efeb37ab893aa317802289aa10193b869de8b89 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 31 Jul 2025 12:38:50 -0600 Subject: [PATCH 060/114] Make nibbles function private --- .changeset/khaki-hats-leave.md | 5 --- contracts/utils/Bytes.sol | 10 ------ contracts/utils/cryptography/TrieProof.sol | 14 ++++++-- test/utils/Bytes.t.sol | 41 ---------------------- 4 files changed, 12 insertions(+), 58 deletions(-) delete mode 100644 .changeset/khaki-hats-leave.md diff --git a/.changeset/khaki-hats-leave.md b/.changeset/khaki-hats-leave.md deleted file mode 100644 index 021df0ff083..00000000000 --- a/.changeset/khaki-hats-leave.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'openzeppelin-solidity': minor ---- - -`Bytes`: Add a `nibbles` function to split each byte into two nibbles. diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 4f27b7908ff..2fabf4872db 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -98,16 +98,6 @@ library Bytes { return result; } - /// @dev Split each byte in `value` into two nibbles (4 bits each). - function nibbles(bytes memory value) internal pure returns (bytes memory) { - uint256 length = value.length; - bytes memory nibbles_ = new bytes(length * 2); - for (uint256 i = 0; i < length; i++) { - (nibbles_[i * 2], nibbles_[i * 2 + 1]) = (value[i] & 0xf0, value[i] & 0x0f); - } - return nibbles_; - } - /** * @dev Moves the content of `buffer`, from `start` (included) to the end of `buffer` to the start of that buffer. * diff --git a/contracts/utils/cryptography/TrieProof.sol b/contracts/utils/cryptography/TrieProof.sol index 4751772f8d3..8f050159b11 100644 --- a/contracts/utils/cryptography/TrieProof.sol +++ b/contracts/utils/cryptography/TrieProof.sol @@ -93,7 +93,7 @@ library TrieProof { ) internal pure returns (bytes memory value, ProofError) { if (key.length == 0) return ("", ProofError.EMPTY_KEY); // Convert key to nibbles (4-bit values) and begin processing from the root - return _processInclusionProof(_decodeProof(proof), key.nibbles(), bytes.concat(root), 0, radix); + return _processInclusionProof(_decodeProof(proof), _nibbles(key), bytes.concat(root), 0, radix); } /// @dev Main recursive function that traverses the trie using the provided proof. @@ -226,7 +226,7 @@ library TrieProof { * The path is stored as the first element in the node's decoded array. */ function _path(Node memory node) private pure returns (bytes memory) { - return node.decoded[0].readBytes().nibbles(); + return _nibbles(node.decoded[0].readBytes()); } /** @@ -241,4 +241,14 @@ library TrieProof { } return length; } + + /// @dev Split each byte in `value` into two nibbles (4 bits each). + function _nibbles(bytes memory value) internal pure returns (bytes memory) { + uint256 length = value.length; + bytes memory nibbles_ = new bytes(length * 2); + for (uint256 i = 0; i < length; i++) { + (nibbles_[i * 2], nibbles_[i * 2 + 1]) = (value[i] & 0xf0, value[i] & 0x0f); + } + return nibbles_; + } } diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index fcb73c50c0d..d6eb2fc2c1c 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -155,31 +155,6 @@ contract BytesTest is Test { } } - function testIndexOf(bytes memory buffer, bytes1 s) public pure { - testIndexOf(buffer, s, 0); - } - - function testIndexOf(bytes memory buffer, bytes1 s, uint256 pos) public pure { - uint256 result = Bytes.indexOf(buffer, s, pos); - - // Should not be found before result - for (uint256 i = pos; i < Math.min(buffer.length, result); ++i) assertNotEq(buffer[i], s); - if (result != type(uint256).max) assertEq(buffer[result], s); - } - - function testLastIndexOf(bytes memory buffer, bytes1 s) public pure { - testLastIndexOf(buffer, s, 0); - } - - function testLastIndexOf(bytes memory buffer, bytes1 s, uint256 pos) public pure { - pos = bound(pos, 0, buffer.length); - uint256 result = Bytes.lastIndexOf(buffer, s, pos); - - // Should not be found before result - for (uint256 i = pos; i < Math.min(buffer.length, result); ++i) assertNotEq(buffer[i], s); - if (result != type(uint256).max) assertEq(buffer[result], s); - } - function testNibbles(bytes memory value) public pure { bytes memory result = Bytes.nibbles(value); assertEq(result.length, value.length * 2); @@ -193,22 +168,6 @@ contract BytesTest is Test { } } - function testSymbolicCountLeadingZeroes(uint256 x) public pure { - uint256 result = Bytes.clz(x); - assertLe(result, 32); // [0, 32] - - if (x != 0) { - uint256 firstNonZeroBytePos = 32 - result - 1; - uint256 byteValue = (x >> (firstNonZeroBytePos * 8)) & 0xff; - assertNotEq(byteValue, 0); - - // x != 0 implies result < 32 - // most significant byte should be non-zero - uint256 msbValue = (x >> (248 - result * 8)) & 0xff; - assertNotEq(msbValue, 0); - } - } - // REVERSE BITS function testSymbolicReverseBytes32(bytes32 value) public pure { assertEq(Bytes.reverseBytes32(Bytes.reverseBytes32(value)), value); From 00ff228d9ff679d72a8e706075d6e7e90f6406a2 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 31 Jul 2025 12:39:56 -0600 Subject: [PATCH 061/114] Remove nibbles test --- test/utils/Bytes.t.sol | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index d6eb2fc2c1c..9fdcd47c2d1 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -155,19 +155,6 @@ contract BytesTest is Test { } } - function testNibbles(bytes memory value) public pure { - bytes memory result = Bytes.nibbles(value); - assertEq(result.length, value.length * 2); - for (uint256 i = 0; i < value.length; i++) { - bytes1 originalByte = value[i]; - bytes1 highNibble = result[i * 2]; - bytes1 lowNibble = result[i * 2 + 1]; - - assertEq(highNibble, originalByte & 0xf0); - assertEq(lowNibble, originalByte & 0x0f); - } - } - // REVERSE BITS function testSymbolicReverseBytes32(bytes32 value) public pure { assertEq(Bytes.reverseBytes32(Bytes.reverseBytes32(value)), value); From c58c7fd40b93f6a2aa2512f9ff47b7755034d323 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 31 Jul 2025 12:44:36 -0600 Subject: [PATCH 062/114] Remove TrieProof library --- .changeset/shaky-phones-mix.md | 5 - contracts/utils/cryptography/README.adoc | 3 - contracts/utils/cryptography/TrieProof.sol | 254 --------------------- 3 files changed, 262 deletions(-) delete mode 100644 .changeset/shaky-phones-mix.md delete mode 100644 contracts/utils/cryptography/TrieProof.sol diff --git a/.changeset/shaky-phones-mix.md b/.changeset/shaky-phones-mix.md deleted file mode 100644 index 410af473108..00000000000 --- a/.changeset/shaky-phones-mix.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'openzeppelin-solidity': minor ---- - -`TrieProof`: Add library for verifying Ethereum Merkle-Patricia trie inclusion proofs. diff --git a/contracts/utils/cryptography/README.adoc b/contracts/utils/cryptography/README.adoc index 5b608c3215b..6c222d7c6ba 100644 --- a/contracts/utils/cryptography/README.adoc +++ b/contracts/utils/cryptography/README.adoc @@ -11,7 +11,6 @@ A collection of contracts and libraries that implement various signature validat * {SignatureChecker}: A library helper to support regular ECDSA from EOAs as well as ERC-1271 signatures for smart contracts. * {Hashes}: Commonly used hash functions. * {MerkleProof}: Functions for verifying https://en.wikipedia.org/wiki/Merkle_tree[Merkle Tree] proofs. - * {TrieProof}: Library for verifying Ethereum Merkle-Patricia trie inclusion proofs. * {EIP712}: Contract with functions to allow processing signed typed structure data according to https://eips.ethereum.org/EIPS/eip-712[EIP-712]. * {ERC7739Utils}: Utilities library that implements a defensive rehashing mechanism to prevent replayability of smart contract signatures based on ERC-7739. * {WebAuthn}: Library for verifying WebAuthn Authentication Assertions. @@ -39,8 +38,6 @@ A collection of contracts and libraries that implement various signature validat {{MerkleProof}} -{{TrieProof}} - {{EIP712}} {{ERC7739Utils}} diff --git a/contracts/utils/cryptography/TrieProof.sol b/contracts/utils/cryptography/TrieProof.sol deleted file mode 100644 index 8f050159b11..00000000000 --- a/contracts/utils/cryptography/TrieProof.sol +++ /dev/null @@ -1,254 +0,0 @@ -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.27; - -import {Bytes} from "../Bytes.sol"; -import {Strings} from "../Strings.sol"; -import {RLP} from "../RLP.sol"; -import {Math} from "../math/Math.sol"; - -/** - * @dev Library for verifying Ethereum Merkle-Patricia trie inclusion proofs. - * - * Ethereum's State Trie state layout is a 4-item array of `[nonce, balance, storageRoot, codeHash]` - * See https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie[Merkle-Patricia trie] - */ -library TrieProof { - using Bytes for bytes; - using RLP for *; - using Strings for string; - - enum Prefix { - EXTENSION_EVEN, // 0 - Extension node with even length path - EXTENSION_ODD, // 1 - Extension node with odd length path - LEAF_EVEN, // 2 - Leaf node with even length path - LEAF_ODD // 3 - Leaf node with odd length path - } - - enum ProofError { - NO_ERROR, // No error occurred during proof verification - EMPTY_KEY, // The provided key is empty - INDEX_OUT_OF_BOUNDS, // Array index access is out of bounds - INVALID_ROOT_HASH, // The provided root hash doesn't match the proof - INVALID_LARGE_INTERNAL_HASH, // Internal node hash exceeds expected size - INVALID_INTERNAL_NODE_HASH, // Internal node hash doesn't match expected value - EMPTY_VALUE, // The value to verify is empty - INVALID_EXTRA_PROOF_ELEMENT, // Proof contains unexpected additional elements - INVALID_PATH_REMAINDER, // Path remainder doesn't match expected value - INVALID_KEY_REMAINDER, // Key remainder doesn't match expected value - UNKNOWN_NODE_PREFIX, // Node prefix is not recognized - UNPARSEABLE_NODE, // Node cannot be parsed from RLP encoding - INVALID_PROOF // General proof validation failure - } - - struct Node { - bytes encoded; // Raw RLP encoded node - RLP.Item[] decoded; // Decoded RLP items - } - - /// @dev The radix of the Ethereum trie (hexadecimal = 16) - uint256 internal constant EVM_TREE_RADIX = 16; - /// @dev Number of items in leaf or extension nodes (always 2) - uint256 internal constant LEAF_OR_EXTENSION_NODE_LENGTH = 2; - - /** - * @dev Verifies a `proof` against a given `key`, `value`, `and root` hash - * using the default Ethereum radix (16). - */ - function verify( - bytes memory key, - bytes memory value, - bytes[] memory proof, - bytes32 root - ) internal pure returns (bool) { - return verify(key, value, proof, root, EVM_TREE_RADIX); - } - - /// @dev Same as {verify} but with a custom radix. - function verify( - bytes memory key, - bytes memory value, - bytes[] memory proof, - bytes32 root, - uint256 radix - ) internal pure returns (bool) { - (bytes memory processedValue, ProofError err) = processProof(key, proof, root, radix); - return string(processedValue).equal(string(value)) && err == ProofError.NO_ERROR; - } - - /// @dev Processes a proof for a given key using default Ethereum radix (16) and returns the processed value. - function processProof( - bytes memory key, - bytes[] memory proof, - bytes32 root - ) internal pure returns (bytes memory value, ProofError) { - return processProof(key, proof, root, EVM_TREE_RADIX); - } - - /// @dev Same as {processProof} but with a custom radix. - function processProof( - bytes memory key, - bytes[] memory proof, - bytes32 root, - uint256 radix - ) internal pure returns (bytes memory value, ProofError) { - if (key.length == 0) return ("", ProofError.EMPTY_KEY); - // Convert key to nibbles (4-bit values) and begin processing from the root - return _processInclusionProof(_decodeProof(proof), _nibbles(key), bytes.concat(root), 0, radix); - } - - /// @dev Main recursive function that traverses the trie using the provided proof. - function _processInclusionProof( - Node[] memory trieProof, - bytes memory key, - bytes memory nodeId, - uint256 keyIndex, - uint256 radix - ) private pure returns (bytes memory value, ProofError err) { - uint256 branchNodeLength = radix + 1; // Branch nodes have radix+1 items (values + 1 for stored value) - - for (uint256 i = 0; i < trieProof.length; i++) { - Node memory node = trieProof[i]; - - // ensure we haven't overshot the key - if (keyIndex > key.length) return ("", ProofError.INDEX_OUT_OF_BOUNDS); - err = _validateNodeHashes(nodeId, node, keyIndex); - if (err != ProofError.NO_ERROR) return ("", err); - - uint256 nodeLength = node.decoded.length; - - // must be either a branch or leaf/extension node - if (nodeLength != branchNodeLength && nodeLength != LEAF_OR_EXTENSION_NODE_LENGTH) - return ("", ProofError.UNPARSEABLE_NODE); - - if (nodeLength == branchNodeLength) { - // If we've consumed the entire key, the value must be in the last slot - if (keyIndex == key.length) return _validateLastItem(node.decoded[radix], trieProof, i); - - // Otherwise, continue down the branch specified by the next nibble in the key - uint8 branchKey = uint8(key[keyIndex]); - (nodeId, keyIndex) = (_id(node.decoded[branchKey]), keyIndex + 1); - } else if (nodeLength == LEAF_OR_EXTENSION_NODE_LENGTH) { - return _processLeafOrExtension(node, trieProof, key, nodeId, keyIndex, i); - } - } - - // If we've gone through all proof elements without finding a value, the proof is invalid - return ("", ProofError.INVALID_PROOF); - } - - /// @dev Validates the node hashes at different levels of the proof. - function _validateNodeHashes( - bytes memory nodeId, - Node memory node, - uint256 keyIndex - ) private pure returns (ProofError) { - if (keyIndex == 0 && !string(bytes.concat(keccak256(node.encoded))).equal(string(nodeId))) - return ProofError.INVALID_ROOT_HASH; // Root node must match root hash - if (node.encoded.length >= 32 && !string(bytes.concat(keccak256(node.encoded))).equal(string(nodeId))) - return ProofError.INVALID_LARGE_INTERNAL_HASH; // Large nodes are stored as hashes - if (!string(node.encoded).equal(string(nodeId))) return ProofError.INVALID_INTERNAL_NODE_HASH; // Small nodes must match directly - return ProofError.NO_ERROR; // No error - } - - /** - * @dev Processes a leaf or extension node in the trie proof. - * - * For leaf nodes, validates that the key matches completely and returns the value. - * For extension nodes, continues traversal by updating the node ID and key index. - */ - function _processLeafOrExtension( - Node memory node, - Node[] memory trieProof, - bytes memory key, - bytes memory nodeId, - uint256 keyIndex, - uint256 i - ) private pure returns (bytes memory value, ProofError err) { - bytes memory path = _path(node); - uint8 prefix = uint8(path[0]); - uint8 offset = 2 - (prefix % 2); // Calculate offset based on even/odd path length - bytes memory pathRemainder = Bytes.slice(path, offset); // Path after the prefix - bytes memory keyRemainder = Bytes.slice(key, keyIndex); // Remaining key to match - uint256 sharedNibbleLength = _sharedNibbleLength(pathRemainder, keyRemainder); - - // Path must match at least partially with our key - if (sharedNibbleLength == 0) return ("", ProofError.INVALID_PATH_REMAINDER); - if (prefix > uint8(type(Prefix).max)) return ("", ProofError.UNKNOWN_NODE_PREFIX); - - // Leaf node (terminal) - return its value if key matches completely - if (Prefix(prefix) == Prefix.LEAF_EVEN || Prefix(prefix) == Prefix.LEAF_ODD) { - if (keyRemainder.length == 0) return ("", ProofError.INVALID_KEY_REMAINDER); - return _validateLastItem(node.decoded[1], trieProof, i); - } - - // Extension node (non-terminal) - continue to next node - // Increment keyIndex by the number of nibbles consumed - (nodeId, keyIndex) = (_id(node.decoded[1]), keyIndex + sharedNibbleLength); - } - - /** - * @dev Validates that we've reached a valid leaf value and this is the last proof element. - * Ensures the value is not empty and no extra proof elements exist. - */ - function _validateLastItem( - RLP.Item memory item, - Node[] memory trieProof, - uint256 i - ) private pure returns (bytes memory value, ProofError) { - bytes memory value_ = item.readBytes(); - if (value_.length == 0) return ("", ProofError.EMPTY_VALUE); - if (i != trieProof.length - 1) return ("", ProofError.INVALID_EXTRA_PROOF_ELEMENT); - return (value_, ProofError.NO_ERROR); - } - - /** - * @dev Converts raw proof bytes into structured Node objects with RLP parsing. - * Transforms each proof element into a Node with both encoded and decoded forms. - */ - function _decodeProof(bytes[] memory proof) private pure returns (Node[] memory proof_) { - uint256 length = proof.length; - proof_ = new Node[](length); - for (uint256 i = 0; i < length; i++) { - proof_[i] = Node(proof[i], proof[i].readList()); - } - } - - /** - * @dev Extracts the node ID (hash or raw data based on size). - * For small nodes (<32 bytes), returns the raw bytes; for large nodes, returns the hash. - */ - function _id(RLP.Item memory node) private pure returns (bytes memory) { - return node.length < 32 ? node.readRawBytes() : node.readBytes(); - } - - /** - * @dev Extracts the path from a leaf or extension node. - * The path is stored as the first element in the node's decoded array. - */ - function _path(Node memory node) private pure returns (bytes memory) { - return _nibbles(node.decoded[0].readBytes()); - } - - /** - * @dev Calculates the number of shared nibbles between two byte arrays. - * Used to determine how much of a path matches a key during trie traversal. - */ - function _sharedNibbleLength(bytes memory _a, bytes memory _b) private pure returns (uint256 shared_) { - uint256 max = Math.max(_a.length, _b.length); - uint256 length; - while (length < max && _a[length] == _b[length]) { - length++; - } - return length; - } - - /// @dev Split each byte in `value` into two nibbles (4 bits each). - function _nibbles(bytes memory value) internal pure returns (bytes memory) { - uint256 length = value.length; - bytes memory nibbles_ = new bytes(length * 2); - for (uint256 i = 0; i < length; i++) { - (nibbles_[i * 2], nibbles_[i * 2 + 1]) = (value[i] & 0xf0, value[i] & 0x0f); - } - return nibbles_; - } -} From d1aa944116ccc2183321722fb79791d5511f7629 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Fri, 1 Aug 2025 14:26:41 -0600 Subject: [PATCH 063/114] Add some tests --- contracts/utils/RLP.sol | 44 +++++----- test/helpers/enums.js | 1 + test/utils/RLP.test.js | 180 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 22 deletions(-) create mode 100644 test/utils/RLP.test.js diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 2ee7f7d21a1..78dcfc9aa96 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -33,8 +33,8 @@ library RLP { } enum ItemType { - DATA_ITEM, // Single data value - LIST_ITEM // List of RLP encoded items + Data, // Single data value + List // List of RLP encoded items } /** @@ -62,7 +62,7 @@ library RLP { /** * @dev Encodes an array of bytes using RLP (as a list). - * First it {_flatten}s the list of byte arrays, then encodes it with the list prefix. + * First it {_flatten}s the list of encoded items, then encodes it with the list prefix. */ function encode(bytes[] memory list) internal pure returns (bytes memory) { bytes memory flattened = _flatten(list); @@ -120,9 +120,9 @@ library RLP { } /// @dev Decodes an RLP encoded list into an array of RLP Items. See {_decodeLength} - function readList(Item memory item) internal pure returns (Item[] memory) { + function decodeList(Item memory item) internal pure returns (Item[] memory) { (uint256 listOffset, uint256 listLength, ItemType itemType) = _decodeLength(item); - require(itemType == ItemType.LIST_ITEM, RLPUnexpectedType(ItemType.LIST_ITEM, itemType)); + require(itemType == ItemType.List, RLPUnexpectedType(ItemType.List, itemType)); uint256 expectedLength = listOffset + listLength; require(expectedLength == item.length, RLPContentLengthMismatch(expectedLength, item.length)); Item[] memory items = new Item[](32); @@ -144,15 +144,15 @@ library RLP { return items; } - /// @dev Same as {readList} but for `bytes`. See {toItem}. - function readList(bytes memory value) internal pure returns (Item[] memory) { - return readList(toItem(value)); + /// @dev Same as {decodeList} but for `bytes`. See {toItem}. + function decodeList(bytes memory value) internal pure returns (Item[] memory) { + return decodeList(toItem(value)); } /// @dev Decodes an RLP encoded item. - function readBytes(Item memory item) internal pure returns (bytes memory) { + function decodeBytes(Item memory item) internal pure returns (bytes memory) { (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); - require(itemType == ItemType.DATA_ITEM, RLPUnexpectedType(ItemType.DATA_ITEM, itemType)); + require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); uint256 expectedLength = itemOffset + itemLength; require(expectedLength == item.length, RLPContentLengthMismatch(expectedLength, item.length)); @@ -162,13 +162,13 @@ library RLP { return result; } - /// @dev Same as {readBytes} but for `bytes`. See {toItem}. - function readBytes(bytes memory item) internal pure returns (bytes memory) { - return readBytes(toItem(item)); + /// @dev Same as {decodeBytes} but for `bytes`. See {toItem}. + function decodeBytes(bytes memory item) internal pure returns (bytes memory) { + return decodeBytes(toItem(item)); } /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. - function readRawBytes(Item memory item) internal pure returns (bytes memory) { + function decodeRawBytes(Item memory item) internal pure returns (bytes memory) { uint256 itemLength = item.length; bytes memory result = new bytes(itemLength); _copy(_addOffset(_asPointer(result), 32), item.ptr, itemLength); @@ -203,13 +203,13 @@ library RLP { return abi.encodePacked( bytes1(uint8(bytesLength) + uint8(offset) + SHORT_THRESHOLD), - bytes32(length).reverseBytes32() // to big-endian + _binaryBuffer(length) // already in big-endian, minimal representation ); } /// @dev Converts a uint256 to minimal binary representation, removing leading zeros. function _binaryBuffer(uint256 value) private pure returns (bytes memory) { - return abi.encodePacked(value).slice(value.clz()); + return abi.encodePacked(value).slice(value.clz() / 8); } /// @dev Concatenates all byte arrays in the `list` sequentially. Returns a flattened buffer. @@ -220,7 +220,7 @@ library RLP { for (uint256 i = 0; i < list.length; i++) { bytes memory item = list[i]; uint256 length = item.length; - _copy(dataPtr, _asPointer(item), length); + _copy(dataPtr, _addOffset(_asPointer(item), 32), length); dataPtr = _addOffset(dataPtr, length); } return flattened; @@ -245,7 +245,7 @@ library RLP { uint256 prefix = uint8(_loadByte(item.ptr, 0)); // Single byte below 128 - if (prefix < SHORT_OFFSET) return (0, 1, ItemType.DATA_ITEM); + if (prefix < SHORT_OFFSET) return (0, 1, ItemType.Data); // Short string (0-55 bytes) if (prefix < LONG_LENGTH_OFFSET) return _decodeShortString(prefix - SHORT_OFFSET, item); @@ -253,7 +253,7 @@ library RLP { // Long string (>55 bytes) if (prefix < LONG_OFFSET) { (offset, length) = _decodeLong(prefix - LONG_LENGTH_OFFSET, item); - return (offset, length, ItemType.DATA_ITEM); + return (offset, length, ItemType.Data); } // Short list @@ -261,7 +261,7 @@ library RLP { // Long list (offset, length) = _decodeLong(prefix - SHORT_LIST_OFFSET, item); - return (offset, length, ItemType.LIST_ITEM); + return (offset, length, ItemType.List); } /// @dev Decodes a short string (0-55 bytes). The first byte contains the length, and the rest is the payload. @@ -271,7 +271,7 @@ library RLP { ) private pure returns (uint256 offset, uint256 length, ItemType) { require(item.length > strLength, RLPInvalidDataRemainder(strLength, item.length)); require(strLength != 1 || _loadByte(_addOffset(item.ptr, 1), 0) >= bytes1(SHORT_OFFSET)); - return (1, strLength, ItemType.DATA_ITEM); + return (1, strLength, ItemType.Data); } /// @dev Decodes a short list (0-55 bytes). The first byte contains the length of the entire list. @@ -280,7 +280,7 @@ library RLP { Item memory item ) private pure returns (uint256 offset, uint256 length, ItemType) { require(item.length > listLength, RLPInvalidDataRemainder(listLength, item.length)); - return (1, listLength, ItemType.LIST_ITEM); + return (1, listLength, ItemType.List); } /// @dev Decodes a long string or list (>55 bytes). The first byte indicates the length of the length, followed by the length itself. diff --git a/test/helpers/enums.js b/test/helpers/enums.js index 6adbf64ad82..804262d4e7a 100644 --- a/test/helpers/enums.js +++ b/test/helpers/enums.js @@ -11,4 +11,5 @@ module.exports = { Rounding: EnumTyped('Floor', 'Ceil', 'Trunc', 'Expand'), OperationState: Enum('Unset', 'Waiting', 'Ready', 'Done'), RevertType: EnumTyped('None', 'RevertWithoutMessage', 'RevertWithMessage', 'RevertWithCustomError', 'Panic'), + ItemType: Enum('Data', 'List'), }; diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js new file mode 100644 index 00000000000..6d915ab8fd6 --- /dev/null +++ b/test/utils/RLP.test.js @@ -0,0 +1,180 @@ +const { ethers } = require('hardhat'); +const { expect } = require('chai'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +async function fixture() { + const mock = await ethers.deployContract('$RLP'); + + // Resolve function overload ambiguities like in Math.test.js + mock.$encode_bytes = mock['$encode(bytes)']; + mock.$encode_list = mock['$encode(bytes[])']; + mock.$encode_string = mock['$encode(string)']; + mock.$encode_address = mock['$encode(address)']; + mock.$encode_uint256 = mock['$encode(uint256)']; + mock.$encode_bytes32 = mock['$encode(bytes32)']; + mock.$encode_bool = mock['$encode(bool)']; + mock.$decodeBytes_item = mock['$decodeBytes((uint256,bytes32))']; + mock.$decodeBytes_bytes = mock['$decodeBytes(bytes)']; + mock.$decodeList_item = mock['$decodeList((uint256,bytes32))']; + mock.$decodeList_bytes = mock['$decodeList(bytes)']; + + return { mock }; +} + +describe('RLP', function () { + beforeEach(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + describe('encoding', function () { + it('encodes zero', async function () { + await expect(this.mock.$encode_uint256(0)).to.eventually.equal('0x80'); + }); + + it('encodes single byte < 128', async function () { + await expect(this.mock.$encode_bytes('0x00')).to.eventually.equal('0x00'); + await expect(this.mock.$encode_bytes('0x01')).to.eventually.equal('0x01'); + await expect(this.mock.$encode_bytes('0x7f')).to.eventually.equal('0x7f'); + }); + + it('encodes single byte >= 128', async function () { + await expect(this.mock.$encode_bytes('0x80')).to.eventually.equal('0x8180'); + await expect(this.mock.$encode_bytes('0xff')).to.eventually.equal('0x81ff'); + }); + + it('encodes short strings (0-55 bytes)', async function () { + // 1 byte + await expect(this.mock.$encode_bytes('0xab')).to.eventually.equal('0x81ab'); + + // 2 bytes + await expect(this.mock.$encode_bytes('0x1234')).to.eventually.equal('0x821234'); + + // 55 bytes (maximum for short encoding) + const fiftyFiveBytes = '0x' + '00'.repeat(55); + const expectedShort = '0xb7' + '00'.repeat(55); + await expect(this.mock.$encode_bytes(fiftyFiveBytes)).to.eventually.equal(expectedShort); + }); + + it('encodes long strings (>55 bytes)', async function () { + // 56 bytes (minimum for long encoding) + const fiftySixBytes = '0x' + '00'.repeat(56); + const expectedLong = '0xb838' + '00'.repeat(56); + await expect(this.mock.$encode_bytes(fiftySixBytes)).to.eventually.equal(expectedLong); + + // 100 bytes + const hundredBytes = '0x' + '00'.repeat(100); + const expectedHundred = '0xb864' + '00'.repeat(100); + await expect(this.mock.$encode_bytes(hundredBytes)).to.eventually.equal(expectedHundred); + }); + + it('encodes strings', async function () { + await expect(this.mock.$encode_string('')).to.eventually.equal('0x80'); + await expect(this.mock.$encode_string('dog')).to.eventually.equal('0x83646f67'); + await expect( + this.mock.$encode_string('Lorem ipsum dolor sit amet, consectetur adipisicing elit'), + ).to.eventually.equal( + '0xb8384c6f72656d20697073756d20646f6c6f722073697420616d65742c20636f6e7365637465747572206164697069736963696e6720656c6974', + ); + }); + + it('encodes addresses', async function () { + const addr = '0x1234567890123456789012345678901234567890'; + await expect(this.mock.$encode_address(addr)).to.eventually.equal('0x941234567890123456789012345678901234567890'); + }); + + it('encodes uint256', async function () { + await expect(this.mock.$encode_uint256(0)).to.eventually.equal('0x80'); + await expect(this.mock.$encode_uint256(1)).to.eventually.equal('0x01'); + await expect(this.mock.$encode_uint256(127)).to.eventually.equal('0x7f'); + await expect(this.mock.$encode_uint256(128)).to.eventually.equal('0x8180'); + await expect(this.mock.$encode_uint256(256)).to.eventually.equal('0x820100'); + await expect(this.mock.$encode_uint256(1024)).to.eventually.equal('0x820400'); + await expect(this.mock.$encode_uint256(0xffffff)).to.eventually.equal('0x83ffffff'); + }); + + it('encodes bytes32', async function () { + await expect( + this.mock.$encode_bytes32('0x0000000000000000000000000000000000000000000000000000000000000000'), + ).to.eventually.equal('0x80'); + await expect( + this.mock.$encode_bytes32('0x0000000000000000000000000000000000000000000000000000000000000001'), + ).to.eventually.equal('0x01'); + await expect( + this.mock.$encode_bytes32('0x1000000000000000000000000000000000000000000000000000000000000000'), + ).to.eventually.equal('0xa01000000000000000000000000000000000000000000000000000000000000000'); + }); + + it('encodes booleans', async function () { + await expect(this.mock.$encode_bool(false)).to.eventually.equal('0x80'); // 0 + await expect(this.mock.$encode_bool(true)).to.eventually.equal('0x01'); // 1 + }); + + it('encodes strict booleans', async function () { + await expect(this.mock.$encodeStrict(false)).to.eventually.equal('0x80'); // empty + await expect(this.mock.$encodeStrict(true)).to.eventually.equal('0x01'); // 0x01 + }); + + const validTests = [ + // Basic string encoding + { name: 'empty string', input: '' }, + { name: 'dog', input: 'dog' }, + { + name: 'Lorem ipsum', + input: 'Lorem ipsum dolor sit amet, consectetur adipisicing elit', + }, + + // Numeric encoding + { name: 'small integer 1', input: 1 }, + { name: 'small integer 16', input: 16 }, + { name: 'small integer 79', input: 79 }, + { name: 'small integer 127', input: 127 }, + { name: 'medium integer 128', input: 128 }, + { name: 'medium integer 1000', input: 1000 }, + { name: 'medium integer 100000', input: 100000 }, + + // List encoding + { name: 'empty list', input: [] }, + { name: 'list of strings', input: ['dog', 'god', 'cat'] }, + ]; + + validTests.forEach(({ name, input }) => { + it(`encodes ${name}`, async function () { + let encoded; + let expected; + + if (typeof input === 'string') { + encoded = await this.mock.$encode_string(input); + expected = ethers.encodeRlp(ethers.toUtf8Bytes(input)); + } else if (typeof input === 'number') { + encoded = await this.mock.$encode_uint256(input); + expected = ethers.encodeRlp(ethers.toBeHex(input)); + } else if (Array.isArray(input)) { + if (input.length === 0) { + encoded = await this.mock.$encode_list(input); + } else { + const encodedItems = input.map(item => ethers.encodeRlp(ethers.toUtf8Bytes(item))); + encoded = await this.mock.$encode_list(encodedItems); + } + expected = ethers.encodeRlp(input.map(item => ethers.toUtf8Bytes(item))); + } + + expect(encoded).to.equal(expected); + }); + }); + + // const invalidTests = [ + // { name: 'short string with invalid length', input: '0x8100' }, + // { name: 'long string with invalid length prefix', input: '0xb800' }, + // { name: 'list with invalid length', input: '0xc100' }, + // { name: 'truncated long string', input: '0xb838' }, + // { name: 'invalid single byte encoding (non-minimal)', input: '0x8100' }, + // ]; + + // invalidTests.forEach(({ name, input }) => { + // it(`encodes ${name} into invalid RLP`, async function () { + // const item = await this.mock.$toItem(input); + // await expect(this.mock.$decodeBytes_bytes(item)).to.be.reverted; + // }); + // }); + }); +}); From 590b7d7a55b3c88a48729ff8b140110f91597836 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Tue, 26 Aug 2025 10:17:52 -1000 Subject: [PATCH 064/114] Use Bytes.concat --- contracts/utils/RLP.sol | 57 ++++++++++------------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 78dcfc9aa96..0f4fbee1ffd 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -4,6 +4,7 @@ pragma solidity ^0.8.27; import {Math} from "./math/Math.sol"; import {Bytes} from "./Bytes.sol"; import {Memory} from "./Memory.sol"; +import {SafeCast} from "./math/SafeCast.sol"; /** * @dev Library for encoding and decoding data in RLP format. @@ -11,7 +12,8 @@ import {Memory} from "./Memory.sol"; * It's used for encoding everything from transactions to blocks to Patricia-Merkle tries. */ library RLP { - using Math for uint256; + using Math for *; + using SafeCast for bool; using Bytes for *; using Memory for *; @@ -62,10 +64,10 @@ library RLP { /** * @dev Encodes an array of bytes using RLP (as a list). - * First it {_flatten}s the list of encoded items, then encodes it with the list prefix. + * First it {Bytes-concat}s the list of encoded items, then encodes it with the list prefix. */ function encode(bytes[] memory list) internal pure returns (bytes memory) { - bytes memory flattened = _flatten(list); + bytes memory flattened = list.concat(); return bytes.concat(_encodeLength(flattened.length, LONG_OFFSET), flattened); } @@ -99,7 +101,7 @@ library RLP { * Use this for ecosystem compatibility; use {encodeStrict} for strict RLP spec compliance. */ function encode(bool value) internal pure returns (bytes memory) { - return encode(value ? uint256(1) : uint256(0)); + return encode(value.toUint()); } /** @@ -184,27 +186,18 @@ library RLP { /** * @dev Encodes a length with appropriate RLP prefix. * - * Uses short encoding for lengths <= 55 bytes (i.e. `abi.encodePacked(bytes1(uint8(length) + uint8(offset)))`). - * Uses long encoding for lengths > 55 bytes See {_encodeLongLength}. + * * For lengths <= 55 bytes, uses short encoding: [ length (1 byte) ++ offset (1 byte) ]. + * * For lengths > 55 bytes, uses long encoding using a length-of-length prefix: + * [ prefix ++ length of the length ++ length in big-endian ]. */ function _encodeLength(uint256 length, uint256 offset) private pure returns (bytes memory) { return length <= SHORT_THRESHOLD ? abi.encodePacked(bytes1(uint8(length) + uint8(offset))) - : _encodeLongLength(length, offset); - } - - /** - * @dev Encodes a long length value (>55 bytes) with a length-of-length prefix. - * Format: [prefix + length of the length] + [length in big-endian] - */ - function _encodeLongLength(uint256 length, uint256 offset) private pure returns (bytes memory) { - uint256 bytesLength = length.log256() + 1; // Result is floored - return - abi.encodePacked( - bytes1(uint8(bytesLength) + uint8(offset) + SHORT_THRESHOLD), - _binaryBuffer(length) // already in big-endian, minimal representation - ); + : abi.encodePacked( + bytes1(uint8(length.log256() + 1) + uint8(offset) + SHORT_THRESHOLD), + _binaryBuffer(length) // already in big-endian, minimal representation + ); } /// @dev Converts a uint256 to minimal binary representation, removing leading zeros. @@ -212,30 +205,6 @@ library RLP { return abi.encodePacked(value).slice(value.clz() / 8); } - /// @dev Concatenates all byte arrays in the `list` sequentially. Returns a flattened buffer. - function _flatten(bytes[] memory list) private pure returns (bytes memory) { - // TODO: Move to Arrays.sol - bytes memory flattened = new bytes(_totalLength(list)); - Memory.Pointer dataPtr = _addOffset(_asPointer(flattened), 32); - for (uint256 i = 0; i < list.length; i++) { - bytes memory item = list[i]; - uint256 length = item.length; - _copy(dataPtr, _addOffset(_asPointer(item), 32), length); - dataPtr = _addOffset(dataPtr, length); - } - return flattened; - } - - /// @dev Sums up the length of each array in the list. - function _totalLength(bytes[] memory list) private pure returns (uint256) { - // TODO: Move to Arrays.sol - uint256 totalLength; - for (uint256 i = 0; i < list.length; i++) { - totalLength += list[i].length; - } - return totalLength; - } - /** * @dev Decodes an RLP `item`'s `length and type from its prefix. * Returns the offset, length, and type of the RLP item based on the encoding rules. From 89280395b1e2347e330caf5fff0ad5428c93634c Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 27 Aug 2025 09:06:43 -1000 Subject: [PATCH 065/114] Use Math.ternary --- contracts/utils/RLP.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 0f4fbee1ffd..5fa1dde255e 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -112,7 +112,7 @@ library RLP { * Use this for strict RLP spec compliance; use {encode} for ecosystem compatibility. */ function encodeStrict(bool value) internal pure returns (bytes memory) { - return value ? abi.encodePacked(bytes1(0x01)) : encode(new bytes(0)); + return abi.encodePacked(bytes1(bytes32(value.ternary(0x01, 0x80))[31])); } /// @dev Creates an RLP Item from a bytes array. From bb027dfeae2a6aec5abf495beacf9507f5bc5ced Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 4 Sep 2025 22:51:59 +0200 Subject: [PATCH 066/114] refactor encoding to reduce memory allocation --- contracts/utils/RLP.sol | 207 +++++++++++++++++++++------------------- test/utils/RLP.test.js | 5 - 2 files changed, 111 insertions(+), 101 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 5fa1dde255e..4ee137e1dac 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -4,7 +4,6 @@ pragma solidity ^0.8.27; import {Math} from "./math/Math.sol"; import {Bytes} from "./Bytes.sol"; import {Memory} from "./Memory.sol"; -import {SafeCast} from "./math/SafeCast.sol"; /** * @dev Library for encoding and decoding data in RLP format. @@ -12,63 +11,84 @@ import {SafeCast} from "./math/SafeCast.sol"; * It's used for encoding everything from transactions to blocks to Patricia-Merkle tries. */ library RLP { - using Math for *; - using SafeCast for bool; - using Bytes for *; using Memory for *; - /// @dev Items with length 0 are not RLP items. - error RLPEmptyItem(); - - /// @dev The `item` is not of the `expected` type. - error RLPUnexpectedType(ItemType expected, ItemType actual); - - /// @dev The item is not long enough to contain the data. - error RLPInvalidDataRemainder(uint256 minLength, uint256 actualLength); - - /// @dev The content length does not match the expected length. - error RLPContentLengthMismatch(uint256 expectedLength, uint256 actualLength); - - struct Item { - uint256 length; // Total length of the item in bytes - Memory.Pointer ptr; // Memory pointer to the start of the item - } - - enum ItemType { - Data, // Single data value - List // List of RLP encoded items - } - /** * @dev Maximum length for data that will be encoded using the short format. * If `data.length <= 55 bytes`, it will be encoded as: `[0x80 + length]` + data. */ uint8 internal constant SHORT_THRESHOLD = 55; - /// @dev Single byte prefix for short strings (0-55 bytes) - uint8 internal constant SHORT_OFFSET = 128; + uint8 internal constant SHORT_OFFSET = 0x80; + /// @dev Prefix for list items (0xC0) + uint8 internal constant LONG_OFFSET = 0xC0; + /// @dev Prefix for long string length (0xB8) uint8 internal constant LONG_LENGTH_OFFSET = SHORT_OFFSET + SHORT_THRESHOLD + 1; // 184 - /// @dev Prefix for list items (0xC0) - uint8 internal constant LONG_OFFSET = LONG_LENGTH_OFFSET + 8; // 192 /// @dev Prefix for long list length (0xF8) uint8 internal constant SHORT_LIST_OFFSET = LONG_OFFSET + SHORT_THRESHOLD + 1; // 248 + /**************************************************************************************************************** + * ENCODING * + ****************************************************************************************************************/ + /** - * @dev Encodes a bytes array using RLP rules. - * Single bytes below 128 are encoded as themselves, otherwise as length prefix + data. + * @dev Convenience method to encode a boolean as RLP. + * + * Boolean `true` is encoded as 0x01, `false` as 0x80 (equivalent to encoding integers 1 and 0). + * This follows the de facto ecosystem standard where booleans are treated as 0/1 integers. */ - function encode(bytes memory buffer) internal pure returns (bytes memory) { - return _isSingleByte(buffer) ? buffer : bytes.concat(_encodeLength(buffer.length, SHORT_OFFSET), buffer); + function encode(bool input) internal pure returns (bytes memory result) { + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, 0x01) + mstore(add(result, 0x20), shl(add(248, mul(7, iszero(input))), 1)) + mstore(0x40, add(result, 0x21)) + } + } + + /// @dev Convenience method to encode an address as RLP bytes (i.e. encoded as packed 20 bytes). + function encode(address input) internal pure returns (bytes memory result) { + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, 0x15) + mstore(add(result, 0x20), or(shl(248, 0x94), shl(88, input))) + mstore(0x40, add(result, 0x35)) + } + } + + /// @dev Convenience method to encode a uint256 as RLP. + function encode(uint256 input) internal pure returns (bytes memory result) { + if (input < SHORT_OFFSET) { + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, 1) + mstore(add(result, 0x20), shl(248, or(input, mul(0x80, iszero(input))))) // zero is encoded as 0x80 + mstore(0x40, add(result, 0x21)) + } + } else { + uint256 length = Math.log256(input) + 1; + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, add(length, 1)) + mstore8(add(result, 0x20), add(length, 0x80)) + mstore(add(result, 0x21), shl(sub(256, mul(8, length)), input)) + mstore(0x40, add(result, add(length, 0x21))) + } + } + } + + /// @dev Same as {encode-uint256-}, but for bytes32. + function encode(bytes32 input) internal pure returns (bytes memory) { + return encode(uint256(input)); } /** - * @dev Encodes an array of bytes using RLP (as a list). - * First it {Bytes-concat}s the list of encoded items, then encodes it with the list prefix. + * @dev Encodes a bytes array using RLP rules. + * Single bytes below 128 are encoded as themselves, otherwise as length prefix + data. */ - function encode(bytes[] memory list) internal pure returns (bytes memory) { - bytes memory flattened = list.concat(); - return bytes.concat(_encodeLength(flattened.length, LONG_OFFSET), flattened); + function encode(bytes memory input) internal pure returns (bytes memory) { + return (input.length == 1 && uint8(input[0]) < SHORT_OFFSET) ? input : _encode(input, SHORT_OFFSET); } /// @dev Convenience method to encode a string as RLP. @@ -76,43 +96,65 @@ library RLP { return encode(bytes(str)); } - /// @dev Convenience method to encode an address as RLP bytes (i.e. encoded as packed 20 bytes). - function encode(address addr) internal pure returns (bytes memory) { - return encode(abi.encodePacked(addr)); + /** + * @dev Encodes an array of bytes using RLP (as a list). + * First it {Bytes-concat}s the list of encoded items, then encodes it with the list prefix. + */ + function encode(bytes[] memory input) internal pure returns (bytes memory) { + return _encode(Bytes.concat(input), LONG_OFFSET); } - /// @dev Convenience method to encode a uint256 as RLP. See {_binaryBuffer}. - function encode(uint256 value) internal pure returns (bytes memory) { - return encode(_binaryBuffer(value)); + function _encode(bytes memory input, uint256 offset) private pure returns (bytes memory result) { + uint256 length = input.length; + if (length <= SHORT_THRESHOLD) { + // Encode "short-bytes" as + // [ 0x80 + input.length | input ] + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, add(length, 1)) + mstore8(add(result, 0x20), add(length, offset)) + mcopy(add(result, 0x21), add(input, 0x20), length) + mstore(0x40, add(result, add(length, 0x21))) + } + } else { + // Encode "long-bytes" as + // [ 0xb7 + input.length.length | input.length | input ] + uint256 lenlength = Math.log256(length) + 1; + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, add(add(length, lenlength), 1)) + mstore8(add(result, 0x20), add(add(lenlength, offset), SHORT_THRESHOLD)) + mstore(add(result, 0x21), shl(sub(256, mul(8, lenlength)), length)) + mcopy(add(result, add(lenlength, 0x21)), add(input, 0x20), length) + mstore(0x40, add(result, add(add(length, lenlength), 0x21))) + } + } } - /// @dev Same as {encode-uint256-}, but for bytes32. - function encode(bytes32 value) internal pure returns (bytes memory) { - return encode(uint256(value)); - } + /**************************************************************************************************************** + * DECODING * + ****************************************************************************************************************/ - /** - * @dev Convenience method to encode a boolean as RLP. - * - * Boolean `true` is encoded as 0x01, `false` as 0x80 (equivalent to encoding integers 1 and 0). - * This follows the de facto ecosystem standard where booleans are treated as 0/1 integers. - * - * NOTE: Both this and {encodeStrict} produce identical encoded bytes at the output level. - * Use this for ecosystem compatibility; use {encodeStrict} for strict RLP spec compliance. - */ - function encode(bool value) internal pure returns (bytes memory) { - return encode(value.toUint()); + /// @dev Items with length 0 are not RLP items. + error RLPEmptyItem(); + + /// @dev The `item` is not of the `expected` type. + error RLPUnexpectedType(ItemType expected, ItemType actual); + + /// @dev The item is not long enough to contain the data. + error RLPInvalidDataRemainder(uint256 minLength, uint256 actualLength); + + /// @dev The content length does not match the expected length. + error RLPContentLengthMismatch(uint256 expectedLength, uint256 actualLength); + + struct Item { + uint256 length; // Total length of the item in bytes + Memory.Pointer ptr; // Memory pointer to the start of the item } - /** - * @dev Strict RLP encoding of a boolean following literal spec interpretation. - * Boolean `true` is encoded as 0x01, `false` as empty bytes (0x80). - * - * NOTE: This is the strict RLP spec interpretation where false represents "empty". - * Use this for strict RLP spec compliance; use {encode} for ecosystem compatibility. - */ - function encodeStrict(bool value) internal pure returns (bytes memory) { - return abi.encodePacked(bytes1(bytes32(value.ternary(0x01, 0x80))[31])); + enum ItemType { + Data, // Single data value + List // List of RLP encoded items } /// @dev Creates an RLP Item from a bytes array. @@ -178,33 +220,6 @@ library RLP { return result; } - /// @dev Checks if a buffer is a single byte below 128 (0x80). Encoded as-is in RLP. - function _isSingleByte(bytes memory buffer) private pure returns (bool) { - return buffer.length == 1 && uint8(buffer[0]) < SHORT_OFFSET; - } - - /** - * @dev Encodes a length with appropriate RLP prefix. - * - * * For lengths <= 55 bytes, uses short encoding: [ length (1 byte) ++ offset (1 byte) ]. - * * For lengths > 55 bytes, uses long encoding using a length-of-length prefix: - * [ prefix ++ length of the length ++ length in big-endian ]. - */ - function _encodeLength(uint256 length, uint256 offset) private pure returns (bytes memory) { - return - length <= SHORT_THRESHOLD - ? abi.encodePacked(bytes1(uint8(length) + uint8(offset))) - : abi.encodePacked( - bytes1(uint8(length.log256() + 1) + uint8(offset) + SHORT_THRESHOLD), - _binaryBuffer(length) // already in big-endian, minimal representation - ); - } - - /// @dev Converts a uint256 to minimal binary representation, removing leading zeros. - function _binaryBuffer(uint256 value) private pure returns (bytes memory) { - return abi.encodePacked(value).slice(value.clz() / 8); - } - /** * @dev Decodes an RLP `item`'s `length and type from its prefix. * Returns the offset, length, and type of the RLP item based on the encoding rules. diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js index 6d915ab8fd6..e296b262fc0 100644 --- a/test/utils/RLP.test.js +++ b/test/utils/RLP.test.js @@ -109,11 +109,6 @@ describe('RLP', function () { await expect(this.mock.$encode_bool(true)).to.eventually.equal('0x01'); // 1 }); - it('encodes strict booleans', async function () { - await expect(this.mock.$encodeStrict(false)).to.eventually.equal('0x80'); // empty - await expect(this.mock.$encodeStrict(true)).to.eventually.equal('0x01'); // 0x01 - }); - const validTests = [ // Basic string encoding { name: 'empty string', input: '' }, From ec4a5c1af9be6ab938623cd48e98335b782a3297 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 4 Sep 2025 23:13:26 +0200 Subject: [PATCH 067/114] refactor tests to use ethers.encodeRlp as a reference --- test/utils/RLP.test.js | 178 ++++++++++++++++------------------------- 1 file changed, 68 insertions(+), 110 deletions(-) diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js index e296b262fc0..4905c3764c2 100644 --- a/test/utils/RLP.test.js +++ b/test/utils/RLP.test.js @@ -2,17 +2,19 @@ const { ethers } = require('hardhat'); const { expect } = require('chai'); const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); +const { generators } = require('../helpers/random'); + async function fixture() { const mock = await ethers.deployContract('$RLP'); // Resolve function overload ambiguities like in Math.test.js - mock.$encode_bytes = mock['$encode(bytes)']; - mock.$encode_list = mock['$encode(bytes[])']; - mock.$encode_string = mock['$encode(string)']; + mock.$encode_bool = mock['$encode(bool)']; mock.$encode_address = mock['$encode(address)']; mock.$encode_uint256 = mock['$encode(uint256)']; mock.$encode_bytes32 = mock['$encode(bytes32)']; - mock.$encode_bool = mock['$encode(bool)']; + mock.$encode_bytes = mock['$encode(bytes)']; + mock.$encode_string = mock['$encode(string)']; + mock.$encode_list = mock['$encode(bytes[])']; mock.$decodeBytes_item = mock['$decodeBytes((uint256,bytes32))']; mock.$decodeBytes_bytes = mock['$decodeBytes(bytes)']; mock.$decodeList_item = mock['$decodeList((uint256,bytes32))']; @@ -27,69 +29,20 @@ describe('RLP', function () { }); describe('encoding', function () { - it('encodes zero', async function () { - await expect(this.mock.$encode_uint256(0)).to.eventually.equal('0x80'); - }); - - it('encodes single byte < 128', async function () { - await expect(this.mock.$encode_bytes('0x00')).to.eventually.equal('0x00'); - await expect(this.mock.$encode_bytes('0x01')).to.eventually.equal('0x01'); - await expect(this.mock.$encode_bytes('0x7f')).to.eventually.equal('0x7f'); - }); - - it('encodes single byte >= 128', async function () { - await expect(this.mock.$encode_bytes('0x80')).to.eventually.equal('0x8180'); - await expect(this.mock.$encode_bytes('0xff')).to.eventually.equal('0x81ff'); - }); - - it('encodes short strings (0-55 bytes)', async function () { - // 1 byte - await expect(this.mock.$encode_bytes('0xab')).to.eventually.equal('0x81ab'); - - // 2 bytes - await expect(this.mock.$encode_bytes('0x1234')).to.eventually.equal('0x821234'); - - // 55 bytes (maximum for short encoding) - const fiftyFiveBytes = '0x' + '00'.repeat(55); - const expectedShort = '0xb7' + '00'.repeat(55); - await expect(this.mock.$encode_bytes(fiftyFiveBytes)).to.eventually.equal(expectedShort); - }); - - it('encodes long strings (>55 bytes)', async function () { - // 56 bytes (minimum for long encoding) - const fiftySixBytes = '0x' + '00'.repeat(56); - const expectedLong = '0xb838' + '00'.repeat(56); - await expect(this.mock.$encode_bytes(fiftySixBytes)).to.eventually.equal(expectedLong); - - // 100 bytes - const hundredBytes = '0x' + '00'.repeat(100); - const expectedHundred = '0xb864' + '00'.repeat(100); - await expect(this.mock.$encode_bytes(hundredBytes)).to.eventually.equal(expectedHundred); - }); - - it('encodes strings', async function () { - await expect(this.mock.$encode_string('')).to.eventually.equal('0x80'); - await expect(this.mock.$encode_string('dog')).to.eventually.equal('0x83646f67'); - await expect( - this.mock.$encode_string('Lorem ipsum dolor sit amet, consectetur adipisicing elit'), - ).to.eventually.equal( - '0xb8384c6f72656d20697073756d20646f6c6f722073697420616d65742c20636f6e7365637465747572206164697069736963696e6720656c6974', - ); + it('encodes booleans', async function () { + await expect(this.mock.$encode_bool(false)).to.eventually.equal('0x80'); // 0 + await expect(this.mock.$encode_bool(true)).to.eventually.equal('0x01'); // 1 }); it('encodes addresses', async function () { - const addr = '0x1234567890123456789012345678901234567890'; - await expect(this.mock.$encode_address(addr)).to.eventually.equal('0x941234567890123456789012345678901234567890'); + const addr = generators.address(); + await expect(this.mock.$encode_address(addr)).to.eventually.equal(ethers.encodeRlp(addr)); }); it('encodes uint256', async function () { - await expect(this.mock.$encode_uint256(0)).to.eventually.equal('0x80'); - await expect(this.mock.$encode_uint256(1)).to.eventually.equal('0x01'); - await expect(this.mock.$encode_uint256(127)).to.eventually.equal('0x7f'); - await expect(this.mock.$encode_uint256(128)).to.eventually.equal('0x8180'); - await expect(this.mock.$encode_uint256(256)).to.eventually.equal('0x820100'); - await expect(this.mock.$encode_uint256(1024)).to.eventually.equal('0x820400'); - await expect(this.mock.$encode_uint256(0xffffff)).to.eventually.equal('0x83ffffff'); + for (const input of [0, 1, 127, 128, 256, 1024, 0xffffff, ethers.MaxUint256]) { + await expect(this.mock.$encode_uint256(input)).to.eventually.equal(ethers.encodeRlp(ethers.toBeArray(input))); + } }); it('encodes bytes32', async function () { @@ -104,57 +57,62 @@ describe('RLP', function () { ).to.eventually.equal('0xa01000000000000000000000000000000000000000000000000000000000000000'); }); - it('encodes booleans', async function () { - await expect(this.mock.$encode_bool(false)).to.eventually.equal('0x80'); // 0 - await expect(this.mock.$encode_bool(true)).to.eventually.equal('0x01'); // 1 + it('encodes empty byte', async function () { + const input = '0x'; + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); + }); + + it('encodes single byte < 128', async function () { + for (const input of ['0x00', '0x01', '0x7f']) { + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); + } + }); + + it('encodes single byte >= 128', async function () { + for (const input of ['0x80', '0xff']) { + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); + } + }); + + it('encodes short buffers (1-55 bytes)', async function () { + for (const input of [ + '0xab', // 1 byte + '0x1234', // 2 bytes + generators.bytes(55), // 55 bytes (maximum for short encoding) + ]) { + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); + } + }); + + it('encodes long buffers (>55 bytes)', async function () { + for (const input of [ + generators.bytes(56), // 56 bytes (minimum for long encoding) + generators.bytes(128), // 128 bytes + ]) { + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); + } + }); + + it('encodes strings', async function () { + for (const input of [ + '', // empty string + 'dog', + 'Lorem ipsum dolor sit amet, consectetur adipisicing elit', + ]) { + await expect(this.mock.$encode_string(input)).to.eventually.equal(ethers.encodeRlp(ethers.toUtf8Bytes(input))); + } }); - const validTests = [ - // Basic string encoding - { name: 'empty string', input: '' }, - { name: 'dog', input: 'dog' }, - { - name: 'Lorem ipsum', - input: 'Lorem ipsum dolor sit amet, consectetur adipisicing elit', - }, - - // Numeric encoding - { name: 'small integer 1', input: 1 }, - { name: 'small integer 16', input: 16 }, - { name: 'small integer 79', input: 79 }, - { name: 'small integer 127', input: 127 }, - { name: 'medium integer 128', input: 128 }, - { name: 'medium integer 1000', input: 1000 }, - { name: 'medium integer 100000', input: 100000 }, - - // List encoding - { name: 'empty list', input: [] }, - { name: 'list of strings', input: ['dog', 'god', 'cat'] }, - ]; - - validTests.forEach(({ name, input }) => { - it(`encodes ${name}`, async function () { - let encoded; - let expected; - - if (typeof input === 'string') { - encoded = await this.mock.$encode_string(input); - expected = ethers.encodeRlp(ethers.toUtf8Bytes(input)); - } else if (typeof input === 'number') { - encoded = await this.mock.$encode_uint256(input); - expected = ethers.encodeRlp(ethers.toBeHex(input)); - } else if (Array.isArray(input)) { - if (input.length === 0) { - encoded = await this.mock.$encode_list(input); - } else { - const encodedItems = input.map(item => ethers.encodeRlp(ethers.toUtf8Bytes(item))); - encoded = await this.mock.$encode_list(encodedItems); - } - expected = ethers.encodeRlp(input.map(item => ethers.toUtf8Bytes(item))); - } - - expect(encoded).to.equal(expected); - }); + it('encode(bytes[])', async function () { + for (const input of [ + [], + ['0x'], + ['0x00'], + ['0x17', '0x42'], + ['0x17', '0x', '0x42', '0x0123456789abcdef', '0x'], + ]) { + await expect(this.mock.$encode_list(input.map(ethers.encodeRlp))).to.eventually.equal(ethers.encodeRlp(input)); + } }); // const invalidTests = [ From b75dc7c3d9bbe8f0f0e9dfe7d1fa3b8d5372c43a Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 4 Sep 2025 23:18:15 +0200 Subject: [PATCH 068/114] fix comments --- contracts/utils/RLP.sol | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 4ee137e1dac..f4e328a42fd 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -71,7 +71,7 @@ library RLP { assembly ("memory-safe") { result := mload(0x40) mstore(result, add(length, 1)) - mstore8(add(result, 0x20), add(length, 0x80)) + mstore8(add(result, 0x20), add(length, SHORT_OFFSET)) mstore(add(result, 0x21), shl(sub(256, mul(8, length)), input)) mstore(0x40, add(result, add(length, 0x21))) } @@ -108,7 +108,7 @@ library RLP { uint256 length = input.length; if (length <= SHORT_THRESHOLD) { // Encode "short-bytes" as - // [ 0x80 + input.length | input ] + // [ offset + input.length | input ] assembly ("memory-safe") { result := mload(0x40) mstore(result, add(length, 1)) @@ -118,7 +118,7 @@ library RLP { } } else { // Encode "long-bytes" as - // [ 0xb7 + input.length.length | input.length | input ] + // [ SHORT_THRESHOLD + offset + input.length.length | input.length | input ] uint256 lenlength = Math.log256(length) + 1; assembly ("memory-safe") { result := mload(0x40) From fc208c813e5315bdbffed0c6bffcb20d07607103 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 4 Sep 2025 23:29:23 +0200 Subject: [PATCH 069/114] use mstore8 to avoid shift --- contracts/utils/RLP.sol | 44 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index f4e328a42fd..8403c00abc0 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -41,9 +41,9 @@ library RLP { function encode(bool input) internal pure returns (bytes memory result) { assembly ("memory-safe") { result := mload(0x40) - mstore(result, 0x01) - mstore(add(result, 0x20), shl(add(248, mul(7, iszero(input))), 1)) - mstore(0x40, add(result, 0x21)) + mstore(result, 0x01) // length of the encoded data: 1 byte + mstore8(add(result, 0x20), shl(mul(7, iszero(input)), 1)) // input + mstore(0x40, add(result, 0x21)) // reserve memory } } @@ -51,9 +51,9 @@ library RLP { function encode(address input) internal pure returns (bytes memory result) { assembly ("memory-safe") { result := mload(0x40) - mstore(result, 0x15) - mstore(add(result, 0x20), or(shl(248, 0x94), shl(88, input))) - mstore(0x40, add(result, 0x35)) + mstore(result, 0x15) // length of the encoded data: 1 (prefix) + 14 (address) + mstore(add(result, 0x20), or(shl(248, 0x94), shl(88, input))) // prefix (0x94 = SHORT_OFFSET + 14) + input + mstore(0x40, add(result, 0x35)) // reserve memory } } @@ -62,18 +62,18 @@ library RLP { if (input < SHORT_OFFSET) { assembly ("memory-safe") { result := mload(0x40) - mstore(result, 1) - mstore(add(result, 0x20), shl(248, or(input, mul(0x80, iszero(input))))) // zero is encoded as 0x80 - mstore(0x40, add(result, 0x21)) + mstore(result, 1) // length of the encoded data: 1 byte + mstore8(add(result, 0x20), or(input, mul(0x80, iszero(input)))) // input (zero is encoded as 0x80) + mstore(0x40, add(result, 0x21)) // reserve memory } } else { uint256 length = Math.log256(input) + 1; assembly ("memory-safe") { result := mload(0x40) - mstore(result, add(length, 1)) - mstore8(add(result, 0x20), add(length, SHORT_OFFSET)) - mstore(add(result, 0x21), shl(sub(256, mul(8, length)), input)) - mstore(0x40, add(result, add(length, 0x21))) + mstore(result, add(length, 1)) // length of the encoded data: 1 (prefix) + length + mstore8(add(result, 0x20), add(length, SHORT_OFFSET)) // prefix: SHORT_OFFSET + length + mstore(add(result, 0x21), shl(sub(256, mul(8, length)), input)) // input (aligned left) + mstore(0x40, add(result, add(length, 0x21))) // reserve memory } } } @@ -111,10 +111,10 @@ library RLP { // [ offset + input.length | input ] assembly ("memory-safe") { result := mload(0x40) - mstore(result, add(length, 1)) - mstore8(add(result, 0x20), add(length, offset)) - mcopy(add(result, 0x21), add(input, 0x20), length) - mstore(0x40, add(result, add(length, 0x21))) + mstore(result, add(length, 1)) // length of the encoded data: 1 (prefix) + input.length + mstore8(add(result, 0x20), add(length, offset)) // prefix: offset + input.length + mcopy(add(result, 0x21), add(input, 0x20), length) // input + mstore(0x40, add(result, add(length, 0x21))) // reserve memory } } else { // Encode "long-bytes" as @@ -122,11 +122,11 @@ library RLP { uint256 lenlength = Math.log256(length) + 1; assembly ("memory-safe") { result := mload(0x40) - mstore(result, add(add(length, lenlength), 1)) - mstore8(add(result, 0x20), add(add(lenlength, offset), SHORT_THRESHOLD)) - mstore(add(result, 0x21), shl(sub(256, mul(8, lenlength)), length)) - mcopy(add(result, add(lenlength, 0x21)), add(input, 0x20), length) - mstore(0x40, add(result, add(add(length, lenlength), 0x21))) + mstore(result, add(add(length, lenlength), 1)) // length of the encoded data: 1 (prefix) + input.length.length + input.length + mstore8(add(result, 0x20), add(add(lenlength, offset), SHORT_THRESHOLD)) // prefix: SHORT_THRESHOLD + offset + input.length.length + mstore(add(result, 0x21), shl(sub(256, mul(8, lenlength)), length)) // input.length + mcopy(add(result, add(lenlength, 0x21)), add(input, 0x20), length) // input + mstore(0x40, add(result, add(add(length, lenlength), 0x21))) // reserve memory } } } From fe0597c6edbd06eadac55c5d217ca6805132c5b0 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Sat, 6 Sep 2025 21:01:13 +0200 Subject: [PATCH 070/114] decoding works --- contracts/utils/RLP.sol | 213 +++++++++++++++++++--------------------- test/utils/RLP.t.sol | 87 ++++++++++++++++ 2 files changed, 189 insertions(+), 111 deletions(-) create mode 100644 test/utils/RLP.t.sol diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 8403c00abc0..e025bbc0dbf 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -4,6 +4,7 @@ pragma solidity ^0.8.27; import {Math} from "./math/Math.sol"; import {Bytes} from "./Bytes.sol"; import {Memory} from "./Memory.sol"; +import {Panic} from "./Panic.sol"; /** * @dev Library for encoding and decoding data in RLP format. @@ -147,7 +148,8 @@ library RLP { /// @dev The content length does not match the expected length. error RLPContentLengthMismatch(uint256 expectedLength, uint256 actualLength); - struct Item { + // Memory slice (equivalent of a calldata slice in memory) + struct Decoder { uint256 length; // Total length of the item in bytes Memory.Pointer ptr; // Memory pointer to the start of the item } @@ -157,155 +159,144 @@ library RLP { List // List of RLP encoded items } - /// @dev Creates an RLP Item from a bytes array. - function toItem(bytes memory value) internal pure returns (Item memory) { - require(value.length != 0, RLPEmptyItem()); // Empty arrays are not RLP items. - return Item(value.length, _addOffset(_asPointer(value), 32)); + function decoder(bytes memory self) internal pure returns (Decoder memory result) { + require(self.length != 0, RLPEmptyItem()); // Empty arrays are not RLP items. + + assembly ("memory-safe") { + mstore(result, mload(self)) + mstore(add(result, 0x20), add(self, 0x20)) + } + } + + function _toBytes(Decoder memory self) private pure returns (bytes memory result) { + return _toBytes(self, 0, self.length); + } + + function _toBytes(Decoder memory self, uint256 offset, uint256 length) private pure returns (bytes memory result) { + // TODO: do we want to emit RLPContentLengthMismatch instead? + // Do we want to check equality? + if (self.length < offset + length) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); + + Memory.Pointer ptr = self.ptr; + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, length) + mcopy(add(result, 0x20), add(ptr, offset), length) + mstore(0x40, add(result, add(0x20, length))) + } + } + + /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. + // TODO: is there a usecase for that? + function readRawBytes(Decoder memory item) internal pure returns (bytes memory) { + return _toBytes(item); + } + + /// @dev Decodes an RLP encoded item. + function readBytes(Decoder memory item) internal pure returns (bytes memory) { + (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); + require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); + + // Length is checked by {toBytes} + return _toBytes(item, itemOffset, itemLength); + } + + function readList(Decoder memory item) internal pure returns (Decoder[] memory) { + return readList(item, 32); } /// @dev Decodes an RLP encoded list into an array of RLP Items. See {_decodeLength} - function decodeList(Item memory item) internal pure returns (Item[] memory) { + function readList(Decoder memory item, uint256 maxListLength) internal pure returns (Decoder[] memory) { (uint256 listOffset, uint256 listLength, ItemType itemType) = _decodeLength(item); require(itemType == ItemType.List, RLPUnexpectedType(ItemType.List, itemType)); - uint256 expectedLength = listOffset + listLength; - require(expectedLength == item.length, RLPContentLengthMismatch(expectedLength, item.length)); - Item[] memory items = new Item[](32); + require(item.length == listOffset + listLength, RLPContentLengthMismatch(listOffset + listLength, item.length)); - uint256 itemCount; + Decoder[] memory list = new Decoder[](maxListLength); + uint256 itemCount; for (uint256 currentOffset = listOffset; currentOffset < item.length; ++itemCount) { (uint256 itemOffset, uint256 itemLength, ) = _decodeLength( - Item(item.length - currentOffset, _addOffset(item.ptr, currentOffset)) + Decoder(item.length - currentOffset, _addOffset(item.ptr, currentOffset)) ); - items[itemCount] = Item(itemLength + itemOffset, _addOffset(item.ptr, currentOffset)); + list[itemCount] = Decoder(itemLength + itemOffset, _addOffset(item.ptr, currentOffset)); currentOffset += itemOffset + itemLength; } // Decrease the array size to match the actual item count. assembly ("memory-safe") { - mstore(items, itemCount) + mstore(list, itemCount) } - return items; - } - - /// @dev Same as {decodeList} but for `bytes`. See {toItem}. - function decodeList(bytes memory value) internal pure returns (Item[] memory) { - return decodeList(toItem(value)); + return list; } - /// @dev Decodes an RLP encoded item. - function decodeBytes(Item memory item) internal pure returns (bytes memory) { - (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); - require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); - uint256 expectedLength = itemOffset + itemLength; - require(expectedLength == item.length, RLPContentLengthMismatch(expectedLength, item.length)); - - bytes memory result = new bytes(itemLength); - _copy(_addOffset(_asPointer(result), 32), _addOffset(item.ptr, itemOffset), itemLength); - - return result; - } - - /// @dev Same as {decodeBytes} but for `bytes`. See {toItem}. + /// @dev Same as {decodeBytes} but for `bytes`. See {decode}. function decodeBytes(bytes memory item) internal pure returns (bytes memory) { - return decodeBytes(toItem(item)); + return readBytes(decoder(item)); } - /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. - function decodeRawBytes(Item memory item) internal pure returns (bytes memory) { - uint256 itemLength = item.length; - bytes memory result = new bytes(itemLength); - _copy(_addOffset(_asPointer(result), 32), item.ptr, itemLength); - - return result; + /// @dev Same as {decodeList} but for `bytes`. See {decode}. + function decodeList(bytes memory value) internal pure returns (Decoder[] memory) { + return readList(decoder(value)); } /** * @dev Decodes an RLP `item`'s `length and type from its prefix. * Returns the offset, length, and type of the RLP item based on the encoding rules. */ - function _decodeLength(Item memory item) private pure returns (uint256 offset, uint256 length, ItemType) { + function _decodeLength(Decoder memory item) private pure returns (uint256 offset, uint256 length, ItemType) { require(item.length != 0, RLPEmptyItem()); - uint256 prefix = uint8(_loadByte(item.ptr, 0)); + uint8 prefix = uint8(bytes1(_load(item.ptr, 0))); + + if (prefix < SHORT_OFFSET) { + // Case: Single byte below 128 + return (0, 1, ItemType.Data); + } else if (prefix < LONG_LENGTH_OFFSET) { + // Case: Short string (0-55 bytes) + uint256 strLength = prefix - SHORT_OFFSET; + require(item.length > strLength, RLPInvalidDataRemainder(strLength, item.length)); + if (strLength == 1) { + require(bytes1(_load(item.ptr, 1)) >= bytes1(SHORT_OFFSET)); + } + return (1, strLength, ItemType.Data); + } else if (prefix < LONG_OFFSET) { + // Case: Long string (>55 bytes) + uint256 lengthLength = prefix - 0xb7; + + require(item.length > lengthLength, RLPInvalidDataRemainder(lengthLength, item.length)); + require(bytes1(_load(item.ptr, 0)) != 0x00); + + uint256 len = uint256(_load(item.ptr, 1)) >> (256 - 8 * lengthLength); + require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); + require(item.length > lengthLength + len, RLPContentLengthMismatch(lengthLength + len, item.length)); + + return (lengthLength + 1, len, ItemType.Data); + } else if (prefix < SHORT_LIST_OFFSET) { + // Case: Short list + uint256 listLength = prefix - LONG_OFFSET; + require(item.length > listLength, RLPInvalidDataRemainder(listLength, item.length)); + return (1, listLength, ItemType.List); + } else { + // Case: Long list + uint256 lengthLength = prefix - 0xf7; - // Single byte below 128 - if (prefix < SHORT_OFFSET) return (0, 1, ItemType.Data); + require(item.length > lengthLength, RLPInvalidDataRemainder(lengthLength, item.length)); + require(bytes1(_load(item.ptr, 0)) != 0x00); - // Short string (0-55 bytes) - if (prefix < LONG_LENGTH_OFFSET) return _decodeShortString(prefix - SHORT_OFFSET, item); + uint256 len = uint256(_load(item.ptr, 1)) >> (256 - 8 * lengthLength); + require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); + require(item.length > lengthLength + len, RLPContentLengthMismatch(lengthLength + len, item.length)); - // Long string (>55 bytes) - if (prefix < LONG_OFFSET) { - (offset, length) = _decodeLong(prefix - LONG_LENGTH_OFFSET, item); - return (offset, length, ItemType.Data); + return (lengthLength + 1, len, ItemType.List); } - - // Short list - if (prefix < SHORT_LIST_OFFSET) return _decodeShortList(prefix - LONG_OFFSET, item); - - // Long list - (offset, length) = _decodeLong(prefix - SHORT_LIST_OFFSET, item); - return (offset, length, ItemType.List); - } - - /// @dev Decodes a short string (0-55 bytes). The first byte contains the length, and the rest is the payload. - function _decodeShortString( - uint256 strLength, - Item memory item - ) private pure returns (uint256 offset, uint256 length, ItemType) { - require(item.length > strLength, RLPInvalidDataRemainder(strLength, item.length)); - require(strLength != 1 || _loadByte(_addOffset(item.ptr, 1), 0) >= bytes1(SHORT_OFFSET)); - return (1, strLength, ItemType.Data); - } - - /// @dev Decodes a short list (0-55 bytes). The first byte contains the length of the entire list. - function _decodeShortList( - uint256 listLength, - Item memory item - ) private pure returns (uint256 offset, uint256 length, ItemType) { - require(item.length > listLength, RLPInvalidDataRemainder(listLength, item.length)); - return (1, listLength, ItemType.List); - } - - /// @dev Decodes a long string or list (>55 bytes). The first byte indicates the length of the length, followed by the length itself. - function _decodeLong(uint256 lengthLength, Item memory item) private pure returns (uint256 offset, uint256 length) { - lengthLength += 1; // 1 byte for the length itself - require(item.length > lengthLength, RLPInvalidDataRemainder(lengthLength, item.length)); - require(_loadByte(item.ptr, 0) != 0x00); - - // Extract the length value from the next bytes - uint256 len = uint256(_load(_addOffset(item.ptr, 1)) >> (256 - 8 * lengthLength)); - require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); - uint256 expectedLength = lengthLength + len; - require(item.length <= expectedLength, RLPContentLengthMismatch(expectedLength, item.length)); - return (lengthLength + 1, len); } function _addOffset(Memory.Pointer ptr, uint256 offset) private pure returns (Memory.Pointer) { return bytes32(uint256(ptr.asBytes32()) + offset).asPointer(); } - function _copy(Memory.Pointer destPtr, Memory.Pointer srcPtr, uint256 length) private pure { - assembly ("memory-safe") { - mcopy(destPtr, srcPtr, length) - } - } - - function _loadByte(Memory.Pointer ptr, uint256 offset) private pure returns (bytes1 v) { - assembly ("memory-safe") { - v := byte(offset, mload(ptr)) - } - } - - function _load(Memory.Pointer ptr) private pure returns (bytes32 v) { - assembly ("memory-safe") { - v := mload(ptr) - } - } - - function _asPointer(bytes memory value) private pure returns (Memory.Pointer ptr) { + function _load(Memory.Pointer ptr, uint256 offset) private pure returns (bytes32 v) { assembly ("memory-safe") { - ptr := value + v := mload(add(ptr, offset)) } } } diff --git a/test/utils/RLP.t.sol b/test/utils/RLP.t.sol new file mode 100644 index 00000000000..227ac6092cb --- /dev/null +++ b/test/utils/RLP.t.sol @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +import {Test} from "forge-std/Test.sol"; + +import {RLP} from "@openzeppelin/contracts/utils/RLP.sol"; + +contract RLPTest is Test { + using RLP for *; + + function testEncodeDecodeBytes(bytes memory input) external pure { + assertEq(input.encode().decodeBytes(), input); + } + + function testEncodeDecodeList(bytes[] memory input) external pure { + // max length for list decoding by default + vm.assume(input.length <= 32); + + bytes[] memory encoded = new bytes[](input.length); + for (uint256 i = 0; i < input.length; ++i) { + encoded[i] = input[i].encode(); + } + + // encode list + decode as list of RLP items + RLP.Decoder[] memory list = encoded.encode().decodeList(); + + assertEq(list.length, input.length); + for (uint256 i = 0; i < input.length; ++i) { + assertEq(list[i].readBytes(), input[i]); + } + } + + // function testEncodeEmpty() external pure { + // assertEq(RLP.encoder().encode(), hex"c0"); + // } + + // function testEncodeUint256(uint256 input) external pure { + // bytes[] memory list = new bytes[](1); + // list[0] = RLP.encode(input); + + // assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + // } + + // function testEncodeAddress(address input) external pure { + // bytes[] memory list = new bytes[](1); + // list[0] = RLP.encode(input); + + // assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + // } + + // function testEncodeBytes(bytes memory input) external pure { + // bytes[] memory list = new bytes[](1); + // list[0] = RLP.encode(input); + + // assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + // } + + // /// forge-config: default.fuzz.runs = 512 + // function testEncodeBytesArray(bytes[] memory input) external pure { + // bytes[] memory list = new bytes[](input.length); + // for (uint256 i = 0; i < input.length; ++i) { + // list[i] = RLP.encode(input[i]); + // } + + // RLP.Encoder memory enc = RLP.encoder(); + // for (uint256 i = 0; i < input.length; ++i) { + // enc.push(input[i]); + // } + // assertEq(enc.encode(), RLP.encode(list)); + // } + + // function testEncodeMultiType(uint256 u, bytes memory b, address a) external pure { + // bytes[] memory list = new bytes[](3); + // list[0] = RLP.encode(u); + // list[1] = RLP.encode(b); + // list[2] = RLP.encode(a); + + // assertEq(RLP.encoder().push(u).push(b).push(a).encode(), RLP.encode(list)); + + // list[0] = RLP.encode(b); + // list[1] = RLP.encode(a); + // list[2] = RLP.encode(u); + + // assertEq(RLP.encoder().push(b).push(a).push(u).encode(), RLP.encode(list)); + // } +} From 106764e223eabcdbda60719c39d6b57de4736130 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Sat, 6 Sep 2025 23:00:38 +0200 Subject: [PATCH 071/114] Add Memory.slice and refactor RLP read/decode --- contracts/utils/Memory.sol | 60 +++++++++++ contracts/utils/RLP.sol | 210 +++++++++++++++++++------------------ test/utils/RLP.t.sol | 16 ++- 3 files changed, 182 insertions(+), 104 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index f09e1c8d626..24b85510e3a 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -41,4 +41,64 @@ library Memory { function asPointer(bytes32 value) internal pure returns (Pointer) { return Pointer.wrap(value); } + + type Slice is bytes32; + + function asSlice(bytes memory self) internal pure returns (Slice result) { + assembly ("memory-safe") { + result := or(shl(128, mload(self)), add(self, 0x20)) + } + } + + function asSlice(uint256 len, Memory.Pointer ptr) internal pure returns (Slice result) { + // TODO: Fail if len or ptr is larger than type(uint128).max ? + assembly ("memory-safe") { + result := or(shl(128, len), ptr) + } + } + + /// @dev Private helper: extract the data corresponding to a Slice (allocate new memory) + function toBytes(Slice self) internal pure returns (bytes memory result) { + uint256 len = length(self); + Memory.Pointer ptr = pointer(self); + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, len) + mcopy(add(result, 0x20), ptr, len) + mstore(0x40, add(add(result, len), 0x20)) + } + } + + /// @dev Private helper: length of a given Slice (equiv to self.length for calldata slices) + function length(Slice self) internal pure returns (uint256 result) { + assembly ("memory-safe") { + result := shr(128, self) + } + } + + /// @dev Private helper: memory location of a given Slice (equiv to self.offset for calldata slices) + function pointer(Slice self) internal pure returns (Memory.Pointer result) { + assembly ("memory-safe") { + result := and(self, shr(128, not(0))) + } + } + + /// @dev Private helper: read a bytes32 buffer from a given Slice at a specific offset + function load(Slice self, uint256 offset) internal pure returns (bytes32 value) { + assembly ("memory-safe") { + value := mload(add(and(self, shr(128, not(0))), offset)) + } + } + + /// @dev Private helper: slice a Slice (equivalent to self[start:] for calldata slices) + function slice(Slice self, uint256 start) internal pure returns (Slice) { + // TODO: check start < _len(self) ? + return asSlice(length(self) - start, asPointer(bytes32(uint256(asBytes32(pointer(self))) + start))); + } + + /// @dev Private helper: slice a Slice (equivalent to self[start:start+length] for calldata slices) + function slice(Slice self, uint256 start, uint256 len) internal pure returns (Slice) { + // TODO: check start + length < _len(self) ? + return asSlice(len, asPointer(bytes32(uint256(asBytes32(pointer(self))) + start))); + } } diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index e025bbc0dbf..36fb9822f52 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -5,6 +5,7 @@ import {Math} from "./math/Math.sol"; import {Bytes} from "./Bytes.sol"; import {Memory} from "./Memory.sol"; import {Panic} from "./Panic.sol"; +import {Packing} from "./Packing.sol"; /** * @dev Library for encoding and decoding data in RLP format. @@ -135,7 +136,6 @@ library RLP { /**************************************************************************************************************** * DECODING * ****************************************************************************************************************/ - /// @dev Items with length 0 are not RLP items. error RLPEmptyItem(); @@ -148,78 +148,72 @@ library RLP { /// @dev The content length does not match the expected length. error RLPContentLengthMismatch(uint256 expectedLength, uint256 actualLength); - // Memory slice (equivalent of a calldata slice in memory) - struct Decoder { - uint256 length; // Total length of the item in bytes - Memory.Pointer ptr; // Memory pointer to the start of the item - } - enum ItemType { Data, // Single data value List // List of RLP encoded items } - function decoder(bytes memory self) internal pure returns (Decoder memory result) { - require(self.length != 0, RLPEmptyItem()); // Empty arrays are not RLP items. - - assembly ("memory-safe") { - mstore(result, mload(self)) - mstore(add(result, 0x20), add(self, 0x20)) - } + /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. + // TODO: is there a usecase for that? + function readRawBytes(Memory.Slice item) internal pure returns (bytes memory) { + return item.toBytes(); } - function _toBytes(Decoder memory self) private pure returns (bytes memory result) { - return _toBytes(self, 0, self.length); - } + /// @dev Decode an RLP encoded bytes32. See {encode-bytes32} + function readBytes32(Memory.Slice item) internal pure returns (bytes32) { + uint256 length = item.length(); + require(length <= 33, RLPContentLengthMismatch(32, length)); - function _toBytes(Decoder memory self, uint256 offset, uint256 length) private pure returns (bytes memory result) { - // TODO: do we want to emit RLPContentLengthMismatch instead? - // Do we want to check equality? - if (self.length < offset + length) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); + (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); + require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); - Memory.Pointer ptr = self.ptr; - assembly ("memory-safe") { - result := mload(0x40) - mstore(result, length) - mcopy(add(result, 0x20), add(ptr, offset), length) - mstore(0x40, add(result, add(0x20, length))) - } + return item.load(itemOffset) >> (256 - 8 * itemLength); } - /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. - // TODO: is there a usecase for that? - function readRawBytes(Decoder memory item) internal pure returns (bytes memory) { - return _toBytes(item); + /// @dev Decode an RLP encoded uint256. See {encode-uint256} + function readUint256(Memory.Slice item) internal pure returns (uint256) { + return uint256(readBytes32(item)); } - /// @dev Decodes an RLP encoded item. - function readBytes(Decoder memory item) internal pure returns (bytes memory) { - (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); + /// @dev Decode an RLP encoded address. See {encode-address} + function readAddress(Memory.Slice item) internal pure returns (address) { + uint256 length = item.length(); + require(length == 1 || length == 21, RLPContentLengthMismatch(21, length)); + return address(uint160(readUint256(item))); + } + + /// @dev Decodes an RLP encoded bytes. See {encode-bytes} + function readBytes(Memory.Slice item) internal pure returns (bytes memory) { + (uint256 offset, uint256 length, ItemType itemType) = _decodeLength(item); require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); // Length is checked by {toBytes} - return _toBytes(item, itemOffset, itemLength); + return item.slice(offset, length).toBytes(); } - function readList(Decoder memory item) internal pure returns (Decoder[] memory) { + /// @dev Decodes an RLP encoded list into an array of RLP Items. This function supports list up to 32 elements + function readList(Memory.Slice item) internal pure returns (Memory.Slice[] memory) { return readList(item, 32); } - /// @dev Decodes an RLP encoded list into an array of RLP Items. See {_decodeLength} - function readList(Decoder memory item, uint256 maxListLength) internal pure returns (Decoder[] memory) { + /** + * @dev Variant of {readList-bytes32} that supports long lists up to `maxListLength`. Setting `maxListLength` to + * a high value will cause important, and costly, memory expansion. + */ + function readList(Memory.Slice item, uint256 maxListLength) internal pure returns (Memory.Slice[] memory) { + uint256 itemLength = item.length(); + (uint256 listOffset, uint256 listLength, ItemType itemType) = _decodeLength(item); require(itemType == ItemType.List, RLPUnexpectedType(ItemType.List, itemType)); - require(item.length == listOffset + listLength, RLPContentLengthMismatch(listOffset + listLength, item.length)); + require(itemLength == listOffset + listLength, RLPContentLengthMismatch(listOffset + listLength, itemLength)); - Decoder[] memory list = new Decoder[](maxListLength); + Memory.Slice[] memory list = new Memory.Slice[](maxListLength); uint256 itemCount; - for (uint256 currentOffset = listOffset; currentOffset < item.length; ++itemCount) { - (uint256 itemOffset, uint256 itemLength, ) = _decodeLength( - Decoder(item.length - currentOffset, _addOffset(item.ptr, currentOffset)) - ); - list[itemCount] = Decoder(itemLength + itemOffset, _addOffset(item.ptr, currentOffset)); - currentOffset += itemOffset + itemLength; + for (uint256 currentOffset = listOffset; currentOffset < itemLength; ++itemCount) { + (uint256 elementOffset, uint256 elementLength, ) = _decodeLength(item.slice(currentOffset)); + list[itemCount] = item.slice(currentOffset, elementLength + elementOffset); + currentOffset += elementOffset + elementLength; } // Decrease the array size to match the actual item count. @@ -229,74 +223,86 @@ library RLP { return list; } + function decodeAddress(bytes memory item) internal pure returns (address) { + return readAddress(item.asSlice()); + } + + function decodeUint256(bytes memory item) internal pure returns (uint256) { + return readUint256(item.asSlice()); + } + + function decodeBytes32(bytes memory item) internal pure returns (bytes32) { + return readBytes32(item.asSlice()); + } + /// @dev Same as {decodeBytes} but for `bytes`. See {decode}. function decodeBytes(bytes memory item) internal pure returns (bytes memory) { - return readBytes(decoder(item)); + return readBytes(item.asSlice()); } /// @dev Same as {decodeList} but for `bytes`. See {decode}. - function decodeList(bytes memory value) internal pure returns (Decoder[] memory) { - return readList(decoder(value)); + function decodeList(bytes memory value) internal pure returns (Memory.Slice[] memory) { + return readList(value.asSlice()); } /** * @dev Decodes an RLP `item`'s `length and type from its prefix. * Returns the offset, length, and type of the RLP item based on the encoding rules. */ - function _decodeLength(Decoder memory item) private pure returns (uint256 offset, uint256 length, ItemType) { - require(item.length != 0, RLPEmptyItem()); - uint8 prefix = uint8(bytes1(_load(item.ptr, 0))); - - if (prefix < SHORT_OFFSET) { - // Case: Single byte below 128 - return (0, 1, ItemType.Data); - } else if (prefix < LONG_LENGTH_OFFSET) { - // Case: Short string (0-55 bytes) - uint256 strLength = prefix - SHORT_OFFSET; - require(item.length > strLength, RLPInvalidDataRemainder(strLength, item.length)); - if (strLength == 1) { - require(bytes1(_load(item.ptr, 1)) >= bytes1(SHORT_OFFSET)); + function _decodeLength( + Memory.Slice item + ) private pure returns (uint256 _offset, uint256 _length, ItemType _itemtype) { + uint256 itemLength = item.length(); + + require(itemLength != 0, RLPEmptyItem()); + uint8 prefix = uint8(bytes1(item.load(0))); + + if (prefix < LONG_OFFSET) { + // CASE: item + if (prefix < SHORT_OFFSET) { + // Case: Single byte below 128 + return (0, 1, ItemType.Data); + } else if (prefix < LONG_LENGTH_OFFSET) { + // Case: Short string (0-55 bytes) + uint256 strLength = prefix - SHORT_OFFSET; + require(itemLength > strLength, RLPInvalidDataRemainder(strLength, itemLength)); + if (strLength == 1) { + require(bytes1(item.load(1)) >= bytes1(SHORT_OFFSET)); + } + return (1, strLength, ItemType.Data); + } else { + // Case: Long string (>55 bytes) + uint256 lengthLength = prefix - 0xb7; + + require(itemLength > lengthLength, RLPInvalidDataRemainder(lengthLength, itemLength)); + require(bytes1(item.load(0)) != 0x00); + + uint256 len = uint256(item.load(1)) >> (256 - 8 * lengthLength); + require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); + require(itemLength > lengthLength + len, RLPContentLengthMismatch(lengthLength + len, itemLength)); + + return (lengthLength + 1, len, ItemType.Data); } - return (1, strLength, ItemType.Data); - } else if (prefix < LONG_OFFSET) { - // Case: Long string (>55 bytes) - uint256 lengthLength = prefix - 0xb7; - - require(item.length > lengthLength, RLPInvalidDataRemainder(lengthLength, item.length)); - require(bytes1(_load(item.ptr, 0)) != 0x00); - - uint256 len = uint256(_load(item.ptr, 1)) >> (256 - 8 * lengthLength); - require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); - require(item.length > lengthLength + len, RLPContentLengthMismatch(lengthLength + len, item.length)); - - return (lengthLength + 1, len, ItemType.Data); - } else if (prefix < SHORT_LIST_OFFSET) { - // Case: Short list - uint256 listLength = prefix - LONG_OFFSET; - require(item.length > listLength, RLPInvalidDataRemainder(listLength, item.length)); - return (1, listLength, ItemType.List); } else { - // Case: Long list - uint256 lengthLength = prefix - 0xf7; - - require(item.length > lengthLength, RLPInvalidDataRemainder(lengthLength, item.length)); - require(bytes1(_load(item.ptr, 0)) != 0x00); - - uint256 len = uint256(_load(item.ptr, 1)) >> (256 - 8 * lengthLength); - require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); - require(item.length > lengthLength + len, RLPContentLengthMismatch(lengthLength + len, item.length)); - - return (lengthLength + 1, len, ItemType.List); - } - } - - function _addOffset(Memory.Pointer ptr, uint256 offset) private pure returns (Memory.Pointer) { - return bytes32(uint256(ptr.asBytes32()) + offset).asPointer(); - } - - function _load(Memory.Pointer ptr, uint256 offset) private pure returns (bytes32 v) { - assembly ("memory-safe") { - v := mload(add(ptr, offset)) + // Case: list + if (prefix < SHORT_LIST_OFFSET) { + // Case: Short list + uint256 listLength = prefix - LONG_OFFSET; + require(item.length() > listLength, RLPInvalidDataRemainder(listLength, itemLength)); + return (1, listLength, ItemType.List); + } else { + // Case: Long list + uint256 lengthLength = prefix - 0xf7; + + require(itemLength > lengthLength, RLPInvalidDataRemainder(lengthLength, itemLength)); + require(bytes1(item.load(0)) != 0x00); + + uint256 len = uint256(item.load(1)) >> (256 - 8 * lengthLength); + require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); + require(itemLength > lengthLength + len, RLPContentLengthMismatch(lengthLength + len, itemLength)); + + return (lengthLength + 1, len, ItemType.List); + } } } } diff --git a/test/utils/RLP.t.sol b/test/utils/RLP.t.sol index 227ac6092cb..441827579f0 100644 --- a/test/utils/RLP.t.sol +++ b/test/utils/RLP.t.sol @@ -3,12 +3,24 @@ pragma solidity ^0.8.20; import {Test} from "forge-std/Test.sol"; - import {RLP} from "@openzeppelin/contracts/utils/RLP.sol"; +import {Memory} from "@openzeppelin/contracts/utils/Memory.sol"; contract RLPTest is Test { using RLP for *; + function testEncodeDecodeAddress(address input) external pure { + assertEq(input.encode().decodeAddress(), input); + } + + function testEncodeDecodeUint256(uint256 input) external pure { + assertEq(input.encode().decodeUint256(), input); + } + + function testEncodeDecodeBytes32(bytes32 input) external pure { + assertEq(input.encode().decodeBytes32(), input); + } + function testEncodeDecodeBytes(bytes memory input) external pure { assertEq(input.encode().decodeBytes(), input); } @@ -23,7 +35,7 @@ contract RLPTest is Test { } // encode list + decode as list of RLP items - RLP.Decoder[] memory list = encoded.encode().decodeList(); + Memory.Slice[] memory list = encoded.encode().decodeList(); assertEq(list.length, input.length); for (uint256 i = 0; i < input.length; ++i) { From b55b1dba4e866030e17b315c23cf12637688b993 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Sat, 6 Sep 2025 23:13:55 +0200 Subject: [PATCH 072/114] testing --- contracts/utils/RLP.sol | 19 ++-- test/utils/RLP.test.js | 221 +++++++++++++++++++++------------------- 2 files changed, 122 insertions(+), 118 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 36fb9822f52..edec051586a 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -25,11 +25,6 @@ library RLP { /// @dev Prefix for list items (0xC0) uint8 internal constant LONG_OFFSET = 0xC0; - /// @dev Prefix for long string length (0xB8) - uint8 internal constant LONG_LENGTH_OFFSET = SHORT_OFFSET + SHORT_THRESHOLD + 1; // 184 - /// @dev Prefix for long list length (0xF8) - uint8 internal constant SHORT_LIST_OFFSET = LONG_OFFSET + SHORT_THRESHOLD + 1; // 248 - /**************************************************************************************************************** * ENCODING * ****************************************************************************************************************/ @@ -235,12 +230,10 @@ library RLP { return readBytes32(item.asSlice()); } - /// @dev Same as {decodeBytes} but for `bytes`. See {decode}. function decodeBytes(bytes memory item) internal pure returns (bytes memory) { return readBytes(item.asSlice()); } - /// @dev Same as {decodeList} but for `bytes`. See {decode}. function decodeList(bytes memory value) internal pure returns (Memory.Slice[] memory) { return readList(value.asSlice()); } @@ -262,20 +255,20 @@ library RLP { if (prefix < SHORT_OFFSET) { // Case: Single byte below 128 return (0, 1, ItemType.Data); - } else if (prefix < LONG_LENGTH_OFFSET) { + } else if (prefix <= SHORT_OFFSET + SHORT_THRESHOLD) { // Case: Short string (0-55 bytes) uint256 strLength = prefix - SHORT_OFFSET; require(itemLength > strLength, RLPInvalidDataRemainder(strLength, itemLength)); if (strLength == 1) { - require(bytes1(item.load(1)) >= bytes1(SHORT_OFFSET)); + require(bytes1(item.load(1)) >= bytes1(SHORT_OFFSET)); // TODO: custom error for sanity checks } return (1, strLength, ItemType.Data); } else { // Case: Long string (>55 bytes) - uint256 lengthLength = prefix - 0xb7; + uint256 lengthLength = prefix - SHORT_OFFSET - SHORT_THRESHOLD; require(itemLength > lengthLength, RLPInvalidDataRemainder(lengthLength, itemLength)); - require(bytes1(item.load(0)) != 0x00); + require(bytes1(item.load(0)) != 0x00); // TODO: custom error for sanity checks uint256 len = uint256(item.load(1)) >> (256 - 8 * lengthLength); require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); @@ -285,14 +278,14 @@ library RLP { } } else { // Case: list - if (prefix < SHORT_LIST_OFFSET) { + if (prefix <= LONG_OFFSET + SHORT_THRESHOLD) { // Case: Short list uint256 listLength = prefix - LONG_OFFSET; require(item.length() > listLength, RLPInvalidDataRemainder(listLength, itemLength)); return (1, listLength, ItemType.List); } else { // Case: Long list - uint256 lengthLength = prefix - 0xf7; + uint256 lengthLength = prefix - LONG_OFFSET - SHORT_THRESHOLD; require(itemLength > lengthLength, RLPInvalidDataRemainder(lengthLength, itemLength)); require(bytes1(item.load(0)) != 0x00); diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js index 4905c3764c2..ba0c9f55e9c 100644 --- a/test/utils/RLP.test.js +++ b/test/utils/RLP.test.js @@ -15,10 +15,6 @@ async function fixture() { mock.$encode_bytes = mock['$encode(bytes)']; mock.$encode_string = mock['$encode(string)']; mock.$encode_list = mock['$encode(bytes[])']; - mock.$decodeBytes_item = mock['$decodeBytes((uint256,bytes32))']; - mock.$decodeBytes_bytes = mock['$decodeBytes(bytes)']; - mock.$decodeList_item = mock['$decodeList((uint256,bytes32))']; - mock.$decodeList_bytes = mock['$decodeList(bytes)']; return { mock }; } @@ -28,106 +24,121 @@ describe('RLP', function () { Object.assign(this, await loadFixture(fixture)); }); - describe('encoding', function () { - it('encodes booleans', async function () { - await expect(this.mock.$encode_bool(false)).to.eventually.equal('0x80'); // 0 - await expect(this.mock.$encode_bool(true)).to.eventually.equal('0x01'); // 1 - }); - - it('encodes addresses', async function () { - const addr = generators.address(); - await expect(this.mock.$encode_address(addr)).to.eventually.equal(ethers.encodeRlp(addr)); - }); - - it('encodes uint256', async function () { - for (const input of [0, 1, 127, 128, 256, 1024, 0xffffff, ethers.MaxUint256]) { - await expect(this.mock.$encode_uint256(input)).to.eventually.equal(ethers.encodeRlp(ethers.toBeArray(input))); - } - }); - - it('encodes bytes32', async function () { - await expect( - this.mock.$encode_bytes32('0x0000000000000000000000000000000000000000000000000000000000000000'), - ).to.eventually.equal('0x80'); - await expect( - this.mock.$encode_bytes32('0x0000000000000000000000000000000000000000000000000000000000000001'), - ).to.eventually.equal('0x01'); - await expect( - this.mock.$encode_bytes32('0x1000000000000000000000000000000000000000000000000000000000000000'), - ).to.eventually.equal('0xa01000000000000000000000000000000000000000000000000000000000000000'); - }); - - it('encodes empty byte', async function () { - const input = '0x'; - await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); - }); - - it('encodes single byte < 128', async function () { - for (const input of ['0x00', '0x01', '0x7f']) { - await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); - } - }); - - it('encodes single byte >= 128', async function () { - for (const input of ['0x80', '0xff']) { - await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); - } - }); - - it('encodes short buffers (1-55 bytes)', async function () { - for (const input of [ - '0xab', // 1 byte - '0x1234', // 2 bytes - generators.bytes(55), // 55 bytes (maximum for short encoding) - ]) { - await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); - } - }); - - it('encodes long buffers (>55 bytes)', async function () { - for (const input of [ - generators.bytes(56), // 56 bytes (minimum for long encoding) - generators.bytes(128), // 128 bytes - ]) { - await expect(this.mock.$encode_bytes(input)).to.eventually.equal(ethers.encodeRlp(input)); - } - }); - - it('encodes strings', async function () { - for (const input of [ - '', // empty string - 'dog', - 'Lorem ipsum dolor sit amet, consectetur adipisicing elit', - ]) { - await expect(this.mock.$encode_string(input)).to.eventually.equal(ethers.encodeRlp(ethers.toUtf8Bytes(input))); - } - }); - - it('encode(bytes[])', async function () { - for (const input of [ - [], - ['0x'], - ['0x00'], - ['0x17', '0x42'], - ['0x17', '0x', '0x42', '0x0123456789abcdef', '0x'], - ]) { - await expect(this.mock.$encode_list(input.map(ethers.encodeRlp))).to.eventually.equal(ethers.encodeRlp(input)); - } - }); - - // const invalidTests = [ - // { name: 'short string with invalid length', input: '0x8100' }, - // { name: 'long string with invalid length prefix', input: '0xb800' }, - // { name: 'list with invalid length', input: '0xc100' }, - // { name: 'truncated long string', input: '0xb838' }, - // { name: 'invalid single byte encoding (non-minimal)', input: '0x8100' }, - // ]; - - // invalidTests.forEach(({ name, input }) => { - // it(`encodes ${name} into invalid RLP`, async function () { - // const item = await this.mock.$toItem(input); - // await expect(this.mock.$decodeBytes_bytes(item)).to.be.reverted; - // }); - // }); + it('encode booleans', async function () { + await expect(this.mock.$encode_bool(false)).to.eventually.equal('0x80'); // 0 + await expect(this.mock.$encode_bool(true)).to.eventually.equal('0x01'); // 1 }); + + it('encode/decode addresses', async function () { + const addr = generators.address(); + const expected = ethers.encodeRlp(addr); + + await expect(this.mock.$encode_address(addr)).to.eventually.equal(expected); + await expect(this.mock.$decodeAddress(expected)).to.eventually.equal(addr); + }); + + it('encode/decode uint256', async function () { + for (const input of [0, 1, 127, 128, 256, 1024, 0xffffff, ethers.MaxUint256]) { + const expected = ethers.encodeRlp(ethers.toBeArray(input)); + + await expect(this.mock.$encode_uint256(input)).to.eventually.equal(expected); + await expect(this.mock.$decodeUint256(expected)).to.eventually.equal(input); + } + }); + + it('encode/decode bytes32', async function () { + for (const { input, expected } of [ + { input: '0x0000000000000000000000000000000000000000000000000000000000000000', expected: '0x80' }, + { input: '0x0000000000000000000000000000000000000000000000000000000000000001', expected: '0x01' }, + { + input: '0x1000000000000000000000000000000000000000000000000000000000000000', + expected: '0xa01000000000000000000000000000000000000000000000000000000000000000', + }, + ]) { + await expect(this.mock.$encode_bytes32(input)).to.eventually.equal(expected); + await expect(this.mock.$decodeBytes32(expected)).to.eventually.equal(input); + } + }); + + it('encode/decode empty byte', async function () { + const input = '0x'; + const expected = ethers.encodeRlp(input); + + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(expected); + await expect(this.mock.$decodeBytes(expected)).to.eventually.equal(input); + }); + + it('encode/decode single byte < 128', async function () { + for (const input of ['0x00', '0x01', '0x7f']) { + const expected = ethers.encodeRlp(input); + + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(expected); + await expect(this.mock.$decodeBytes(expected)).to.eventually.equal(input); + } + }); + + it('encode/decode single byte >= 128', async function () { + for (const input of ['0x80', '0xff']) { + const expected = ethers.encodeRlp(input); + + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(expected); + await expect(this.mock.$decodeBytes(expected)).to.eventually.equal(input); + } + }); + + it('encode/decode short buffers (1-55 bytes)', async function () { + for (const input of [ + '0xab', // 1 byte + '0x1234', // 2 bytes + generators.bytes(55), // 55 bytes (maximum for short encoding) + ]) { + const expected = ethers.encodeRlp(input); + + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(expected); + await expect(this.mock.$decodeBytes(expected)).to.eventually.equal(input); + } + }); + + it('encode/decode long buffers (>55 bytes)', async function () { + for (const input of [ + generators.bytes(56), // 56 bytes (minimum for long encoding) + generators.bytes(128), // 128 bytes + ]) { + const expected = ethers.encodeRlp(input); + + await expect(this.mock.$encode_bytes(input)).to.eventually.equal(expected); + await expect(this.mock.$decodeBytes(expected)).to.eventually.equal(input); + } + }); + + it('encodes strings', async function () { + for (const input of [ + '', // empty string + 'dog', + 'Lorem ipsum dolor sit amet, consectetur adipisicing elit', + ]) { + await expect(this.mock.$encode_string(input)).to.eventually.equal(ethers.encodeRlp(ethers.toUtf8Bytes(input))); + } + }); + + it('encodes array (bytes[])', async function () { + for (const input of [[], ['0x'], ['0x00'], ['0x17', '0x42'], ['0x17', '0x', '0x42', '0x0123456789abcdef', '0x']]) { + await expect(this.mock.$encode_list(input.map(ethers.encodeRlp))).to.eventually.equal(ethers.encodeRlp(input)); + } + }); + + // const invalidTests = [ + // { name: 'short string with invalid length', input: '0x8100' }, + // { name: 'long string with invalid length prefix', input: '0xb800' }, + // { name: 'list with invalid length', input: '0xc100' }, + // { name: 'truncated long string', input: '0xb838' }, + // { name: 'invalid single byte encoding (non-minimal)', input: '0x8100' }, + // ]; + + // invalidTests.forEach(({ name, input }) => { + // it(`encodes ${name} into invalid RLP`, async function () { + // const item = await this.mock.$toItem(input); + // await expect(this.mock.$decodeBytes_bytes(item)).to.be.reverted; + // }); + // }); }); From e490d7965ba52b483ab0837662263b6c7891ff41 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Sat, 6 Sep 2025 23:19:18 +0200 Subject: [PATCH 073/114] inspired by --- contracts/utils/RLP.sol | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index edec051586a..f1e321cd52b 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -11,6 +11,10 @@ import {Packing} from "./Packing.sol"; * @dev Library for encoding and decoding data in RLP format. * Recursive Length Prefix (RLP) is the main encoding method used to serialize objects in Ethereum. * It's used for encoding everything from transactions to blocks to Patricia-Merkle tries. + * + * Inspired by + * * https://github.com/succinctlabs/optimism-bedrock-contracts/blob/main/rlp/RLPWriter.sol + * * https://github.com/succinctlabs/optimism-bedrock-contracts/blob/main/rlp/RLPReader.sol */ library RLP { using Memory for *; @@ -131,6 +135,7 @@ library RLP { /**************************************************************************************************************** * DECODING * ****************************************************************************************************************/ + /// @dev Items with length 0 are not RLP items. error RLPEmptyItem(); From bde82517f3ac7aa968c0097a30c7489d428c19aa Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Sat, 6 Sep 2025 23:22:18 +0200 Subject: [PATCH 074/114] remove unecessary imports --- contracts/utils/RLP.sol | 2 -- 1 file changed, 2 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index f1e321cd52b..5cd3e4bf01a 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -4,8 +4,6 @@ pragma solidity ^0.8.27; import {Math} from "./math/Math.sol"; import {Bytes} from "./Bytes.sol"; import {Memory} from "./Memory.sol"; -import {Panic} from "./Panic.sol"; -import {Packing} from "./Packing.sol"; /** * @dev Library for encoding and decoding data in RLP format. From 8b6ea8c281493eeeab89c7c6b00ae674d20ea687 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Sun, 7 Sep 2025 14:07:12 +0200 Subject: [PATCH 075/114] read bool and string --- contracts/utils/RLP.sol | 43 ++++++++++++++++++++++++++++------------- test/utils/RLP.test.js | 12 +++++++++--- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 5cd3e4bf01a..9b5c18738b7 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -157,27 +157,32 @@ library RLP { return item.toBytes(); } - /// @dev Decode an RLP encoded bytes32. See {encode-bytes32} - function readBytes32(Memory.Slice item) internal pure returns (bytes32) { + /// @dev Decode an RLP encoded bool. See {encode-bool} + function readBool(Memory.Slice item) internal pure returns (bool) { + return readUint256(item) != 0; + } + + /// @dev Decode an RLP encoded address. See {encode-address} + function readAddress(Memory.Slice item) internal pure returns (address) { + uint256 length = item.length(); + require(length == 1 || length == 21, RLPContentLengthMismatch(21, length)); + return address(uint160(readUint256(item))); + } + + /// @dev Decode an RLP encoded uint256. See {encode-uint256} + function readUint256(Memory.Slice item) internal pure returns (uint256) { uint256 length = item.length(); require(length <= 33, RLPContentLengthMismatch(32, length)); (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); - return item.load(itemOffset) >> (256 - 8 * itemLength); - } - - /// @dev Decode an RLP encoded uint256. See {encode-uint256} - function readUint256(Memory.Slice item) internal pure returns (uint256) { - return uint256(readBytes32(item)); + return uint256(item.load(itemOffset)) >> (256 - 8 * itemLength); } - /// @dev Decode an RLP encoded address. See {encode-address} - function readAddress(Memory.Slice item) internal pure returns (address) { - uint256 length = item.length(); - require(length == 1 || length == 21, RLPContentLengthMismatch(21, length)); - return address(uint160(readUint256(item))); + /// @dev Decode an RLP encoded bytes32. See {encode-bytes32} + function readBytes32(Memory.Slice item) internal pure returns (bytes32) { + return bytes32(readUint256(item)); } /// @dev Decodes an RLP encoded bytes. See {encode-bytes} @@ -189,6 +194,10 @@ library RLP { return item.slice(offset, length).toBytes(); } + function readString(Memory.Slice item) internal pure returns (string memory) { + return string(readBytes(item)); + } + /// @dev Decodes an RLP encoded list into an array of RLP Items. This function supports list up to 32 elements function readList(Memory.Slice item) internal pure returns (Memory.Slice[] memory) { return readList(item, 32); @@ -221,6 +230,10 @@ library RLP { return list; } + function decodeBool(bytes memory item) internal pure returns (bool) { + return readBool(item.asSlice()); + } + function decodeAddress(bytes memory item) internal pure returns (address) { return readAddress(item.asSlice()); } @@ -237,6 +250,10 @@ library RLP { return readBytes(item.asSlice()); } + function decodeString(bytes memory item) internal pure returns (string memory) { + return readString(item.asSlice()); + } + function decodeList(bytes memory value) internal pure returns (Memory.Slice[] memory) { return readList(value.asSlice()); } diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js index ba0c9f55e9c..856a93a48fa 100644 --- a/test/utils/RLP.test.js +++ b/test/utils/RLP.test.js @@ -24,9 +24,12 @@ describe('RLP', function () { Object.assign(this, await loadFixture(fixture)); }); - it('encode booleans', async function () { + it('encode/decode booleans', async function () { await expect(this.mock.$encode_bool(false)).to.eventually.equal('0x80'); // 0 await expect(this.mock.$encode_bool(true)).to.eventually.equal('0x01'); // 1 + + await expect(this.mock.$decodeBool('0x80')).to.eventually.equal(false); // 0 + await expect(this.mock.$decodeBool('0x01')).to.eventually.equal(true); // 1 }); it('encode/decode addresses', async function () { @@ -111,13 +114,16 @@ describe('RLP', function () { } }); - it('encodes strings', async function () { + it('encode/decode strings', async function () { for (const input of [ '', // empty string 'dog', 'Lorem ipsum dolor sit amet, consectetur adipisicing elit', ]) { - await expect(this.mock.$encode_string(input)).to.eventually.equal(ethers.encodeRlp(ethers.toUtf8Bytes(input))); + const expected = ethers.encodeRlp(ethers.toUtf8Bytes(input)); + + await expect(this.mock.$encode_string(input)).to.eventually.equal(expected); + await expect(this.mock.$decodeString(expected)).to.eventually.equal(input); } }); From dba642d4da178a34c8f8799fff3a6dfa814d7537 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Mon, 8 Sep 2025 10:56:59 +0200 Subject: [PATCH 076/114] Bytes.Accumulator & RLP.encoder --- contracts/utils/Bytes.sol | 61 ++++++++++++++++++++++++ contracts/utils/RLP.sol | 95 ++++++++++++++++++++++++++++++------- test/utils/Bytes.t.sol | 12 ++++- test/utils/RLP.t.sol | 98 +++++++++++++++++++++++---------------- 4 files changed, 207 insertions(+), 59 deletions(-) diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index a92d81f0aed..ef24d437437 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -4,6 +4,7 @@ pragma solidity ^0.8.24; import {Math} from "./math/Math.sol"; +import {Memory} from "./Memory.sol"; /** * @dev Bytes operations. @@ -244,4 +245,64 @@ library Bytes { value := mload(add(add(buffer, 0x20), offset)) } } + + struct Accumulator { + Memory.Pointer head; + Memory.Pointer tail; + } + + struct AccumulatorEntry { + Memory.Pointer next; + bytes data; + } + + function accumulator() internal pure returns (Accumulator memory self) { + self.head = Memory.asPointer(0x00); + self.tail = Memory.asPointer(0x00); + } + + function push(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { + Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: Memory.asPointer(0x00), data: data})); + + if (Memory.asBytes32(self.head) == 0x00) { + self.head = ptr; + self.tail = ptr; + } else { + _asAccumulatorEntry(self.tail).next = ptr; + self.tail = ptr; + } + + return self; + } + + function concat(Accumulator memory self) internal pure returns (bytes memory result) { + assembly ("memory-safe") { + result := mload(0x40) + let ptr := add(result, 0x20) + for { + let it := mload(self) + } iszero(iszero(it)) { + it := mload(it) + } { + let buffer := mload(add(it, 0x20)) + let length := mload(buffer) + mcopy(ptr, add(buffer, 0x20), length) + ptr := add(ptr, length) + } + mstore(result, sub(ptr, add(result, 0x20))) + mstore(0x40, ptr) + } + } + + function _asPtr(AccumulatorEntry memory item) private pure returns (Memory.Pointer ptr) { + assembly ("memory-safe") { + ptr := item + } + } + + function _asAccumulatorEntry(Memory.Pointer ptr) private pure returns (AccumulatorEntry memory item) { + assembly ("memory-safe") { + item := ptr + } + } } diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 9b5c18738b7..690b9f07859 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -15,6 +15,7 @@ import {Memory} from "./Memory.sol"; * * https://github.com/succinctlabs/optimism-bedrock-contracts/blob/main/rlp/RLPReader.sol */ library RLP { + using Bytes for *; using Memory for *; /** @@ -28,11 +29,72 @@ library RLP { uint8 internal constant LONG_OFFSET = 0xC0; /**************************************************************************************************************** - * ENCODING * + * ENCODING - ENCODER * + ****************************************************************************************************************/ + + struct Encoder { + Bytes.Accumulator acc; + } + + /// @dev Create an empty RLP Encoder. + function encoder() internal pure returns (Encoder memory enc) { + enc.acc = Bytes.accumulator(); + } + + /// @dev Add a boolean to a given RLP Encoder. + function push(Encoder memory self, bool input) internal pure returns (Encoder memory) { + self.acc.push(encode(input)); + return self; + } + + /// @dev Add an address to a given RLP Encoder. + function push(Encoder memory self, address input) internal pure returns (Encoder memory) { + self.acc.push(encode(input)); + return self; + } + + /// @dev Add a uint256 to a given RLP Encoder. + function push(Encoder memory self, uint256 input) internal pure returns (Encoder memory) { + self.acc.push(encode(input)); + return self; + } + + /// @dev Add a bytes32 to a given RLP Encoder. + function push(Encoder memory self, bytes32 input) internal pure returns (Encoder memory) { + self.acc.push(encode(input)); + return self; + } + + /// @dev Add a bytes buffer to a given RLP Encoder. + function push(Encoder memory self, bytes memory input) internal pure returns (Encoder memory) { + self.acc.push(encode(input)); + return self; + } + + /// @dev Add a string to a given RLP Encoder. + function push(Encoder memory self, string memory input) internal pure returns (Encoder memory) { + self.acc.push(encode(input)); + return self; + } + + /// @dev Add an array of bytes to a given RLP Encoder. + function push(Encoder memory self, bytes[] memory input) internal pure returns (Encoder memory) { + self.acc.push(encode(input)); + return self; + } + + /// @dev Add an (input) Encoder to a (target) Encoder. The input is RLP encoded as a list of bytes, and added to the target Encoder. + function push(Encoder memory self, Encoder memory input) internal pure returns (Encoder memory) { + self.acc.push(encode(input)); + return self; + } + + /**************************************************************************************************************** + * ENCODING - TO BYTES * ****************************************************************************************************************/ /** - * @dev Convenience method to encode a boolean as RLP. + * @dev Encode a boolean as RLP. * * Boolean `true` is encoded as 0x01, `false` as 0x80 (equivalent to encoding integers 1 and 0). * This follows the de facto ecosystem standard where booleans are treated as 0/1 integers. @@ -46,7 +108,7 @@ library RLP { } } - /// @dev Convenience method to encode an address as RLP bytes (i.e. encoded as packed 20 bytes). + /// @dev Encode an address as RLP. function encode(address input) internal pure returns (bytes memory result) { assembly ("memory-safe") { result := mload(0x40) @@ -56,7 +118,7 @@ library RLP { } } - /// @dev Convenience method to encode a uint256 as RLP. + /// @dev Encode a uint256 as RLP. function encode(uint256 input) internal pure returns (bytes memory result) { if (input < SHORT_OFFSET) { assembly ("memory-safe") { @@ -77,30 +139,29 @@ library RLP { } } - /// @dev Same as {encode-uint256-}, but for bytes32. + /// @dev Encode a bytes32 as RLP. Type alias for {encode-uint256-}. function encode(bytes32 input) internal pure returns (bytes memory) { return encode(uint256(input)); } - /** - * @dev Encodes a bytes array using RLP rules. - * Single bytes below 128 are encoded as themselves, otherwise as length prefix + data. - */ + /// @dev Encode a bytes buffer as RLP. function encode(bytes memory input) internal pure returns (bytes memory) { return (input.length == 1 && uint8(input[0]) < SHORT_OFFSET) ? input : _encode(input, SHORT_OFFSET); } - /// @dev Convenience method to encode a string as RLP. - function encode(string memory str) internal pure returns (bytes memory) { - return encode(bytes(str)); + /// @dev Encode a string as RLP. Type alias for {encode-bytes-}. + function encode(string memory input) internal pure returns (bytes memory) { + return encode(bytes(input)); } - /** - * @dev Encodes an array of bytes using RLP (as a list). - * First it {Bytes-concat}s the list of encoded items, then encodes it with the list prefix. - */ + /// @dev Encode an array of bytes as RLP. function encode(bytes[] memory input) internal pure returns (bytes memory) { - return _encode(Bytes.concat(input), LONG_OFFSET); + return _encode(input.concat(), LONG_OFFSET); + } + + /// @dev Encode an encoder (list of bytes) as RLP + function encode(Encoder memory self) internal pure returns (bytes memory result) { + return _encode(self.acc.concat(), LONG_OFFSET); } function _encode(bytes memory input, uint256 offset) private pure returns (bytes memory result) { diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index e01d933460d..013edd1c2f8 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -7,7 +7,7 @@ import {Math} from "@openzeppelin/contracts/utils/math/Math.sol"; import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; contract BytesTest is Test { - using Bytes for bytes; + using Bytes for *; function testSymbolicEqual(bytes memory a) public pure { assertTrue(Bytes.equal(a, a)); @@ -224,6 +224,16 @@ contract BytesTest is Test { } } + // Accumulator + function testAccumulator(bytes[] calldata input) public { + Bytes.Accumulator memory acc = Bytes.accumulator(); + + for (uint256 i = 0; i < input.length; ++i) { + acc.push(input[i]); + } + assertEq(acc.concat(), input.concat()); + } + // Helpers function _dirtyBytes16(bytes16 value) private pure returns (bytes16 dirty) { assembly ("memory-safe") { diff --git a/test/utils/RLP.t.sol b/test/utils/RLP.t.sol index 441827579f0..73784f258f1 100644 --- a/test/utils/RLP.t.sol +++ b/test/utils/RLP.t.sol @@ -9,6 +9,8 @@ import {Memory} from "@openzeppelin/contracts/utils/Memory.sol"; contract RLPTest is Test { using RLP for *; + // To bytes + function testEncodeDecodeAddress(address input) external pure { assertEq(input.encode().decodeAddress(), input); } @@ -25,6 +27,11 @@ contract RLPTest is Test { assertEq(input.encode().decodeBytes(), input); } + function testEncodeDecodeString(string memory input) external pure { + assertEq(input.encode().decodeString(), input); + } + + /// forge-config: default.fuzz.runs = 512 function testEncodeDecodeList(bytes[] memory input) external pure { // max length for list decoding by default vm.assume(input.length <= 32); @@ -43,57 +50,66 @@ contract RLPTest is Test { } } - // function testEncodeEmpty() external pure { - // assertEq(RLP.encoder().encode(), hex"c0"); - // } + // Encoder - // function testEncodeUint256(uint256 input) external pure { - // bytes[] memory list = new bytes[](1); - // list[0] = RLP.encode(input); + function testEncodeEmpty() external pure { + assertEq(RLP.encoder().encode(), hex"c0"); + } - // assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); - // } + function testEncodeAddress(address input) external pure { + bytes[] memory list = new bytes[](1); + list[0] = RLP.encode(input); - // function testEncodeAddress(address input) external pure { - // bytes[] memory list = new bytes[](1); - // list[0] = RLP.encode(input); + assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + } - // assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); - // } + function testEncodeUint256(uint256 input) external pure { + bytes[] memory list = new bytes[](1); + list[0] = RLP.encode(input); - // function testEncodeBytes(bytes memory input) external pure { - // bytes[] memory list = new bytes[](1); - // list[0] = RLP.encode(input); + assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + } - // assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); - // } + function testEncodeBytes(bytes memory input) external pure { + bytes[] memory list = new bytes[](1); + list[0] = RLP.encode(input); - // /// forge-config: default.fuzz.runs = 512 - // function testEncodeBytesArray(bytes[] memory input) external pure { - // bytes[] memory list = new bytes[](input.length); - // for (uint256 i = 0; i < input.length; ++i) { - // list[i] = RLP.encode(input[i]); - // } + assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + } - // RLP.Encoder memory enc = RLP.encoder(); - // for (uint256 i = 0; i < input.length; ++i) { - // enc.push(input[i]); - // } - // assertEq(enc.encode(), RLP.encode(list)); - // } + function testEncodeString(string memory input) external pure { + bytes[] memory list = new bytes[](1); + list[0] = RLP.encode(input); - // function testEncodeMultiType(uint256 u, bytes memory b, address a) external pure { - // bytes[] memory list = new bytes[](3); - // list[0] = RLP.encode(u); - // list[1] = RLP.encode(b); - // list[2] = RLP.encode(a); + assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + } - // assertEq(RLP.encoder().push(u).push(b).push(a).encode(), RLP.encode(list)); + /// forge-config: default.fuzz.runs = 512 + function testEncodeBytesArray(bytes[] memory input) external pure { + bytes[] memory list = new bytes[](input.length); + for (uint256 i = 0; i < input.length; ++i) { + list[i] = RLP.encode(input[i]); + } + + RLP.Encoder memory enc = RLP.encoder(); + for (uint256 i = 0; i < input.length; ++i) { + enc.push(input[i]); + } + assertEq(enc.encode(), RLP.encode(list)); + } + + function testEncodeMultiType(uint256 u, bytes memory b, address a) external pure { + bytes[] memory list = new bytes[](3); + list[0] = RLP.encode(u); + list[1] = RLP.encode(b); + list[2] = RLP.encode(a); - // list[0] = RLP.encode(b); - // list[1] = RLP.encode(a); - // list[2] = RLP.encode(u); + assertEq(RLP.encoder().push(u).push(b).push(a).encode(), RLP.encode(list)); - // assertEq(RLP.encoder().push(b).push(a).push(u).encode(), RLP.encode(list)); - // } + list[0] = RLP.encode(b); + list[1] = RLP.encode(a); + list[2] = RLP.encode(u); + + assertEq(RLP.encoder().push(b).push(a).push(u).encode(), RLP.encode(list)); + } } From 5e3ce7c7adbfe7f653cbc3ff507e65b1bc18e41c Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Mon, 8 Sep 2025 11:02:40 +0200 Subject: [PATCH 077/114] reorder --- contracts/utils/RLP.sol | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 690b9f07859..0ae772ba63a 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -18,6 +18,23 @@ library RLP { using Bytes for *; using Memory for *; + /// @dev Items with length 0 are not RLP items. + error RLPEmptyItem(); + + /// @dev The `item` is not of the `expected` type. + error RLPUnexpectedType(ItemType expected, ItemType actual); + + /// @dev The item is not long enough to contain the data. + error RLPInvalidDataRemainder(uint256 minLength, uint256 actualLength); + + /// @dev The content length does not match the expected length. + error RLPContentLengthMismatch(uint256 expectedLength, uint256 actualLength); + + enum ItemType { + Data, // Single data value + List // List of RLP encoded items + } + /** * @dev Maximum length for data that will be encoded using the short format. * If `data.length <= 55 bytes`, it will be encoded as: `[0x80 + length]` + data. @@ -192,26 +209,9 @@ library RLP { } /**************************************************************************************************************** - * DECODING * + * DECODING - READ FROM AN RLP ENCODED MEMORY SLICE * ****************************************************************************************************************/ - /// @dev Items with length 0 are not RLP items. - error RLPEmptyItem(); - - /// @dev The `item` is not of the `expected` type. - error RLPUnexpectedType(ItemType expected, ItemType actual); - - /// @dev The item is not long enough to contain the data. - error RLPInvalidDataRemainder(uint256 minLength, uint256 actualLength); - - /// @dev The content length does not match the expected length. - error RLPContentLengthMismatch(uint256 expectedLength, uint256 actualLength); - - enum ItemType { - Data, // Single data value - List // List of RLP encoded items - } - /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. // TODO: is there a usecase for that? function readRawBytes(Memory.Slice item) internal pure returns (bytes memory) { @@ -291,6 +291,10 @@ library RLP { return list; } + /**************************************************************************************************************** + * DECODING - FROM BYTES * + ****************************************************************************************************************/ + function decodeBool(bytes memory item) internal pure returns (bool) { return readBool(item.asSlice()); } From 902b222b0d3ef106808c0cc4d2c0aeaf2be51ff2 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 16:04:46 +0200 Subject: [PATCH 078/114] document Bytes.accumulator --- contracts/utils/Bytes.sol | 25 ++++++++++++++++++++++++- contracts/utils/RLP.sol | 2 +- test/utils/Bytes.t.sol | 24 +++++++++++++++++++----- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index ef24d437437..65c8aede22c 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -246,21 +246,29 @@ library Bytes { } } + /** + * @dev Bytes accumulator: a linked list of `bytes`. + * + * Note: This is a memory structure that SHOULD not be put in storage. + */ struct Accumulator { Memory.Pointer head; Memory.Pointer tail; } + /// @dev Item (list node) in a bytes accumulator struct AccumulatorEntry { Memory.Pointer next; bytes data; } + /// @dev Create a new (empty) accumulator function accumulator() internal pure returns (Accumulator memory self) { self.head = Memory.asPointer(0x00); self.tail = Memory.asPointer(0x00); } + /// @dev Add a bytes buffer to (the end of) an Accumulator function push(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: Memory.asPointer(0x00), data: data})); @@ -275,7 +283,22 @@ library Bytes { return self; } - function concat(Accumulator memory self) internal pure returns (bytes memory result) { + /// @dev Add a bytes buffer to (the beginning of) an Accumulator + function shift(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { + Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: self.head, data: data})); + + if (Memory.asBytes32(self.head) == 0x00) { + self.head = ptr; + self.tail = ptr; + } else { + self.head = ptr; + } + + return self; + } + + /// @dev Flatten all the bytes entries in an Accumulator into a single buffer + function flatten(Accumulator memory self) internal pure returns (bytes memory result) { assembly ("memory-safe") { result := mload(0x40) let ptr := add(result, 0x20) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 0ae772ba63a..40829c943b2 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -178,7 +178,7 @@ library RLP { /// @dev Encode an encoder (list of bytes) as RLP function encode(Encoder memory self) internal pure returns (bytes memory result) { - return _encode(self.acc.concat(), LONG_OFFSET); + return _encode(self.acc.flatten(), LONG_OFFSET); } function _encode(bytes memory input, uint256 offset) private pure returns (bytes memory result) { diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index 013edd1c2f8..297c2ab5593 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -225,13 +225,27 @@ contract BytesTest is Test { } // Accumulator - function testAccumulator(bytes[] calldata input) public { + function testAccumulatorPushShift() public pure { + Bytes.Accumulator memory acc = Bytes.accumulator(); // + acc.push(hex"11"); // 11 + acc.push(hex"22"); // 1122 + acc.shift(hex"33"); // 331122 + acc.shift(hex"44"); // 44331122 + acc.push(hex"55"); // 4433112255 + acc.shift(hex"66"); // 664433112255 + assertEq(acc.flatten(), hex"664433112255"); + } + + function testAccumulatorPush(bytes[] calldata input) public pure { Bytes.Accumulator memory acc = Bytes.accumulator(); + for (uint256 i = 0; i < input.length; ++i) acc.push(input[i]); + assertEq(acc.flatten(), input.concat()); + } - for (uint256 i = 0; i < input.length; ++i) { - acc.push(input[i]); - } - assertEq(acc.concat(), input.concat()); + function testAccumulatorShift(bytes[] calldata input) public pure { + Bytes.Accumulator memory acc = Bytes.accumulator(); + for (uint256 i = input.length; i > 0; --i) acc.shift(input[i - 1]); + assertEq(acc.flatten(), input.concat()); } // Helpers From b301b0615f54574cb969f095cceb2a606932b935 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 18:03:17 +0200 Subject: [PATCH 079/114] pragma fix --- contracts/token/ERC20/extensions/ERC4626.sol | 2 +- contracts/utils/Memory.sol | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts/token/ERC20/extensions/ERC4626.sol b/contracts/token/ERC20/extensions/ERC4626.sol index adc906b1ef4..3b0eeb06b20 100644 --- a/contracts/token/ERC20/extensions/ERC4626.sol +++ b/contracts/token/ERC20/extensions/ERC4626.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // OpenZeppelin Contracts (last updated v5.4.0) (token/ERC20/extensions/ERC4626.sol) -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {IERC20, IERC20Metadata, ERC20} from "../ERC20.sol"; import {SafeERC20} from "../utils/SafeERC20.sol"; diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 24b85510e3a..c55dbe5a734 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; /** * @dev Utilities to manipulate memory. From 1b112b25b3495847756005017391c50db3df30b1 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 18:05:36 +0200 Subject: [PATCH 080/114] mock stateless --- contracts/mocks/Stateless.sol | 1 + 1 file changed, 1 insertion(+) diff --git a/contracts/mocks/Stateless.sol b/contracts/mocks/Stateless.sol index 232fbe83d4e..675bc0ce164 100644 --- a/contracts/mocks/Stateless.sol +++ b/contracts/mocks/Stateless.sol @@ -44,6 +44,7 @@ import {P256} from "../utils/cryptography/P256.sol"; import {Packing} from "../utils/Packing.sol"; import {Panic} from "../utils/Panic.sol"; import {RelayedCall} from "../utils/RelayedCall.sol"; +import {RLP} from "../utils/RLP.sol"; import {RSA} from "../utils/cryptography/RSA.sol"; import {SafeCast} from "../utils/math/SafeCast.sol"; import {SafeERC20} from "../token/ERC20/utils/SafeERC20.sol"; From 9c0e92401c9cffe8dbe42d56a7abbc865d5b57cd Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 18:28:19 +0200 Subject: [PATCH 081/114] slither --- contracts/utils/RLP.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 40829c943b2..01613aa11fa 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -120,7 +120,7 @@ library RLP { assembly ("memory-safe") { result := mload(0x40) mstore(result, 0x01) // length of the encoded data: 1 byte - mstore8(add(result, 0x20), shl(mul(7, iszero(input)), 1)) // input + mstore8(add(result, 0x20), add(mul(iszero(input), 0x7f), 1)) // input mstore(0x40, add(result, 0x21)) // reserve memory } } From c48e6238839eb2f45ac952ce2a2db549ce9a9928 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 18:48:26 +0200 Subject: [PATCH 082/114] Memory slice doc --- contracts/utils/Memory.sol | 61 ++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index c55dbe5a734..9e3722e6ba9 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -2,6 +2,8 @@ pragma solidity ^0.8.24; +import {Panic} from "./Panic.sol"; + /** * @dev Utilities to manipulate memory. * @@ -44,61 +46,68 @@ library Memory { type Slice is bytes32; + /// @dev Get a slice representation of a bytes object in memory function asSlice(bytes memory self) internal pure returns (Slice result) { assembly ("memory-safe") { result := or(shl(128, mload(self)), add(self, 0x20)) } } - function asSlice(uint256 len, Memory.Pointer ptr) internal pure returns (Slice result) { + /// @dev Private helper: create a slice from raw values (length and pointer) + function _asSlice(uint256 len, Memory.Pointer ptr) private pure returns (Slice result) { // TODO: Fail if len or ptr is larger than type(uint128).max ? assembly ("memory-safe") { result := or(shl(128, len), ptr) } } - /// @dev Private helper: extract the data corresponding to a Slice (allocate new memory) - function toBytes(Slice self) internal pure returns (bytes memory result) { - uint256 len = length(self); - Memory.Pointer ptr = pointer(self); + /// @dev Returns the memory location of a given slice (equiv to self.offset for calldata slices) + function _pointer(Slice self) private pure returns (Memory.Pointer result) { assembly ("memory-safe") { - result := mload(0x40) - mstore(result, len) - mcopy(add(result, 0x20), ptr, len) - mstore(0x40, add(add(result, len), 0x20)) + result := and(self, shr(128, not(0))) } } - /// @dev Private helper: length of a given Slice (equiv to self.length for calldata slices) + /// @dev Returns the length of a given slice (equiv to self.length for calldata slices) function length(Slice self) internal pure returns (uint256 result) { assembly ("memory-safe") { result := shr(128, self) } } - /// @dev Private helper: memory location of a given Slice (equiv to self.offset for calldata slices) - function pointer(Slice self) internal pure returns (Memory.Pointer result) { - assembly ("memory-safe") { - result := and(self, shr(128, not(0))) - } + /// @dev Offset a memory slice (equivalent to self[start:] for calldata slices) + function slice(Slice self, uint256 offset) internal pure returns (Slice) { + if (offset > length(self)) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); + return _asSlice(length(self) - offset, asPointer(bytes32(uint256(asBytes32(_pointer(self))) + offset))); + } + + /// @dev Offset and cut a Slice (equivalent to self[start:start+length] for calldata slices) + function slice(Slice self, uint256 offset, uint256 len) internal pure returns (Slice) { + if (offset + len > length(self)) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); + return _asSlice(len, asPointer(bytes32(uint256(asBytes32(_pointer(self))) + offset))); } - /// @dev Private helper: read a bytes32 buffer from a given Slice at a specific offset + /** + * @dev Read a bytes32 buffer from a given Slice at a specific offset + * + * Note:If offset > length(slice) - 32, part of the return value will be out of bound and should be ignored. + */ function load(Slice self, uint256 offset) internal pure returns (bytes32 value) { + if (offset >= length(self)) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); assembly ("memory-safe") { value := mload(add(and(self, shr(128, not(0))), offset)) } } - /// @dev Private helper: slice a Slice (equivalent to self[start:] for calldata slices) - function slice(Slice self, uint256 start) internal pure returns (Slice) { - // TODO: check start < _len(self) ? - return asSlice(length(self) - start, asPointer(bytes32(uint256(asBytes32(pointer(self))) + start))); - } - - /// @dev Private helper: slice a Slice (equivalent to self[start:start+length] for calldata slices) - function slice(Slice self, uint256 start, uint256 len) internal pure returns (Slice) { - // TODO: check start + length < _len(self) ? - return asSlice(len, asPointer(bytes32(uint256(asBytes32(pointer(self))) + start))); + /// @dev Extract the data corresponding to a Slice (allocate new memory) + function toBytes(Slice self) internal pure returns (bytes memory result) { + uint256 len = length(self); + Memory.Pointer ptr = _pointer(self); + assembly ("memory-safe") { + result := mload(0x40) + mstore(result, len) + mcopy(add(result, 0x20), ptr, len) + mstore(0x40, add(add(result, len), 0x20)) + } } } From b10ae8c64c83c36a21db446d3c2f9a39aaa2eab3 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 18:52:34 +0200 Subject: [PATCH 083/114] Update enums.js --- test/helpers/enums.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/helpers/enums.js b/test/helpers/enums.js index 804262d4e7a..6adbf64ad82 100644 --- a/test/helpers/enums.js +++ b/test/helpers/enums.js @@ -11,5 +11,4 @@ module.exports = { Rounding: EnumTyped('Floor', 'Ceil', 'Trunc', 'Expand'), OperationState: Enum('Unset', 'Waiting', 'Ready', 'Done'), RevertType: EnumTyped('None', 'RevertWithoutMessage', 'RevertWithMessage', 'RevertWithCustomError', 'Panic'), - ItemType: Enum('Data', 'List'), }; From d41131868a66d4979534b26502916574ef00dbd0 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 21:46:25 +0200 Subject: [PATCH 084/114] avoid load error --- contracts/utils/RLP.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 01613aa11fa..4b0d42a0dfc 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -238,7 +238,7 @@ library RLP { (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); - return uint256(item.load(itemOffset)) >> (256 - 8 * itemLength); + return itemLength == 0 ? 0 : uint256(item.load(itemOffset)) >> (256 - 8 * itemLength); } /// @dev Decode an RLP encoded bytes32. See {encode-bytes32} From df528afe2c812c774f629ac3cbe7925f43bc5e26 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 21:52:17 +0200 Subject: [PATCH 085/114] fuzz test Memory slices --- test/utils/Memory.t.sol | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/utils/Memory.t.sol b/test/utils/Memory.t.sol index 8ed9b4c43bc..86fb901b38c 100644 --- a/test/utils/Memory.t.sol +++ b/test/utils/Memory.t.sol @@ -3,9 +3,11 @@ pragma solidity ^0.8.20; import {Test} from "forge-std/Test.sol"; +import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; import {Memory} from "@openzeppelin/contracts/utils/Memory.sol"; contract MemoryTest is Test { + using Bytes for *; using Memory for *; // - first 0x80 bytes are reserved (scratch + FMP + zero) @@ -18,4 +20,19 @@ contract MemoryTest is Test { ptr.asPointer().setFreeMemoryPointer(); assertEq(Memory.getFreeMemoryPointer().asBytes32(), ptr); } + + function testAsSliceToBytes(bytes memory input) public pure { + assertEq(input.asSlice().toBytes(), input); + } + + function testSlice(bytes memory input, uint256 offset) public pure { + offset = bound(offset, 0, input.length); + assertEq(input.asSlice().slice(offset).toBytes(), input.slice(offset)); + } + + function testSlice(bytes memory input, uint256 offset, uint256 length) public pure { + offset = bound(offset, 0, input.length); + length = bound(length, 0, input.length - offset); + assertEq(input.asSlice().slice(offset, length).toBytes(), input.slice(offset, offset + length)); + } } From da61a5e4e5152bb8b66143a702a80717589d1e51 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 10 Sep 2025 23:30:22 +0200 Subject: [PATCH 086/114] coverage --- test/utils/RLP.t.sol | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/test/utils/RLP.t.sol b/test/utils/RLP.t.sol index 73784f258f1..7bc40d6024d 100644 --- a/test/utils/RLP.t.sol +++ b/test/utils/RLP.t.sol @@ -9,7 +9,11 @@ import {Memory} from "@openzeppelin/contracts/utils/Memory.sol"; contract RLPTest is Test { using RLP for *; - // To bytes + // Encode -> Decode + + function testEncodeDecodeBool(bool input) external pure { + assertEq(input.encode().decodeBool(), input); + } function testEncodeDecodeAddress(address input) external pure { assertEq(input.encode().decodeAddress(), input); @@ -50,12 +54,19 @@ contract RLPTest is Test { } } - // Encoder + // List encoder function testEncodeEmpty() external pure { assertEq(RLP.encoder().encode(), hex"c0"); } + function testEncodeBool(bool input) external pure { + bytes[] memory list = new bytes[](1); + list[0] = RLP.encode(input); + + assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + } + function testEncodeAddress(address input) external pure { bytes[] memory list = new bytes[](1); list[0] = RLP.encode(input); @@ -70,6 +81,13 @@ contract RLPTest is Test { assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); } + function testEncodeBytes32(bytes32 input) external pure { + bytes[] memory list = new bytes[](1); + list[0] = RLP.encode(input); + + assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + } + function testEncodeBytes(bytes memory input) external pure { bytes[] memory list = new bytes[](1); list[0] = RLP.encode(input); @@ -86,16 +104,18 @@ contract RLPTest is Test { /// forge-config: default.fuzz.runs = 512 function testEncodeBytesArray(bytes[] memory input) external pure { - bytes[] memory list = new bytes[](input.length); - for (uint256 i = 0; i < input.length; ++i) { - list[i] = RLP.encode(input[i]); - } + bytes[] memory list = new bytes[](1); + list[0] = RLP.encode(input); - RLP.Encoder memory enc = RLP.encoder(); - for (uint256 i = 0; i < input.length; ++i) { - enc.push(input[i]); - } - assertEq(enc.encode(), RLP.encode(list)); + assertEq(RLP.encoder().push(input).encode(), RLP.encode(list)); + } + + function testEncodeEncoder(bytes memory input) external pure { + bytes[] memory list = new bytes[](1); + list[0] = RLP.encode(input); + list[0] = RLP.encode(list); + + assertEq(RLP.encoder().push(RLP.encoder().push(input)).encode(), RLP.encode(list)); } function testEncodeMultiType(uint256 u, bytes memory b, address a) external pure { From 283ce7c2dffef3c87cefb222ed7b2165a2f9bda6 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Mon, 15 Sep 2025 15:39:41 +0200 Subject: [PATCH 087/114] remove unused code --- contracts/utils/RLP.sol | 6 ------ 1 file changed, 6 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 4b0d42a0dfc..4a82d46a3ac 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -212,12 +212,6 @@ library RLP { * DECODING - READ FROM AN RLP ENCODED MEMORY SLICE * ****************************************************************************************************************/ - /// @dev Reads the raw bytes of an RLP item without decoding the content. Includes prefix bytes. - // TODO: is there a usecase for that? - function readRawBytes(Memory.Slice item) internal pure returns (bytes memory) { - return item.toBytes(); - } - /// @dev Decode an RLP encoded bool. See {encode-bool} function readBool(Memory.Slice item) internal pure returns (bool) { return readUint256(item) != 0; From 33b57b5777f27f542faa4273b4c39124f61a8120 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Mon, 15 Sep 2025 21:23:52 +0200 Subject: [PATCH 088/114] Move Bytes.Accumulator to a dedicated file in utils/structs --- .changeset/modern-moments-raise.md | 5 ++ .changeset/wise-webs-fly.md | 5 ++ contracts/mocks/Stateless.sol | 1 + contracts/utils/Bytes.sol | 84 --------------------- contracts/utils/README.adoc | 3 + contracts/utils/RLP.sol | 6 +- contracts/utils/structs/Accumulators.sol | 94 ++++++++++++++++++++++++ test/utils/Accumulators.t.sol | 35 +++++++++ test/utils/Bytes.t.sol | 24 ------ 9 files changed, 147 insertions(+), 110 deletions(-) create mode 100644 .changeset/modern-moments-raise.md create mode 100644 .changeset/wise-webs-fly.md create mode 100644 contracts/utils/structs/Accumulators.sol create mode 100644 test/utils/Accumulators.t.sol diff --git a/.changeset/modern-moments-raise.md b/.changeset/modern-moments-raise.md new file mode 100644 index 00000000000..38526551e0f --- /dev/null +++ b/.changeset/modern-moments-raise.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`Memory`: Add a UDVT for handling slices or memory space similarly to calldata slices. diff --git a/.changeset/wise-webs-fly.md b/.changeset/wise-webs-fly.md new file mode 100644 index 00000000000..22955c997b8 --- /dev/null +++ b/.changeset/wise-webs-fly.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +Accumulators: A library for merging an arbitrary dynamic number of bytes buffers. diff --git a/contracts/mocks/Stateless.sol b/contracts/mocks/Stateless.sol index 675bc0ce164..0b74c9a0ee3 100644 --- a/contracts/mocks/Stateless.sol +++ b/contracts/mocks/Stateless.sol @@ -4,6 +4,7 @@ pragma solidity ^0.8.26; // We keep these imports and a dummy contract just to we can run the test suite after transpilation. +import {Accumulators} from "../utils/structs/Accumulators.sol"; import {Address} from "../utils/Address.sol"; import {Arrays} from "../utils/Arrays.sol"; import {AuthorityUtils} from "../access/manager/AuthorityUtils.sol"; diff --git a/contracts/utils/Bytes.sol b/contracts/utils/Bytes.sol index 65c8aede22c..a92d81f0aed 100644 --- a/contracts/utils/Bytes.sol +++ b/contracts/utils/Bytes.sol @@ -4,7 +4,6 @@ pragma solidity ^0.8.24; import {Math} from "./math/Math.sol"; -import {Memory} from "./Memory.sol"; /** * @dev Bytes operations. @@ -245,87 +244,4 @@ library Bytes { value := mload(add(add(buffer, 0x20), offset)) } } - - /** - * @dev Bytes accumulator: a linked list of `bytes`. - * - * Note: This is a memory structure that SHOULD not be put in storage. - */ - struct Accumulator { - Memory.Pointer head; - Memory.Pointer tail; - } - - /// @dev Item (list node) in a bytes accumulator - struct AccumulatorEntry { - Memory.Pointer next; - bytes data; - } - - /// @dev Create a new (empty) accumulator - function accumulator() internal pure returns (Accumulator memory self) { - self.head = Memory.asPointer(0x00); - self.tail = Memory.asPointer(0x00); - } - - /// @dev Add a bytes buffer to (the end of) an Accumulator - function push(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { - Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: Memory.asPointer(0x00), data: data})); - - if (Memory.asBytes32(self.head) == 0x00) { - self.head = ptr; - self.tail = ptr; - } else { - _asAccumulatorEntry(self.tail).next = ptr; - self.tail = ptr; - } - - return self; - } - - /// @dev Add a bytes buffer to (the beginning of) an Accumulator - function shift(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { - Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: self.head, data: data})); - - if (Memory.asBytes32(self.head) == 0x00) { - self.head = ptr; - self.tail = ptr; - } else { - self.head = ptr; - } - - return self; - } - - /// @dev Flatten all the bytes entries in an Accumulator into a single buffer - function flatten(Accumulator memory self) internal pure returns (bytes memory result) { - assembly ("memory-safe") { - result := mload(0x40) - let ptr := add(result, 0x20) - for { - let it := mload(self) - } iszero(iszero(it)) { - it := mload(it) - } { - let buffer := mload(add(it, 0x20)) - let length := mload(buffer) - mcopy(ptr, add(buffer, 0x20), length) - ptr := add(ptr, length) - } - mstore(result, sub(ptr, add(result, 0x20))) - mstore(0x40, ptr) - } - } - - function _asPtr(AccumulatorEntry memory item) private pure returns (Memory.Pointer ptr) { - assembly ("memory-safe") { - ptr := item - } - } - - function _asAccumulatorEntry(Memory.Pointer ptr) private pure returns (AccumulatorEntry memory item) { - assembly ("memory-safe") { - item := ptr - } - } } diff --git a/contracts/utils/README.adoc b/contracts/utils/README.adoc index e5fb74b363b..3e1a4adad21 100644 --- a/contracts/utils/README.adoc +++ b/contracts/utils/README.adoc @@ -13,6 +13,7 @@ Miscellaneous contracts and libraries containing utility functions you can use t * {ReentrancyGuard}: A modifier that can prevent reentrancy during certain functions. * {ReentrancyGuardTransient}: Variant of {ReentrancyGuard} that uses transient storage (https://eips.ethereum.org/EIPS/eip-1153[EIP-1153]). * {ERC165}, {ERC165Checker}: Utilities for inspecting interfaces supported by contracts. + * {Accumulators}: A library for merging an arbitrary dynamic number of bytes buffers. * {BitMaps}: A simple library to manage boolean value mapped to a numerical index in an efficient way. * {Checkpoints}: A data structure to store values mapped to a strictly increasing key. Can be used for storing and accessing values over time. * {CircularBuffer}: A data structure to store the last N values pushed to it. @@ -85,6 +86,8 @@ Ethereum contracts have no native concept of an interface, so applications must == Data Structures +{{Accumulators}} + {{BitMaps}} {{Checkpoints}} diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 4a82d46a3ac..e8a7c78dd2e 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -2,6 +2,7 @@ pragma solidity ^0.8.27; import {Math} from "./math/Math.sol"; +import {Accumulators} from "./structs/Accumulators.sol"; import {Bytes} from "./Bytes.sol"; import {Memory} from "./Memory.sol"; @@ -15,6 +16,7 @@ import {Memory} from "./Memory.sol"; * * https://github.com/succinctlabs/optimism-bedrock-contracts/blob/main/rlp/RLPReader.sol */ library RLP { + using Accumulators for *; using Bytes for *; using Memory for *; @@ -50,12 +52,12 @@ library RLP { ****************************************************************************************************************/ struct Encoder { - Bytes.Accumulator acc; + Accumulators.Accumulator acc; } /// @dev Create an empty RLP Encoder. function encoder() internal pure returns (Encoder memory enc) { - enc.acc = Bytes.accumulator(); + enc.acc = Accumulators.accumulator(); } /// @dev Add a boolean to a given RLP Encoder. diff --git a/contracts/utils/structs/Accumulators.sol b/contracts/utils/structs/Accumulators.sol new file mode 100644 index 00000000000..78230654f93 --- /dev/null +++ b/contracts/utils/structs/Accumulators.sol @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +import {Memory} from "../Memory.sol"; + +/** + * @dev Structure concatenating an arbitrary number of bytes buffers with limited memory allocation. + */ +library Accumulators { + /** + * @dev Bytes accumulator: a linked list of `bytes`. + * + * Note: This is a memory structure that SHOULD not be put in storage. + */ + struct Accumulator { + Memory.Pointer head; + Memory.Pointer tail; + } + + /// @dev Item (list node) in a bytes accumulator + struct AccumulatorEntry { + Memory.Pointer next; + Memory.Slice data; + } + + /// @dev Create a new (empty) accumulator + function accumulator() internal pure returns (Accumulator memory self) { + self.head = Memory.asPointer(0x00); + self.tail = Memory.asPointer(0x00); + } + + /// @dev Add a bytes buffer to (the end of) an Accumulator + function push(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { + Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: Memory.asPointer(0x00), data: Memory.asSlice(data)})); + + if (Memory.asBytes32(self.head) == 0x00) { + self.head = ptr; + self.tail = ptr; + } else { + _asAccumulatorEntry(self.tail).next = ptr; + self.tail = ptr; + } + + return self; + } + + /// @dev Add a bytes buffer to (the beginning of) an Accumulator + function shift(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { + Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: self.head, data: Memory.asSlice(data)})); + + if (Memory.asBytes32(self.head) == 0x00) { + self.head = ptr; + self.tail = ptr; + } else { + self.head = ptr; + } + + return self; + } + + /// @dev Flatten all the bytes entries in an Accumulator into a single buffer + function flatten(Accumulator memory self) internal pure returns (bytes memory result) { + assembly ("memory-safe") { + result := mload(0x40) + let ptr := add(result, 0x20) + for { + let it := mload(self) + } iszero(iszero(it)) { + it := mload(it) + } { + let slice := mload(add(it, 0x20)) + let offset := and(slice, shr(128, not(0))) + let length := shr(128, slice) + mcopy(ptr, offset, length) + ptr := add(ptr, length) + } + mstore(result, sub(ptr, add(result, 0x20))) + mstore(0x40, ptr) + } + } + + function _asPtr(AccumulatorEntry memory item) private pure returns (Memory.Pointer ptr) { + assembly ("memory-safe") { + ptr := item + } + } + + function _asAccumulatorEntry(Memory.Pointer ptr) private pure returns (AccumulatorEntry memory item) { + assembly ("memory-safe") { + item := ptr + } + } +} diff --git a/test/utils/Accumulators.t.sol b/test/utils/Accumulators.t.sol new file mode 100644 index 00000000000..83c64e9f17f --- /dev/null +++ b/test/utils/Accumulators.t.sol @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +import {Test} from "forge-std/Test.sol"; +import {Accumulators} from "@openzeppelin/contracts/utils/structs/Accumulators.sol"; +import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; + +contract AccumulatorsTest is Test { + using Accumulators for *; + + // Accumulator + function testAccumulatorPushShift() public pure { + Accumulators.Accumulator memory acc = Accumulators.accumulator(); // + acc.push(hex"11"); // 11 + acc.push(hex"22"); // 1122 + acc.shift(hex"33"); // 331122 + acc.shift(hex"44"); // 44331122 + acc.push(hex"55"); // 4433112255 + acc.shift(hex"66"); // 664433112255 + assertEq(acc.flatten(), hex"664433112255"); + } + + function testAccumulatorPush(bytes[] calldata input) public pure { + Accumulators.Accumulator memory acc = Accumulators.accumulator(); + for (uint256 i = 0; i < input.length; ++i) acc.push(input[i]); + assertEq(acc.flatten(), Bytes.concat(input)); + } + + function testAccumulatorShift(bytes[] calldata input) public pure { + Accumulators.Accumulator memory acc = Accumulators.accumulator(); + for (uint256 i = input.length; i > 0; --i) acc.shift(input[i - 1]); + assertEq(acc.flatten(), Bytes.concat(input)); + } +} diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index 297c2ab5593..09cad052780 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -224,30 +224,6 @@ contract BytesTest is Test { } } - // Accumulator - function testAccumulatorPushShift() public pure { - Bytes.Accumulator memory acc = Bytes.accumulator(); // - acc.push(hex"11"); // 11 - acc.push(hex"22"); // 1122 - acc.shift(hex"33"); // 331122 - acc.shift(hex"44"); // 44331122 - acc.push(hex"55"); // 4433112255 - acc.shift(hex"66"); // 664433112255 - assertEq(acc.flatten(), hex"664433112255"); - } - - function testAccumulatorPush(bytes[] calldata input) public pure { - Bytes.Accumulator memory acc = Bytes.accumulator(); - for (uint256 i = 0; i < input.length; ++i) acc.push(input[i]); - assertEq(acc.flatten(), input.concat()); - } - - function testAccumulatorShift(bytes[] calldata input) public pure { - Bytes.Accumulator memory acc = Bytes.accumulator(); - for (uint256 i = input.length; i > 0; --i) acc.shift(input[i - 1]); - assertEq(acc.flatten(), input.concat()); - } - // Helpers function _dirtyBytes16(bytes16 value) private pure returns (bytes16 dirty) { assembly ("memory-safe") { From 8b86b1b82e9312e0e34c29a458267cb00704adda Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Mon, 15 Sep 2025 21:25:37 +0200 Subject: [PATCH 089/114] add changeset entry --- .changeset/itchy-turkeys-allow.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/itchy-turkeys-allow.md diff --git a/.changeset/itchy-turkeys-allow.md b/.changeset/itchy-turkeys-allow.md new file mode 100644 index 00000000000..caab673c15c --- /dev/null +++ b/.changeset/itchy-turkeys-allow.md @@ -0,0 +1,5 @@ +--- +'openzeppelin-solidity': minor +--- + +`RLP`: Add a library for encoding and decoding data in Ethereum's Recursive Length Prefix format. From 785ae842ac453ebe18c2575a227bc6b31c62fb45 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Mon, 15 Sep 2025 21:26:12 +0200 Subject: [PATCH 090/114] Update test/utils/Bytes.t.sol --- test/utils/Bytes.t.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils/Bytes.t.sol b/test/utils/Bytes.t.sol index 09cad052780..e01d933460d 100644 --- a/test/utils/Bytes.t.sol +++ b/test/utils/Bytes.t.sol @@ -7,7 +7,7 @@ import {Math} from "@openzeppelin/contracts/utils/math/Math.sol"; import {Bytes} from "@openzeppelin/contracts/utils/Bytes.sol"; contract BytesTest is Test { - using Bytes for *; + using Bytes for bytes; function testSymbolicEqual(bytes memory a) public pure { assertTrue(Bytes.equal(a, a)); From 452d4b8fab845291137ff1a242cf7b0b08323db7 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Mon, 15 Sep 2025 21:26:45 +0200 Subject: [PATCH 091/114] Update .changeset/wise-webs-fly.md --- .changeset/wise-webs-fly.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/wise-webs-fly.md b/.changeset/wise-webs-fly.md index 22955c997b8..5fe2c548c6f 100644 --- a/.changeset/wise-webs-fly.md +++ b/.changeset/wise-webs-fly.md @@ -2,4 +2,4 @@ 'openzeppelin-solidity': minor --- -Accumulators: A library for merging an arbitrary dynamic number of bytes buffers. +`Accumulators`: A library for merging an arbitrary dynamic number of bytes buffers. From 00af6a47d6adb8456ec2a2fd554902941dfe1586 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Tue, 16 Sep 2025 14:33:04 +0200 Subject: [PATCH 092/114] pragma --- contracts/utils/structs/Accumulators.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/utils/structs/Accumulators.sol b/contracts/utils/structs/Accumulators.sol index 78230654f93..12f3853f08a 100644 --- a/contracts/utils/structs/Accumulators.sol +++ b/contracts/utils/structs/Accumulators.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; import {Memory} from "../Memory.sol"; From 27dd11900321e7dab56bcb19a832b3814227454c Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Tue, 16 Sep 2025 14:40:20 +0200 Subject: [PATCH 093/114] update solidity-coverage --- package-lock.json | 17 ++++++++++++----- package.json | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index d454e9303a4..b06655bae3b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,7 +46,7 @@ "solhint": "^6.0.0", "solhint-plugin-openzeppelin": "file:scripts/solhint-custom", "solidity-ast": "^0.4.50", - "solidity-coverage": "^0.8.14", + "solidity-coverage": "^0.8.16", "solidity-docgen": "^0.6.0-beta.29", "undici": "^7.4.0", "yargs": "^18.0.0" @@ -9853,14 +9853,14 @@ } }, "node_modules/solidity-coverage": { - "version": "0.8.15", - "resolved": "https://registry.npmjs.org/solidity-coverage/-/solidity-coverage-0.8.15.tgz", - "integrity": "sha512-qH7290NKww4/t/qFvnSEePEzON0k025IGVlwc8wo8Q6p+h1Tt6fV2M0k3yfsps3TomZYTROsfPXjx7MSnwD5uA==", + "version": "0.8.16", + "resolved": "https://registry.npmjs.org/solidity-coverage/-/solidity-coverage-0.8.16.tgz", + "integrity": "sha512-qKqgm8TPpcnCK0HCDLJrjbOA2tQNEJY4dHX/LSSQ9iwYFS973MwjtgYn2Iv3vfCEQJTj5xtm4cuUMzlJsJSMbg==", "dev": true, "license": "ISC", "dependencies": { "@ethersproject/abi": "^5.0.9", - "@solidity-parser/parser": "^0.19.0", + "@solidity-parser/parser": "^0.20.1", "chalk": "^2.4.2", "death": "^1.1.0", "difflib": "^0.2.4", @@ -9886,6 +9886,13 @@ "hardhat": "^2.11.0" } }, + "node_modules/solidity-coverage/node_modules/@solidity-parser/parser": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@solidity-parser/parser/-/parser-0.20.2.tgz", + "integrity": "sha512-rbu0bzwNvMcwAjH86hiEAcOeRI2EeK8zCkHDrFykh/Al8mvJeFmjy3UrE7GYQjNwOgbGUUtCn5/k8CB8zIu7QA==", + "dev": true, + "license": "MIT" + }, "node_modules/solidity-coverage/node_modules/ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", diff --git a/package.json b/package.json index 285a074ed63..479b3002bca 100644 --- a/package.json +++ b/package.json @@ -89,7 +89,7 @@ "solhint": "^6.0.0", "solhint-plugin-openzeppelin": "file:scripts/solhint-custom", "solidity-ast": "^0.4.50", - "solidity-coverage": "^0.8.14", + "solidity-coverage": "^0.8.16", "solidity-docgen": "^0.6.0-beta.29", "undici": "^7.4.0", "yargs": "^18.0.0" From 93f4d4bacbf9f07a32a80fc631a372bad0997698 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Tue, 16 Sep 2025 14:58:18 +0200 Subject: [PATCH 094/114] add slice support to Accumulator --- contracts/utils/structs/Accumulators.sol | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/contracts/utils/structs/Accumulators.sol b/contracts/utils/structs/Accumulators.sol index 12f3853f08a..62f292d637f 100644 --- a/contracts/utils/structs/Accumulators.sol +++ b/contracts/utils/structs/Accumulators.sol @@ -32,7 +32,12 @@ library Accumulators { /// @dev Add a bytes buffer to (the end of) an Accumulator function push(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { - Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: Memory.asPointer(0x00), data: Memory.asSlice(data)})); + return push(self, Memory.asSlice(data)); + } + + /// @dev Add a memory slice to (the end of) an Accumulator + function push(Accumulator memory self, Memory.Slice data) internal pure returns (Accumulator memory) { + Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: Memory.asPointer(0x00), data: data})); if (Memory.asBytes32(self.head) == 0x00) { self.head = ptr; @@ -47,7 +52,12 @@ library Accumulators { /// @dev Add a bytes buffer to (the beginning of) an Accumulator function shift(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { - Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: self.head, data: Memory.asSlice(data)})); + return shift(self, Memory.asSlice(data)); + } + + /// @dev Add a memory slice to (the beginning of) an Accumulator + function shift(Accumulator memory self, Memory.Slice data) internal pure returns (Accumulator memory) { + Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: self.head, data: data})); if (Memory.asBytes32(self.head) == 0x00) { self.head = ptr; From 0e503b4fd5b29e171d97f731d1d65bb4d87c9a17 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Tue, 16 Sep 2025 17:21:35 +0200 Subject: [PATCH 095/114] revert solidity-coverage update --- package-lock.json | 17 +++++------------ package.json | 2 +- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index b06655bae3b..d454e9303a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,7 +46,7 @@ "solhint": "^6.0.0", "solhint-plugin-openzeppelin": "file:scripts/solhint-custom", "solidity-ast": "^0.4.50", - "solidity-coverage": "^0.8.16", + "solidity-coverage": "^0.8.14", "solidity-docgen": "^0.6.0-beta.29", "undici": "^7.4.0", "yargs": "^18.0.0" @@ -9853,14 +9853,14 @@ } }, "node_modules/solidity-coverage": { - "version": "0.8.16", - "resolved": "https://registry.npmjs.org/solidity-coverage/-/solidity-coverage-0.8.16.tgz", - "integrity": "sha512-qKqgm8TPpcnCK0HCDLJrjbOA2tQNEJY4dHX/LSSQ9iwYFS973MwjtgYn2Iv3vfCEQJTj5xtm4cuUMzlJsJSMbg==", + "version": "0.8.15", + "resolved": "https://registry.npmjs.org/solidity-coverage/-/solidity-coverage-0.8.15.tgz", + "integrity": "sha512-qH7290NKww4/t/qFvnSEePEzON0k025IGVlwc8wo8Q6p+h1Tt6fV2M0k3yfsps3TomZYTROsfPXjx7MSnwD5uA==", "dev": true, "license": "ISC", "dependencies": { "@ethersproject/abi": "^5.0.9", - "@solidity-parser/parser": "^0.20.1", + "@solidity-parser/parser": "^0.19.0", "chalk": "^2.4.2", "death": "^1.1.0", "difflib": "^0.2.4", @@ -9886,13 +9886,6 @@ "hardhat": "^2.11.0" } }, - "node_modules/solidity-coverage/node_modules/@solidity-parser/parser": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@solidity-parser/parser/-/parser-0.20.2.tgz", - "integrity": "sha512-rbu0bzwNvMcwAjH86hiEAcOeRI2EeK8zCkHDrFykh/Al8mvJeFmjy3UrE7GYQjNwOgbGUUtCn5/k8CB8zIu7QA==", - "dev": true, - "license": "MIT" - }, "node_modules/solidity-coverage/node_modules/ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", diff --git a/package.json b/package.json index 479b3002bca..285a074ed63 100644 --- a/package.json +++ b/package.json @@ -89,7 +89,7 @@ "solhint": "^6.0.0", "solhint-plugin-openzeppelin": "file:scripts/solhint-custom", "solidity-ast": "^0.4.50", - "solidity-coverage": "^0.8.16", + "solidity-coverage": "^0.8.14", "solidity-docgen": "^0.6.0-beta.29", "undici": "^7.4.0", "yargs": "^18.0.0" From 9cfacebb84a8a1e8686b06b375e00b1ff515fb83 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 17 Sep 2025 15:05:11 +0200 Subject: [PATCH 096/114] Update .changeset/modern-moments-raise.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ernesto García --- .changeset/modern-moments-raise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/modern-moments-raise.md b/.changeset/modern-moments-raise.md index 38526551e0f..3e7f91c5390 100644 --- a/.changeset/modern-moments-raise.md +++ b/.changeset/modern-moments-raise.md @@ -2,4 +2,4 @@ 'openzeppelin-solidity': minor --- -`Memory`: Add a UDVT for handling slices or memory space similarly to calldata slices. +`Memory`: Add a UDVT for handling slices on memory space similarly to calldata slices. From 07f5db8d4eaa35d9f760ce602faf5fe0cb7eb68f Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 17 Sep 2025 15:14:44 +0200 Subject: [PATCH 097/114] move fuzzing test --- test/utils/{ => structs}/Accumulators.t.sol | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/utils/{ => structs}/Accumulators.t.sol (100%) diff --git a/test/utils/Accumulators.t.sol b/test/utils/structs/Accumulators.t.sol similarity index 100% rename from test/utils/Accumulators.t.sol rename to test/utils/structs/Accumulators.t.sol From 8534802a59d40348e0c420f8dc2be528c3b4c2d4 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 17 Sep 2025 15:33:53 +0200 Subject: [PATCH 098/114] zero out-of-slice bytes when doing load(Slice,uint256) --- contracts/utils/Memory.sol | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 9e3722e6ba9..d92dac5a87a 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -3,6 +3,7 @@ pragma solidity ^0.8.24; import {Panic} from "./Panic.sol"; +import {Math} from "./math/Math.sol"; /** * @dev Utilities to manipulate memory. @@ -90,12 +91,13 @@ library Memory { /** * @dev Read a bytes32 buffer from a given Slice at a specific offset * - * Note:If offset > length(slice) - 32, part of the return value will be out of bound and should be ignored. + * Note: If offset > length(slice) - 32, part of the return value will be out of bound of the slice. These bytes are zeroed. */ function load(Slice self, uint256 offset) internal pure returns (bytes32 value) { - if (offset >= length(self)) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); + uint256 outOfBoundBytes = Math.saturatingSub(32 + offset, length(self)); + if (outOfBoundBytes > 31) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); assembly ("memory-safe") { - value := mload(add(and(self, shr(128, not(0))), offset)) + value := and(mload(add(and(self, shr(128, not(0))), offset)), shl(mul(8, outOfBoundBytes), not(0))) } } From c3df360eb9474b6226d35466020d4700ca1bae98 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 17 Sep 2025 15:45:22 +0200 Subject: [PATCH 099/114] doc --- contracts/utils/Memory.sol | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index d92dac5a87a..b249e60a2e1 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -54,9 +54,15 @@ library Memory { } } - /// @dev Private helper: create a slice from raw values (length and pointer) + /** + * @dev Private helper: create a slice from raw values (length and pointer) + * + * NOTE: this function MUST NOT be called with `len` or `ptr` that exceed `2**128-1`. This should never be + * the case of slices produced by `asSlice(bytes)`, and function that reduce the scope of slices + * (`slice(Slice,uint256)` and `slice(Slice,uint256, uint256)`) should not cause this issue if the parent slice is + * correct. + */ function _asSlice(uint256 len, Memory.Pointer ptr) private pure returns (Slice result) { - // TODO: Fail if len or ptr is larger than type(uint128).max ? assembly ("memory-safe") { result := or(shl(128, len), ptr) } From 8c98d4a43309c5503de2e750f62928693fa090c2 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 17 Sep 2025 18:26:03 +0200 Subject: [PATCH 100/114] dec -> hex --- contracts/utils/Memory.sol | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index b249e60a2e1..70eeec1b9ea 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -100,8 +100,9 @@ library Memory { * Note: If offset > length(slice) - 32, part of the return value will be out of bound of the slice. These bytes are zeroed. */ function load(Slice self, uint256 offset) internal pure returns (bytes32 value) { - uint256 outOfBoundBytes = Math.saturatingSub(32 + offset, length(self)); - if (outOfBoundBytes > 31) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); + uint256 outOfBoundBytes = Math.saturatingSub(0x20 + offset, length(self)); + if (outOfBoundBytes > 0x1f) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); + assembly ("memory-safe") { value := and(mload(add(and(self, shr(128, not(0))), offset)), shl(mul(8, outOfBoundBytes), not(0))) } From 6efa28d27f33c331f3dad8cdc4f25f1fdfb4b8c4 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Wed, 17 Sep 2025 21:45:31 +0200 Subject: [PATCH 101/114] up --- contracts/utils/Memory.sol | 2 +- contracts/utils/structs/Accumulators.sol | 35 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 70eeec1b9ea..64ad527d534 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -97,7 +97,7 @@ library Memory { /** * @dev Read a bytes32 buffer from a given Slice at a specific offset * - * Note: If offset > length(slice) - 32, part of the return value will be out of bound of the slice. These bytes are zeroed. + * NOTE: If offset > length(slice) - 0x20, part of the return value will be out of bound of the slice. These bytes are zeroed. */ function load(Slice self, uint256 offset) internal pure returns (bytes32 value) { uint256 outOfBoundBytes = Math.saturatingSub(0x20 + offset, length(self)); diff --git a/contracts/utils/structs/Accumulators.sol b/contracts/utils/structs/Accumulators.sol index 62f292d637f..5d816346ad7 100644 --- a/contracts/utils/structs/Accumulators.sol +++ b/contracts/utils/structs/Accumulators.sol @@ -6,12 +6,31 @@ import {Memory} from "../Memory.sol"; /** * @dev Structure concatenating an arbitrary number of bytes buffers with limited memory allocation. + * + * The Accumulators library provides a memory-efficient alternative to repeated concatenation of bytes. + * Instead of copying data on each concatenation (O(n**2) complexity), it builds a linked list of references + * to existing data and performs a single memory allocation during flattening (O(n) complexity). + * + * Uses 0x00 as sentinel value for empty state (i.e. null pointers) + * + * == How it works + * + * 1. Create an empty accumulator with null head/tail pointers + * 2. Add data using {push} (append) or {shift} (prepend). It creates linked list nodes + * 3. Each node stores a reference to existing data (no copying) + * 4. Call {flatten} to materialize the final concatenated result in a single operation + * + * == Performance + * + * * Addition: O(1) per operation (just pointer manipulation) + * * Flattening: O(n) single pass with one memory allocation + * * Memory: Minimal overhead until flattening (only stores references) */ library Accumulators { /** * @dev Bytes accumulator: a linked list of `bytes`. * - * Note: This is a memory structure that SHOULD not be put in storage. + * NOTE: This is a memory structure that SHOULD not be put in storage. */ struct Accumulator { Memory.Pointer head; @@ -26,8 +45,8 @@ library Accumulators { /// @dev Create a new (empty) accumulator function accumulator() internal pure returns (Accumulator memory self) { - self.head = Memory.asPointer(0x00); - self.tail = Memory.asPointer(0x00); + self.head = _nullPtr(); + self.tail = _nullPtr(); } /// @dev Add a bytes buffer to (the end of) an Accumulator @@ -37,9 +56,9 @@ library Accumulators { /// @dev Add a memory slice to (the end of) an Accumulator function push(Accumulator memory self, Memory.Slice data) internal pure returns (Accumulator memory) { - Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: Memory.asPointer(0x00), data: data})); + Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: _nullPtr(), data: data})); - if (Memory.asBytes32(self.head) == 0x00) { + if (Memory.asBytes32(self.head) == Memory.asBytes32(_nullPtr())) { self.head = ptr; self.tail = ptr; } else { @@ -59,7 +78,7 @@ library Accumulators { function shift(Accumulator memory self, Memory.Slice data) internal pure returns (Accumulator memory) { Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: self.head, data: data})); - if (Memory.asBytes32(self.head) == 0x00) { + if (Memory.asBytes32(self.head) == Memory.asBytes32(_nullPtr())) { self.head = ptr; self.tail = ptr; } else { @@ -101,4 +120,8 @@ library Accumulators { item := ptr } } + + function _nullPtr() private pure returns (Memory.Pointer) { + return Memory.asPointer(0x00); + } } From 74786169d475f0f2e47f38cc3a6cba1defa57d54 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 18 Sep 2025 10:44:37 +0200 Subject: [PATCH 102/114] Reorder private functions --- contracts/utils/Memory.sol | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 64ad527d534..20c01942fbf 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -54,27 +54,6 @@ library Memory { } } - /** - * @dev Private helper: create a slice from raw values (length and pointer) - * - * NOTE: this function MUST NOT be called with `len` or `ptr` that exceed `2**128-1`. This should never be - * the case of slices produced by `asSlice(bytes)`, and function that reduce the scope of slices - * (`slice(Slice,uint256)` and `slice(Slice,uint256, uint256)`) should not cause this issue if the parent slice is - * correct. - */ - function _asSlice(uint256 len, Memory.Pointer ptr) private pure returns (Slice result) { - assembly ("memory-safe") { - result := or(shl(128, len), ptr) - } - } - - /// @dev Returns the memory location of a given slice (equiv to self.offset for calldata slices) - function _pointer(Slice self) private pure returns (Memory.Pointer result) { - assembly ("memory-safe") { - result := and(self, shr(128, not(0))) - } - } - /// @dev Returns the length of a given slice (equiv to self.length for calldata slices) function length(Slice self) internal pure returns (uint256 result) { assembly ("memory-safe") { @@ -119,4 +98,25 @@ library Memory { mstore(0x40, add(add(result, len), 0x20)) } } + + /** + * @dev Private helper: create a slice from raw values (length and pointer) + * + * NOTE: this function MUST NOT be called with `len` or `ptr` that exceed `2**128-1`. This should never be + * the case of slices produced by `asSlice(bytes)`, and function that reduce the scope of slices + * (`slice(Slice,uint256)` and `slice(Slice,uint256, uint256)`) should not cause this issue if the parent slice is + * correct. + */ + function _asSlice(uint256 len, Memory.Pointer ptr) private pure returns (Slice result) { + assembly ("memory-safe") { + result := or(shl(128, len), ptr) + } + } + + /// @dev Returns the memory location of a given slice (equiv to self.offset for calldata slices) + function _pointer(Slice self) private pure returns (Memory.Pointer result) { + assembly ("memory-safe") { + result := and(self, shr(128, not(0))) + } + } } From fb62ca3826d45c7133be6a1c56b1094b0655f799 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 18 Sep 2025 10:47:19 +0200 Subject: [PATCH 103/114] Docs --- contracts/utils/Memory.sol | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 20c01942fbf..2cf7741a31c 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -9,9 +9,11 @@ import {Math} from "./math/Math.sol"; * @dev Utilities to manipulate memory. * * Memory is a contiguous and dynamic byte array in which Solidity stores non-primitive types. - * This library provides functions to manipulate pointers to this dynamic array. + * This library provides functions to manipulate pointers to this dynamic array and work with slices of it. * - * WARNING: When manipulating memory, make sure to follow the Solidity documentation + * Slices provide a view into a portion of memory without copying data, enabling efficient substring operations. + * + * WARNING: When manipulating memory pointers or slices, make sure to follow the Solidity documentation * guidelines for https://docs.soliditylang.org/en/v0.8.20/assembly.html#memory-safety[Memory Safety]. */ library Memory { From 698221fe754922fdfe10c16a9b5e3651d0f18dfb Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 18 Sep 2025 10:51:34 +0200 Subject: [PATCH 104/114] Add to CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beaeed885ee..35abf70d56a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ### Breaking changes -- Update minimum pragma to 0.8.24 in `Votes`, `VotesExtended`, `ERC20Votes`, `Strings`, `ERC1155URIStorage`, `MessageHashUtils`, `ERC721URIStorage`, `ERC721Votes`, `ERC721Wrapper`, `ERC721Burnable`, `ERC721Consecutive`, `ERC721Enumerable`, `ERC721Pausable`, `ERC721Royalty`, `ERC721Wrapper`, `EIP712`, and `ERC7739`. ([#5726](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5726)) +- Update minimum pragma to 0.8.24 in `Votes`, `VotesExtended`, `ERC20Votes`, `Strings`, `ERC1155URIStorage`, `MessageHashUtils`, `ERC721URIStorage`, `ERC721Votes`, `ERC721Wrapper`, `ERC721Burnable`, `ERC721Consecutive`, `ERC721Enumerable`, `ERC721Pausable`, `ERC721Royalty`, `ERC721Wrapper`, `EIP712`, `ERC4626` and `ERC7739`. ([#5726](https://github.com/OpenZeppelin/openzeppelin-contracts/pull/5726)) ### Deprecation From 2bd9cc4e5ec444158c02b7134a66635847b3378f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Thu, 18 Sep 2025 11:02:48 +0200 Subject: [PATCH 105/114] Delete .changeset/lovely-cooks-add.md --- .changeset/lovely-cooks-add.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/lovely-cooks-add.md diff --git a/.changeset/lovely-cooks-add.md b/.changeset/lovely-cooks-add.md deleted file mode 100644 index 6637c92478d..00000000000 --- a/.changeset/lovely-cooks-add.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'openzeppelin-solidity': minor ---- - -`RLP`: Add library for Ethereum's Recursive Length Prefix encoding/decoding. From fefd03922da679fd1a17268dbee2076ea2215a61 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 18 Sep 2025 11:23:00 +0200 Subject: [PATCH 106/114] Uncomment invalid tests --- test/utils/RLP.test.js | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js index 856a93a48fa..09561b52ca6 100644 --- a/test/utils/RLP.test.js +++ b/test/utils/RLP.test.js @@ -133,18 +133,17 @@ describe('RLP', function () { } }); - // const invalidTests = [ - // { name: 'short string with invalid length', input: '0x8100' }, - // { name: 'long string with invalid length prefix', input: '0xb800' }, - // { name: 'list with invalid length', input: '0xc100' }, - // { name: 'truncated long string', input: '0xb838' }, - // { name: 'invalid single byte encoding (non-minimal)', input: '0x8100' }, - // ]; - - // invalidTests.forEach(({ name, input }) => { - // it(`encodes ${name} into invalid RLP`, async function () { - // const item = await this.mock.$toItem(input); - // await expect(this.mock.$decodeBytes_bytes(item)).to.be.reverted; - // }); - // }); + const invalidTests = [ + { name: 'short string with invalid length', input: '0x8100' }, + { name: 'long string with invalid length prefix', input: '0xb800' }, + { name: 'list with invalid length', input: '0xc100' }, + { name: 'truncated long string', input: '0xb838' }, + { name: 'invalid single byte encoding (non-minimal)', input: '0x8100' }, + ]; + + invalidTests.forEach(({ name, input }) => { + it(`rejects ${name}`, async function () { + await expect(this.mock.$decodeBytes(input)).to.be.reverted; + }); + }); }); From f2e5c8123e653a54ee13692307f0b285988f45d3 Mon Sep 17 00:00:00 2001 From: ernestognw Date: Thu, 18 Sep 2025 12:15:28 +0200 Subject: [PATCH 107/114] up --- contracts/utils/RLP.sol | 17 +++++++++++++++-- test/utils/RLP.test.js | 13 +++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index e8a7c78dd2e..2cc73dcaf83 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -12,6 +12,7 @@ import {Memory} from "./Memory.sol"; * It's used for encoding everything from transactions to blocks to Patricia-Merkle tries. * * Inspired by + * * * https://github.com/succinctlabs/optimism-bedrock-contracts/blob/main/rlp/RLPWriter.sol * * https://github.com/succinctlabs/optimism-bedrock-contracts/blob/main/rlp/RLPReader.sol */ @@ -131,8 +132,8 @@ library RLP { function encode(address input) internal pure returns (bytes memory result) { assembly ("memory-safe") { result := mload(0x40) - mstore(result, 0x15) // length of the encoded data: 1 (prefix) + 14 (address) - mstore(add(result, 0x20), or(shl(248, 0x94), shl(88, input))) // prefix (0x94 = SHORT_OFFSET + 14) + input + mstore(result, 0x15) // length of the encoded data: 1 (prefix) + 0x14 (address) + mstore(add(result, 0x20), or(shl(248, 0x94), shl(88, input))) // prefix (0x94 = SHORT_OFFSET + 0x14) + input mstore(0x40, add(result, 0x35)) // reserve memory } } @@ -291,34 +292,46 @@ library RLP { * DECODING - FROM BYTES * ****************************************************************************************************************/ + /// @dev Decode an RLP encoded bool from bytes. See {readBool} function decodeBool(bytes memory item) internal pure returns (bool) { return readBool(item.asSlice()); } + /// @dev Decode an RLP encoded address from bytes. See {readAddress} function decodeAddress(bytes memory item) internal pure returns (address) { return readAddress(item.asSlice()); } + /// @dev Decode an RLP encoded uint256 from bytes. See {readUint256} function decodeUint256(bytes memory item) internal pure returns (uint256) { return readUint256(item.asSlice()); } + /// @dev Decode an RLP encoded bytes32 from bytes. See {readBytes32} function decodeBytes32(bytes memory item) internal pure returns (bytes32) { return readBytes32(item.asSlice()); } + /// @dev Decode an RLP encoded bytes from bytes. See {readBytes} function decodeBytes(bytes memory item) internal pure returns (bytes memory) { return readBytes(item.asSlice()); } + /// @dev Decode an RLP encoded string from bytes. See {readString} function decodeString(bytes memory item) internal pure returns (string memory) { return readString(item.asSlice()); } + /// @dev Decode an RLP encoded list from bytes. See {readList} function decodeList(bytes memory value) internal pure returns (Memory.Slice[] memory) { return readList(value.asSlice()); } + /// @dev Decode an RLP encoded list from bytes with custom maxListLength. See {readList-Memory-Slice-uint256-} + function decodeList(bytes memory value, uint256 maxListLength) internal pure returns (Memory.Slice[] memory) { + return readList(value.asSlice(), maxListLength); + } + /** * @dev Decodes an RLP `item`'s `length and type from its prefix. * Returns the offset, length, and type of the RLP item based on the encoding rules. diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js index 09561b52ca6..6da1866fb00 100644 --- a/test/utils/RLP.test.js +++ b/test/utils/RLP.test.js @@ -133,6 +133,19 @@ describe('RLP', function () { } }); + it('decodes list with custom maxListLength', async function () { + const input = ['0x17', '0x42', '0x0123', '0xabcdef']; + const encoded = ethers.encodeRlp(input); + + // Correct lengths + await expect(this.mock['$decodeList(bytes,uint256)'](encoded, 4)).to.eventually.have.length(4); + await expect(this.mock['$decodeList(bytes,uint256)'](encoded, 10)).to.eventually.have.length(4); + await expect(this.mock['$decodeList(bytes)'](encoded)).to.eventually.have.length(4); + + // Insufficient maxListLength + await expect(this.mock['$decodeList(bytes,uint256)'](encoded, 3)).to.be.reverted; + }); + const invalidTests = [ { name: 'short string with invalid length', input: '0x8100' }, { name: 'long string with invalid length prefix', input: '0xb800' }, From ff4c153e5a2efd8f4dcc48472e2d16ef9b87217e Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 18 Sep 2025 16:28:17 +0200 Subject: [PATCH 108/114] dynamic allocation of space for lists --- contracts/utils/RLP.sol | 41 ++++++++++++++++++++--------------------- test/utils/RLP.test.js | 13 ------------- 2 files changed, 20 insertions(+), 34 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 2cc73dcaf83..1913657abe9 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -252,38 +252,42 @@ library RLP { return item.slice(offset, length).toBytes(); } + /// @dev Decodes an RLP encoded string. See {encode-string} function readString(Memory.Slice item) internal pure returns (string memory) { return string(readBytes(item)); } - /// @dev Decodes an RLP encoded list into an array of RLP Items. This function supports list up to 32 elements - function readList(Memory.Slice item) internal pure returns (Memory.Slice[] memory) { - return readList(item, 32); - } - - /** - * @dev Variant of {readList-bytes32} that supports long lists up to `maxListLength`. Setting `maxListLength` to - * a high value will cause important, and costly, memory expansion. - */ - function readList(Memory.Slice item, uint256 maxListLength) internal pure returns (Memory.Slice[] memory) { + /// @dev Decodes an RLP encoded list into an array of RLP Items. + function readList(Memory.Slice item) internal pure returns (Memory.Slice[] memory list) { uint256 itemLength = item.length(); (uint256 listOffset, uint256 listLength, ItemType itemType) = _decodeLength(item); require(itemType == ItemType.List, RLPUnexpectedType(ItemType.List, itemType)); require(itemLength == listOffset + listLength, RLPContentLengthMismatch(listOffset + listLength, itemLength)); - Memory.Slice[] memory list = new Memory.Slice[](maxListLength); + // Start a buffer in the unallocated space + uint256 ptr; + assembly ("memory-safe") { + list := mload(0x40) + ptr := add(list, 0x20) + } - uint256 itemCount; - for (uint256 currentOffset = listOffset; currentOffset < itemLength; ++itemCount) { + // Get all items in order, and push them to the buffer + for (uint256 currentOffset = listOffset; currentOffset < itemLength; ptr += 0x20) { (uint256 elementOffset, uint256 elementLength, ) = _decodeLength(item.slice(currentOffset)); - list[itemCount] = item.slice(currentOffset, elementLength + elementOffset); + Memory.Slice element = item.slice(currentOffset, elementLength + elementOffset); currentOffset += elementOffset + elementLength; + + // Write item to the buffer + assembly ("memory-safe") { + mstore(ptr, element) + } } - // Decrease the array size to match the actual item count. + // write list length and reserve space assembly ("memory-safe") { - mstore(list, itemCount) + mstore(list, div(sub(ptr, add(list, 0x20)), 0x20)) + mstore(0x40, ptr) } return list; } @@ -327,11 +331,6 @@ library RLP { return readList(value.asSlice()); } - /// @dev Decode an RLP encoded list from bytes with custom maxListLength. See {readList-Memory-Slice-uint256-} - function decodeList(bytes memory value, uint256 maxListLength) internal pure returns (Memory.Slice[] memory) { - return readList(value.asSlice(), maxListLength); - } - /** * @dev Decodes an RLP `item`'s `length and type from its prefix. * Returns the offset, length, and type of the RLP item based on the encoding rules. diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js index 6da1866fb00..09561b52ca6 100644 --- a/test/utils/RLP.test.js +++ b/test/utils/RLP.test.js @@ -133,19 +133,6 @@ describe('RLP', function () { } }); - it('decodes list with custom maxListLength', async function () { - const input = ['0x17', '0x42', '0x0123', '0xabcdef']; - const encoded = ethers.encodeRlp(input); - - // Correct lengths - await expect(this.mock['$decodeList(bytes,uint256)'](encoded, 4)).to.eventually.have.length(4); - await expect(this.mock['$decodeList(bytes,uint256)'](encoded, 10)).to.eventually.have.length(4); - await expect(this.mock['$decodeList(bytes)'](encoded)).to.eventually.have.length(4); - - // Insufficient maxListLength - await expect(this.mock['$decodeList(bytes,uint256)'](encoded, 3)).to.be.reverted; - }); - const invalidTests = [ { name: 'short string with invalid length', input: '0x8100' }, { name: 'long string with invalid length prefix', input: '0xb800' }, From dd242c26f5dcc8d0568ef67b2d35b1e8df096445 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 18 Sep 2025 16:37:52 +0200 Subject: [PATCH 109/114] use a single custom error --- contracts/utils/RLP.sol | 47 +++++++++++++++-------------------------- test/utils/RLP.test.js | 2 +- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 1913657abe9..553c459e9e1 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -21,17 +21,8 @@ library RLP { using Bytes for *; using Memory for *; - /// @dev Items with length 0 are not RLP items. - error RLPEmptyItem(); - - /// @dev The `item` is not of the `expected` type. - error RLPUnexpectedType(ItemType expected, ItemType actual); - - /// @dev The item is not long enough to contain the data. - error RLPInvalidDataRemainder(uint256 minLength, uint256 actualLength); - - /// @dev The content length does not match the expected length. - error RLPContentLengthMismatch(uint256 expectedLength, uint256 actualLength); + /// @dev The item is not properly formatted and cannot de decoded. + error RLPInvalidEncoding(); enum ItemType { Data, // Single data value @@ -223,17 +214,17 @@ library RLP { /// @dev Decode an RLP encoded address. See {encode-address} function readAddress(Memory.Slice item) internal pure returns (address) { uint256 length = item.length(); - require(length == 1 || length == 21, RLPContentLengthMismatch(21, length)); + require(length == 1 || length == 21, RLPInvalidEncoding()); return address(uint160(readUint256(item))); } /// @dev Decode an RLP encoded uint256. See {encode-uint256} function readUint256(Memory.Slice item) internal pure returns (uint256) { uint256 length = item.length(); - require(length <= 33, RLPContentLengthMismatch(32, length)); + require(length <= 33, RLPInvalidEncoding()); (uint256 itemOffset, uint256 itemLength, ItemType itemType) = _decodeLength(item); - require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); + require(itemType == ItemType.Data, RLPInvalidEncoding()); return itemLength == 0 ? 0 : uint256(item.load(itemOffset)) >> (256 - 8 * itemLength); } @@ -246,7 +237,7 @@ library RLP { /// @dev Decodes an RLP encoded bytes. See {encode-bytes} function readBytes(Memory.Slice item) internal pure returns (bytes memory) { (uint256 offset, uint256 length, ItemType itemType) = _decodeLength(item); - require(itemType == ItemType.Data, RLPUnexpectedType(ItemType.Data, itemType)); + require(itemType == ItemType.Data, RLPInvalidEncoding()); // Length is checked by {toBytes} return item.slice(offset, length).toBytes(); @@ -262,8 +253,7 @@ library RLP { uint256 itemLength = item.length(); (uint256 listOffset, uint256 listLength, ItemType itemType) = _decodeLength(item); - require(itemType == ItemType.List, RLPUnexpectedType(ItemType.List, itemType)); - require(itemLength == listOffset + listLength, RLPContentLengthMismatch(listOffset + listLength, itemLength)); + require(itemType == ItemType.List && itemLength == listOffset + listLength, RLPInvalidEncoding()); // Start a buffer in the unallocated space uint256 ptr; @@ -340,7 +330,7 @@ library RLP { ) private pure returns (uint256 _offset, uint256 _length, ItemType _itemtype) { uint256 itemLength = item.length(); - require(itemLength != 0, RLPEmptyItem()); + require(itemLength != 0, RLPInvalidEncoding()); uint8 prefix = uint8(bytes1(item.load(0))); if (prefix < LONG_OFFSET) { @@ -351,21 +341,19 @@ library RLP { } else if (prefix <= SHORT_OFFSET + SHORT_THRESHOLD) { // Case: Short string (0-55 bytes) uint256 strLength = prefix - SHORT_OFFSET; - require(itemLength > strLength, RLPInvalidDataRemainder(strLength, itemLength)); - if (strLength == 1) { - require(bytes1(item.load(1)) >= bytes1(SHORT_OFFSET)); // TODO: custom error for sanity checks - } + require( + itemLength > strLength && (strLength != 1 || bytes1(item.load(1)) >= bytes1(SHORT_OFFSET)), + RLPInvalidEncoding() + ); return (1, strLength, ItemType.Data); } else { // Case: Long string (>55 bytes) uint256 lengthLength = prefix - SHORT_OFFSET - SHORT_THRESHOLD; - require(itemLength > lengthLength, RLPInvalidDataRemainder(lengthLength, itemLength)); - require(bytes1(item.load(0)) != 0x00); // TODO: custom error for sanity checks + require(itemLength > lengthLength && bytes1(item.load(0)) != 0x00, RLPInvalidEncoding()); uint256 len = uint256(item.load(1)) >> (256 - 8 * lengthLength); - require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); - require(itemLength > lengthLength + len, RLPContentLengthMismatch(lengthLength + len, itemLength)); + require(len > SHORT_THRESHOLD && itemLength > lengthLength + len, RLPInvalidEncoding()); return (lengthLength + 1, len, ItemType.Data); } @@ -374,18 +362,17 @@ library RLP { if (prefix <= LONG_OFFSET + SHORT_THRESHOLD) { // Case: Short list uint256 listLength = prefix - LONG_OFFSET; - require(item.length() > listLength, RLPInvalidDataRemainder(listLength, itemLength)); + require(item.length() > listLength, RLPInvalidEncoding()); return (1, listLength, ItemType.List); } else { // Case: Long list uint256 lengthLength = prefix - LONG_OFFSET - SHORT_THRESHOLD; - require(itemLength > lengthLength, RLPInvalidDataRemainder(lengthLength, itemLength)); + require(itemLength > lengthLength, RLPInvalidEncoding()); require(bytes1(item.load(0)) != 0x00); uint256 len = uint256(item.load(1)) >> (256 - 8 * lengthLength); - require(len > SHORT_THRESHOLD, RLPInvalidDataRemainder(SHORT_THRESHOLD, len)); - require(itemLength > lengthLength + len, RLPContentLengthMismatch(lengthLength + len, itemLength)); + require(len > SHORT_THRESHOLD && itemLength > lengthLength + len, RLPInvalidEncoding()); return (lengthLength + 1, len, ItemType.List); } diff --git a/test/utils/RLP.test.js b/test/utils/RLP.test.js index 09561b52ca6..c4d3dda7209 100644 --- a/test/utils/RLP.test.js +++ b/test/utils/RLP.test.js @@ -143,7 +143,7 @@ describe('RLP', function () { invalidTests.forEach(({ name, input }) => { it(`rejects ${name}`, async function () { - await expect(this.mock.$decodeBytes(input)).to.be.reverted; + await expect(this.mock.$decodeBytes(input)).to.be.revertedWithCustomError(this.mock, 'RLPInvalidEncoding'); }); }); }); From 46914e92158aaab2071f33cdc24f233ba5a385b2 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 18 Sep 2025 19:04:40 +0200 Subject: [PATCH 110/114] Update contracts/utils/RLP.sol --- contracts/utils/RLP.sol | 1 - 1 file changed, 1 deletion(-) diff --git a/contracts/utils/RLP.sol b/contracts/utils/RLP.sol index 553c459e9e1..4066e296ff5 100644 --- a/contracts/utils/RLP.sol +++ b/contracts/utils/RLP.sol @@ -279,7 +279,6 @@ library RLP { mstore(list, div(sub(ptr, add(list, 0x20)), 0x20)) mstore(0x40, ptr) } - return list; } /**************************************************************************************************************** From fdcb15f4f91c5bff30a48a06c5801fc0d29b1019 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 18 Sep 2025 19:22:53 +0200 Subject: [PATCH 111/114] Memory pointer equality --- contracts/utils/Memory.sol | 5 +++++ contracts/utils/structs/Accumulators.sol | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 2cf7741a31c..0614f3182e6 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -47,6 +47,11 @@ library Memory { return Pointer.wrap(value); } + /// @dev Equality comparator for memory pointers. + function equal(Pointer ptr1, Pointer ptr2) internal pure returns (bool) { + return Pointer.unwrap(ptr1) == Pointer.unwrap(ptr2); + } + type Slice is bytes32; /// @dev Get a slice representation of a bytes object in memory diff --git a/contracts/utils/structs/Accumulators.sol b/contracts/utils/structs/Accumulators.sol index 5d816346ad7..424db6bbee5 100644 --- a/contracts/utils/structs/Accumulators.sol +++ b/contracts/utils/structs/Accumulators.sol @@ -27,6 +27,8 @@ import {Memory} from "../Memory.sol"; * * Memory: Minimal overhead until flattening (only stores references) */ library Accumulators { + using Memory for *; + /** * @dev Bytes accumulator: a linked list of `bytes`. * @@ -51,14 +53,14 @@ library Accumulators { /// @dev Add a bytes buffer to (the end of) an Accumulator function push(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { - return push(self, Memory.asSlice(data)); + return push(self, data.asSlice()); } /// @dev Add a memory slice to (the end of) an Accumulator function push(Accumulator memory self, Memory.Slice data) internal pure returns (Accumulator memory) { Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: _nullPtr(), data: data})); - if (Memory.asBytes32(self.head) == Memory.asBytes32(_nullPtr())) { + if (_nullPtr().equal(self.head)) { self.head = ptr; self.tail = ptr; } else { @@ -71,14 +73,14 @@ library Accumulators { /// @dev Add a bytes buffer to (the beginning of) an Accumulator function shift(Accumulator memory self, bytes memory data) internal pure returns (Accumulator memory) { - return shift(self, Memory.asSlice(data)); + return shift(self, data.asSlice()); } /// @dev Add a memory slice to (the beginning of) an Accumulator function shift(Accumulator memory self, Memory.Slice data) internal pure returns (Accumulator memory) { Memory.Pointer ptr = _asPtr(AccumulatorEntry({next: self.head, data: data})); - if (Memory.asBytes32(self.head) == Memory.asBytes32(_nullPtr())) { + if (_nullPtr().equal(self.head)) { self.head = ptr; self.tail = ptr; } else { From 0c5d508f239f3ba9ea8238561640ac6c9cab182b Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Fri, 19 Sep 2025 10:47:49 +0200 Subject: [PATCH 112/114] test panic code of out-of-bound slice operations --- test/utils/Memory.test.js | 97 +++++++++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 15 deletions(-) diff --git a/test/utils/Memory.test.js b/test/utils/Memory.test.js index 9c1570aba0a..0a7a1949bf8 100644 --- a/test/utils/Memory.test.js +++ b/test/utils/Memory.test.js @@ -1,6 +1,12 @@ const { ethers } = require('hardhat'); const { expect } = require('chai'); const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); +const { PANIC_CODES } = require('@nomicfoundation/hardhat-chai-matchers/panic'); + +const { generators } = require('../helpers/random'); + +const formatSlice = ({ length, ptr = 0xa0 }) => + ethers.toBeHex((ethers.toBigInt(length) << 128n) | ethers.toBigInt(ptr), 32); async function fixture() { const mock = await ethers.deployContract('$Memory'); @@ -12,28 +18,89 @@ describe('Memory', function () { Object.assign(this, await loadFixture(fixture)); }); - describe('free pointer', function () { - it('sets free memory pointer', async function () { - const ptr = ethers.toBeHex(0xa0, 32); - await expect(this.mock.$setFreeMemoryPointer(ptr)).to.not.be.reverted; + describe('Pointer', function () { + describe('free pointer', function () { + it('sets free memory pointer', async function () { + const ptr = ethers.toBeHex(0xa0, 32); + await expect(this.mock.$setFreeMemoryPointer(ptr)).to.not.be.reverted; + }); + + it('gets free memory pointer', async function () { + await expect(this.mock.$getFreeMemoryPointer()).to.eventually.equal( + ethers.toBeHex(0x80, 32), // Default pointer + ); + }); }); - it('gets free memory pointer', async function () { - await expect(this.mock.$getFreeMemoryPointer()).to.eventually.equal( - ethers.toBeHex(0x80, 32), // Default pointer - ); + describe('pointer conversions', function () { + it('asBytes32', async function () { + const ptr = ethers.toBeHex('0x1234', 32); + await expect(this.mock.$asBytes32(ptr)).to.eventually.equal(ptr); + }); + + it('asPointer', async function () { + const ptr = ethers.toBeHex('0x1234', 32); + await expect(this.mock.$asPointer(ptr)).to.eventually.equal(ptr); + }); }); }); - describe('pointer conversions', function () { - it('asBytes32', async function () { - const ptr = ethers.toBeHex('0x1234', 32); - await expect(this.mock.$asBytes32(ptr)).to.eventually.equal(ptr); + describe('Slices', function () { + it('asSlice', async function () { + for (const length of [0, 20, 32, 256]) { + const buffer = ethers.getBytes(generators.bytes(length)); + await expect(this.mock.$asSlice(buffer)).to.eventually.equal(formatSlice({ length })); + } + }); + + it('length', async function () { + for (const ptr of ['0x00', '0xa0', '0x42a0']) { + for (const length of [0, 20, 32, 256]) { + await expect(this.mock.$length(formatSlice({ length, ptr }))).to.eventually.equal(length); + } + } + }); + + it('slice(Slice,uint256)', async function () { + await expect( + this.mock.getFunction('$slice(bytes32,uint256)')(formatSlice({ length: 0n, ptr: 256n }), 0n), + ).to.eventually.equal(formatSlice({ length: 0n, ptr: 256n })); + await expect( + this.mock.getFunction('$slice(bytes32,uint256)')(formatSlice({ length: 10n, ptr: 256n }), 0n), + ).to.eventually.equal(formatSlice({ length: 10n, ptr: 256n })); + await expect( + this.mock.getFunction('$slice(bytes32,uint256)')(formatSlice({ length: 10n, ptr: 256n }), 8n), + ).to.eventually.equal(formatSlice({ length: 2n, ptr: 264n })); + await expect( + this.mock.getFunction('$slice(bytes32,uint256)')(formatSlice({ length: 10n, ptr: 256n }), 10n), + ).to.eventually.equal(formatSlice({ length: 0n, ptr: 266n })); + await expect( + this.mock.getFunction('$slice(bytes32,uint256)')(formatSlice({ length: 0n, ptr: 256n }), 1n), + ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS); + await expect( + this.mock.getFunction('$slice(bytes32,uint256)')(formatSlice({ length: 10n, ptr: 256n }), 11n), + ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS); }); - it('asPointer', async function () { - const ptr = ethers.toBeHex('0x1234', 32); - await expect(this.mock.$asPointer(ptr)).to.eventually.equal(ptr); + it('slice(Slice,uint256,uint256)', async function () { + await expect( + this.mock.getFunction('$slice(bytes32,uint256,uint256)')(formatSlice({ length: 0n, ptr: 256n }), 0n, 0n), + ).to.eventually.equal(formatSlice({ length: 0n, ptr: 256n })); + await expect( + this.mock.getFunction('$slice(bytes32,uint256,uint256)')(formatSlice({ length: 10n, ptr: 256n }), 0n, 10n), + ).to.eventually.equal(formatSlice({ length: 10n, ptr: 256n })); + await expect( + this.mock.getFunction('$slice(bytes32,uint256,uint256)')(formatSlice({ length: 10n, ptr: 256n }), 0n, 4n), + ).to.eventually.equal(formatSlice({ length: 4n, ptr: 256n })); + await expect( + this.mock.getFunction('$slice(bytes32,uint256,uint256)')(formatSlice({ length: 10n, ptr: 256n }), 4n, 4n), + ).to.eventually.equal(formatSlice({ length: 4n, ptr: 260n })); + await expect( + this.mock.getFunction('$slice(bytes32,uint256,uint256)')(formatSlice({ length: 0n, ptr: 256n }), 0n, 1n), + ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS); + await expect( + this.mock.getFunction('$slice(bytes32,uint256,uint256)')(formatSlice({ length: 10n, ptr: 256n }), 6n, 6n), + ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS); }); }); }); From 42dd3d32bd63451c8c0e13d1f50da810c4056ade Mon Sep 17 00:00:00 2001 From: ernestognw Date: Fri, 19 Sep 2025 10:59:25 +0200 Subject: [PATCH 113/114] Avoid Accumulators NatSpec to show up on docs sidebar --- contracts/utils/structs/Accumulators.sol | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts/utils/structs/Accumulators.sol b/contracts/utils/structs/Accumulators.sol index 424db6bbee5..db58ba540de 100644 --- a/contracts/utils/structs/Accumulators.sol +++ b/contracts/utils/structs/Accumulators.sol @@ -13,14 +13,14 @@ import {Memory} from "../Memory.sol"; * * Uses 0x00 as sentinel value for empty state (i.e. null pointers) * - * == How it works + * ==== How it works * * 1. Create an empty accumulator with null head/tail pointers * 2. Add data using {push} (append) or {shift} (prepend). It creates linked list nodes * 3. Each node stores a reference to existing data (no copying) * 4. Call {flatten} to materialize the final concatenated result in a single operation * - * == Performance + * ==== Performance * * * Addition: O(1) per operation (just pointer manipulation) * * Flattening: O(n) single pass with one memory allocation From 371595d3959a96a030060dd03ae74e6082d5aedd Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Fri, 19 Sep 2025 12:55:26 +0200 Subject: [PATCH 114/114] Pointer forward --- contracts/utils/Memory.sol | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contracts/utils/Memory.sol b/contracts/utils/Memory.sol index 0614f3182e6..7327c741db0 100644 --- a/contracts/utils/Memory.sol +++ b/contracts/utils/Memory.sol @@ -47,6 +47,11 @@ library Memory { return Pointer.wrap(value); } + /// @dev Move a pointer forward by a given offset. + function forward(Pointer ptr, uint256 offset) internal pure returns (Pointer) { + return Pointer.wrap(bytes32(uint256(Pointer.unwrap(ptr)) + offset)); + } + /// @dev Equality comparator for memory pointers. function equal(Pointer ptr1, Pointer ptr2) internal pure returns (bool) { return Pointer.unwrap(ptr1) == Pointer.unwrap(ptr2); @@ -71,13 +76,13 @@ library Memory { /// @dev Offset a memory slice (equivalent to self[start:] for calldata slices) function slice(Slice self, uint256 offset) internal pure returns (Slice) { if (offset > length(self)) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); - return _asSlice(length(self) - offset, asPointer(bytes32(uint256(asBytes32(_pointer(self))) + offset))); + return _asSlice(length(self) - offset, forward(_pointer(self), offset)); } /// @dev Offset and cut a Slice (equivalent to self[start:start+length] for calldata slices) function slice(Slice self, uint256 offset, uint256 len) internal pure returns (Slice) { if (offset + len > length(self)) Panic.panic(Panic.ARRAY_OUT_OF_BOUNDS); - return _asSlice(len, asPointer(bytes32(uint256(asBytes32(_pointer(self))) + offset))); + return _asSlice(len, forward(_pointer(self), offset)); } /**