Skip to content

utilize pydantic models from eest on t8n #1359

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: forks/osaka
Choose a base branch
from
Open

utilize pydantic models from eest on t8n #1359

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
82b7956
utilize pydantic models from eest on t8n
souradeep-das Aug 1, 2025
1537b01
fix: imports
souradeep-das Aug 1, 2025
1f51eaf
convert alloc directly
souradeep-das Aug 14, 2025
7292acd
rem unnecessary checks
souradeep-das Aug 19, 2025
bb826c6
simplify txs
souradeep-das Aug 19, 2025
dc31f96
add dual input option
souradeep-das Aug 19, 2025
e1dc6a3
update t8n interface
souradeep-das Aug 19, 2025
5a17bd9
review fixes
souradeep-das Aug 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 17 additions & 55 deletions src/ethereum_spec_tools/evm_tools/t8n/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

import argparse
import fnmatch
import json
import os
from functools import partial
from typing import Any, TextIO
Expand All @@ -14,7 +13,10 @@

from ethereum import trace
from ethereum.exceptions import EthereumException, InvalidBlock
from ethereum_clis.types import TransitionToolRequest
from ethereum_spec_tools.forks import Hardfork
from ethereum_clis.types import TransitionToolOutput
from ethereum_clis.clis.execution_specs import ExecutionSpecsExceptionMapper

from ..loaders.fixture_loader import Load
from ..loaders.fork_loader import ForkLoad
Expand Down Expand Up @@ -80,26 +82,18 @@ class T8N(Load):
"""The class that carries out the transition"""

def __init__(
self, options: Any, out_file: TextIO, in_file: TextIO
self, options: Any, out_file: TextIO, in_file: TransitionToolRequest
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this require changes in the daemon?

) -> None:
self.out_file = out_file
self.in_file = in_file
self.options = options
self.forks = Hardfork.discover()

if "stdin" in (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would we still be able to consume json files if needed? We will need to continue supporting that use case.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes! with this commit dc31f96 I added the dual input option where we preserve the older json input route

options.input_env,
options.input_alloc,
options.input_txs,
):
stdin = json.load(in_file)
else:
stdin = None

fork_module, self.fork_block = get_module_name(
self.forks, self.options, stdin
self.forks, self.options, in_file.input.env
)
self.fork = ForkLoad(fork_module)
self.exception_mapper = ExecutionSpecsExceptionMapper()

if self.options.trace:
trace_memory = getattr(self.options, "trace.memory", False)
Expand All @@ -122,9 +116,9 @@ def __init__(
)

self.chain_id = parse_hex_or_int(self.options.state_chainid, U64)
self.alloc = Alloc(self, stdin)
self.env = Env(self, stdin)
self.txs = Txs(self, stdin)
self.alloc = Alloc(self, in_file.input.alloc)
self.env = Env(self, in_file.input.env)
self.txs = Txs(self, in_file.input.txs)
self.result = Result(
self.env.block_difficulty, self.env.base_fee_per_gas
)
Expand Down Expand Up @@ -311,45 +305,13 @@ def run(self) -> int:

json_state = self.alloc.to_json()
json_result = self.result.to_json()

json_output = {}

if self.options.output_body == "stdout":
txs_rlp = "0x" + rlp.encode(self.txs.all_txs).hex()
json_output["body"] = txs_rlp
elif self.options.output_body is not None:
txs_rlp_path = os.path.join(
self.options.output_basedir,
self.options.output_body,
)
txs_rlp = "0x" + rlp.encode(self.txs.all_txs).hex()
with open(txs_rlp_path, "w") as f:
json.dump(txs_rlp, f)
self.logger.info(f"Wrote transaction rlp to {txs_rlp_path}")

if self.options.output_alloc == "stdout":
json_output["alloc"] = json_state
else:
alloc_output_path = os.path.join(
self.options.output_basedir,
self.options.output_alloc,
)
with open(alloc_output_path, "w") as f:
json.dump(json_state, f, indent=4)
self.logger.info(f"Wrote alloc to {alloc_output_path}")

if self.options.output_result == "stdout":
json_output["result"] = json_result
else:
result_output_path = os.path.join(
self.options.output_basedir,
self.options.output_result,
)
with open(result_output_path, "w") as f:
json.dump(json_result, f, indent=4)
self.logger.info(f"Wrote result to {result_output_path}")

if json_output:
json.dump(json_output, self.out_file, indent=4)

return 0
txs_rlp = "0x" + rlp.encode(self.txs.all_txs).hex()
json_output["body"] = txs_rlp
json_output["alloc"] = json_state
json_output["result"] = json_result
output: TransitionToolOutput = TransitionToolOutput.model_validate(
json_output, context={"exception_mapper": self.exception_mapper}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could potentially also be simplified without having to convert json and then json to pydantic

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think now a direct conversion or individually validating each part of TransitionToolOutput wouldn't really add any efficiency gains here

)
return output
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm, so this change would break the command-line output? Is that what we want?

150 changes: 56 additions & 94 deletions src/ethereum_spec_tools/evm_tools/t8n/env.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,7 @@
from ethereum_types.numeric import U64, U256, Uint

from ethereum.crypto.hash import Hash32, keccak256
from ethereum.utils.byte import left_pad_zero_bytes
from ethereum.utils.hexadecimal import hex_to_bytes

from ..utils import parse_hex_or_int
from ethereum_test_types.block_types import Environment

if TYPE_CHECKING:
from ethereum_spec_tools.evm_tools.t8n import T8N
Expand Down Expand Up @@ -55,36 +52,29 @@ class Env:
excess_blob_gas: Optional[U64]
requests: Any

def __init__(self, t8n: "T8N", stdin: Optional[Dict] = None):
if t8n.options.input_env == "stdin":
assert stdin is not None
data = stdin["env"]
else:
with open(t8n.options.input_env, "r") as f:
data = json.load(f)

self.coinbase = t8n.fork.hex_to_address(data["currentCoinbase"])
self.block_gas_limit = parse_hex_or_int(data["currentGasLimit"], Uint)
self.block_number = parse_hex_or_int(data["currentNumber"], Uint)
self.block_timestamp = parse_hex_or_int(data["currentTimestamp"], U256)
def __init__(self, t8n: "T8N", stdin: Environment):
self.coinbase = stdin.fee_recipient
self.block_gas_limit = Uint(stdin.gas_limit)
self.block_number = Uint(stdin.number)
self.block_timestamp = U256(stdin.timestamp)

self.read_block_difficulty(data, t8n)
self.read_base_fee_per_gas(data, t8n)
self.read_randao(data, t8n)
self.read_block_hashes(data, t8n)
self.read_ommers(data, t8n)
self.read_withdrawals(data, t8n)
self.read_block_difficulty(stdin, t8n)
self.read_base_fee_per_gas(stdin, t8n)
self.read_randao(stdin, t8n)
self.read_block_hashes(stdin, t8n)
self.read_ommers(stdin, t8n)
self.read_withdrawals(stdin, t8n)

self.parent_beacon_block_root = None
if t8n.fork.is_after_fork("ethereum.cancun"):
if not t8n.options.state_test:
parent_beacon_block_root_hex = data["parentBeaconBlockRoot"]
parent_beacon_block_root_bytes = stdin.parent_beacon_block_root
self.parent_beacon_block_root = (
Bytes32(hex_to_bytes(parent_beacon_block_root_hex))
if parent_beacon_block_root_hex is not None
Bytes32(parent_beacon_block_root_bytes)
if parent_beacon_block_root_bytes is not None
else None
)
self.read_excess_blob_gas(data, t8n)
self.read_excess_blob_gas(stdin, t8n)

def read_excess_blob_gas(self, data: Any, t8n: "T8N") -> None:
"""
Expand All @@ -98,20 +88,14 @@ def read_excess_blob_gas(self, data: Any, t8n: "T8N") -> None:
if not t8n.fork.is_after_fork("ethereum.cancun"):
return

if "currentExcessBlobGas" in data:
self.excess_blob_gas = parse_hex_or_int(
data["currentExcessBlobGas"], U64
)
if hasattr(data, "excess_blob_gas") and data.excess_blob_gas is not None:
self.excess_blob_gas = U64(data.excess_blob_gas)

if "parentExcessBlobGas" in data:
self.parent_excess_blob_gas = parse_hex_or_int(
data["parentExcessBlobGas"], U64
)
if hasattr(data, "parent_excess_blob_gas") and data.parent_excess_blob_gas is not None:
self.parent_excess_blob_gas = U64(data.parent_excess_blob_gas)

if "parentBlobGasUsed" in data:
self.parent_blob_gas_used = parse_hex_or_int(
data["parentBlobGasUsed"], U64
)
if hasattr(data, "parent_blob_gas_used") and data.parent_blob_gas_used is not None:
self.parent_blob_gas_used = U64(data.parent_blob_gas_used)

if self.excess_blob_gas is not None:
return
Expand Down Expand Up @@ -170,25 +154,17 @@ def read_base_fee_per_gas(self, data: Any, t8n: "T8N") -> None:
self.base_fee_per_gas = None

if t8n.fork.is_after_fork("ethereum.london"):
if "currentBaseFee" in data:
self.base_fee_per_gas = parse_hex_or_int(
data["currentBaseFee"], Uint
)
if hasattr(data, "base_fee_per_gas") and data.base_fee_per_gas is not None:
self.base_fee_per_gas = Uint(data.base_fee_per_gas)

if "parentGasUsed" in data:
self.parent_gas_used = parse_hex_or_int(
data["parentGasUsed"], Uint
)
if hasattr(data, "parent_gas_used") and data.parent_gas_used is not None:
self.parent_gas_used = Uint(data.parent_gas_used)

if "parentGasLimit" in data:
self.parent_gas_limit = parse_hex_or_int(
data["parentGasLimit"], Uint
)
if hasattr(data, "parent_gas_limit") and data.parent_gas_limit is not None:
self.parent_gas_limit = Uint(data.parent_gas_limit)

if "parentBaseFee" in data:
self.parent_base_fee_per_gas = parse_hex_or_int(
data["parentBaseFee"], Uint
)
if hasattr(data, "parent_base_fee_per_gas") and data.parent_base_fee_per_gas is not None:
self.parent_base_fee_per_gas = Uint(data.parent_base_fee_per_gas)

if self.base_fee_per_gas is None:
assert self.parent_gas_limit is not None
Expand All @@ -212,30 +188,26 @@ def read_randao(self, data: Any, t8n: "T8N") -> None:
"""
self.prev_randao = None
if t8n.fork.is_after_fork("ethereum.paris"):
# tf tool might not always provide an
# even number of nibbles in the randao
# This could create issues in the
# hex_to_bytes function
current_random = data["currentRandom"]
if current_random.startswith("0x"):
current_random = current_random[2:]

if len(current_random) % 2 == 1:
current_random = "0" + current_random

self.prev_randao = Bytes32(
left_pad_zero_bytes(hex_to_bytes(current_random), 32)
)
self.prev_randao = Bytes32(data.prev_randao.to_bytes(32, "big"))

def read_withdrawals(self, data: Any, t8n: "T8N") -> None:
"""
Read the withdrawals from the data.
"""
self.withdrawals = None
if t8n.fork.is_after_fork("ethereum.shanghai"):
self.withdrawals = tuple(
t8n.json_to_withdrawals(wd) for wd in data["withdrawals"]
)
raw_withdrawals = getattr(data, "withdrawals", None)
if raw_withdrawals:
def to_canonical_withdrawal(raw):
return t8n.fork.Withdrawal(
index=U64(raw.index),
validator_index=U64(raw.validator_index),
address=raw.address,
amount=U256(raw.amount),
)
Comment on lines +392 to +398
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should probably hoist this out of read_withdrawals to avoid creating a bunch of closures that aren't used.

self.withdrawals = tuple(to_canonical_withdrawal(wd) for wd in raw_withdrawals)
else:
self.withdrawals = ()

def read_block_difficulty(self, data: Any, t8n: "T8N") -> None:
"""
Expand All @@ -249,28 +221,22 @@ def read_block_difficulty(self, data: Any, t8n: "T8N") -> None:
self.parent_ommers_hash = None
if t8n.fork.is_after_fork("ethereum.paris"):
return
elif "currentDifficulty" in data:
self.block_difficulty = parse_hex_or_int(
data["currentDifficulty"], Uint
)
elif hasattr(data, "difficulty") and data.difficulty is not None:
self.block_difficulty = Uint(data.difficulty)
else:
self.parent_timestamp = parse_hex_or_int(
data["parentTimestamp"], U256
)
self.parent_difficulty = parse_hex_or_int(
data["parentDifficulty"], Uint
)
self.parent_timestamp = U256(data.parent_timestamp)
self.parent_difficulty = Uint(data.parent_difficulty)
args: List[object] = [
self.block_number,
self.block_timestamp,
self.parent_timestamp,
self.parent_difficulty,
]
if t8n.fork.is_after_fork("ethereum.byzantium"):
if "parentUncleHash" in data:
if hasattr(data, "parent_ommers_hash") and data.parent_ommers_hash is not None:
EMPTY_OMMER_HASH = keccak256(rlp.encode([]))
self.parent_ommers_hash = Hash32(
hex_to_bytes(data["parentUncleHash"])
data.parent_ommers_hash
)
parent_has_ommers = (
self.parent_ommers_hash != EMPTY_OMMER_HASH
Expand All @@ -289,25 +255,21 @@ def read_block_hashes(self, data: Any, t8n: "T8N") -> None:
t8n.fork.is_after_fork("ethereum.prague")
and not t8n.options.state_test
):
self.parent_hash = Hash32(hex_to_bytes(data["parentHash"]))
self.parent_hash = Hash32(data.parent_hash)

# Read the block hashes
block_hashes: List[Any] = []

# The hex key strings provided might not have standard formatting
clean_block_hashes: Dict[int, Hash32] = {}
if "blockHashes" in data:
for key, value in data["blockHashes"].items():
int_key = int(key, 16)
clean_block_hashes[int_key] = Hash32(hex_to_bytes(value))
clean_block_hashes: Dict[int, Hash32] = data.block_hashes

# Store a maximum of 256 block hashes.
max_blockhash_count = min(Uint(256), self.block_number)
for number in range(
self.block_number - max_blockhash_count, self.block_number
):
if number in clean_block_hashes.keys():
block_hashes.append(clean_block_hashes[number])
block_hashes.append(Hash32(clean_block_hashes[number]))
else:
block_hashes.append(None)

Expand All @@ -319,12 +281,12 @@ def read_ommers(self, data: Any, t8n: "T8N") -> None:
needed to obtain the Header.
"""
ommers = []
if "ommers" in data:
for ommer in data["ommers"]:
if hasattr(data, "ommers") and data.ommers is not None:
for ommer in data.ommers:
ommers.append(
Ommer(
ommer["delta"],
t8n.fork.hex_to_address(ommer["address"]),
ommer.delta,
ommer.address,
)
)
self.ommers = ommers
Loading