diff --git a/.github/workflows/on-master.yaml b/.github/workflows/on-master.yaml index 1e0a2bd699d3..df521cdb03e9 100644 --- a/.github/workflows/on-master.yaml +++ b/.github/workflows/on-master.yaml @@ -1,11 +1,13 @@ name: Docker Build and publish to Github on: + workflow_dispatch: push: branches: - statediff - statediff-v2 - statediff-v3 + - v1.10.17-statediff-v3 - v1.10.16-statediff-v3 - v1.10.15-statediff-v3 - v1.10.15-statediff-v2 diff --git a/.gitignore b/.gitignore index ea78e6c56b88..78894cb66ecd 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,13 @@ profile.cov **/yarn-error.log foundry/deployments/local-private-network/geth-linux-amd64 foundry/projects/local-private-network/geth-linux-amd64 + +# Helpful repos +related-repositories/foundry-test/** +related-repositories/hive/** +related-repositories/ipld-eth-db/** +statediff/indexer/database/sql/statediffing_test_file.sql +statediff/statediffing_test_file.sql +statediff/known_gaps.sql +related-repositories/foundry-test/ +related-repositories/ipld-eth-db/ diff --git a/.gitmodules b/.gitmodules index 5723b475803f..2171d3b2c097 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,4 @@ [submodule "evm-benchmarks"] path = tests/evm-benchmarks url = https://github.com/ipsilon/evm-benchmarks - shallow = true -[submodule "foundry/projects/local-private-network/Stateful/lib/ds-test"] - path = foundry/projects/local-private-network/Stateful/lib/ds-test - url = https://github.com/dapphub/ds-test + shallow = true \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..e08e271f3f12 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,247 @@ +language: go +go_import_path: github.com/ethereum/go-ethereum +sudo: false +jobs: + allow_failures: + - stage: build + os: osx + go: 1.17.x + env: + - azure-osx + - azure-ios + - cocoapods-ios + + include: + # This builder only tests code linters on latest version of Go + - stage: lint + os: linux + dist: bionic + go: 1.18.x + env: + - lint + git: + submodules: false # avoid cloning ethereum/tests + script: + - go run build/ci.go lint + + # These builders create the Docker sub-images for multi-arch push and each + # will attempt to push the multi-arch image if they are the last builder + - stage: build + if: type = push + os: linux + arch: amd64 + dist: bionic + go: 1.18.x + env: + - docker + services: + - docker + git: + submodules: false # avoid cloning ethereum/tests + before_install: + - export DOCKER_CLI_EXPERIMENTAL=enabled + script: + - go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go + + - stage: build + if: type = push + os: linux + arch: arm64 + dist: bionic + go: 1.18.x + env: + - docker + services: + - docker + git: + submodules: false # avoid cloning ethereum/tests + before_install: + - export DOCKER_CLI_EXPERIMENTAL=enabled + script: + - go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go + + # This builder does the Ubuntu PPA upload + - stage: build + if: type = push + os: linux + dist: bionic + go: 1.18.x + env: + - ubuntu-ppa + - GO111MODULE=on + git: + submodules: false # avoid cloning ethereum/tests + addons: + apt: + packages: + - devscripts + - debhelper + - dput + - fakeroot + - python-bzrlib + - python-paramiko + script: + - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts + - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " + + # This builder does the Linux Azure uploads + - stage: build + if: type = push + os: linux + dist: bionic + sudo: required + go: 1.18.x + env: + - azure-linux + - GO111MODULE=on + git: + submodules: false # avoid cloning ethereum/tests + addons: + apt: + packages: + - gcc-multilib + script: + # Build for the primary platforms that Trusty can manage + - go run build/ci.go install -dlgo + - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + - go run build/ci.go install -dlgo -arch 386 + - go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + + # Switch over GCC to cross compilation (breaks 386, hence why do it here only) + - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + - sudo ln -s /usr/include/asm-generic /usr/include/asm + + - GOARM=5 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc + - GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + - GOARM=6 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc + - GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + - GOARM=7 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabihf-gcc + - GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc + - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + + # This builder does the Android Maven and Azure uploads + - stage: build + if: type = push + os: linux + dist: bionic + addons: + apt: + packages: + - openjdk-8-jdk + env: + - azure-android + - maven-android + - GO111MODULE=on + git: + submodules: false # avoid cloning ethereum/tests + before_install: + # Install Android and it's dependencies manually, Travis is stale + - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + - curl https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip -o android.zip + - unzip -q android.zip -d $HOME/sdk && rm android.zip + - mv $HOME/sdk/cmdline-tools $HOME/sdk/latest && mkdir $HOME/sdk/cmdline-tools && mv $HOME/sdk/latest $HOME/sdk/cmdline-tools + - export PATH=$PATH:$HOME/sdk/cmdline-tools/latest/bin + - export ANDROID_HOME=$HOME/sdk + + - yes | sdkmanager --licenses >/dev/null + - sdkmanager "platform-tools" "platforms;android-15" "platforms;android-19" "platforms;android-24" "ndk-bundle" + + # Install Go to allow building with + - curl https://dl.google.com/go/go1.18.linux-amd64.tar.gz | tar -xz + - export PATH=`pwd`/go/bin:$PATH + - export GOROOT=`pwd`/go + - export GOPATH=$HOME/go + script: + # Build the Android archive and upload it to Maven Central and Azure + - mkdir -p $GOPATH/src/github.com/ethereum + - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum + - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -signify SIGNIFY_KEY -deploy https://oss.sonatype.org -upload gethstore/builds + + # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads + - stage: build + if: type = push + os: osx + go: 1.18.x + env: + - azure-osx + - azure-ios + - cocoapods-ios + - GO111MODULE=on + git: + submodules: false # avoid cloning ethereum/tests + script: + - go run build/ci.go install -dlgo + - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + + # Build the iOS framework and upload it to CocoaPods and Azure + - gem uninstall cocoapods -a -x + - gem install cocoapods + + - mv ~/.cocoapods/repos/master ~/.cocoapods/repos/master.bak + - sed -i '.bak' 's/repo.join/!repo.join/g' $(dirname `gem which cocoapods`)/cocoapods/sources_manager.rb + - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then git clone --depth=1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master && pod setup --verbose; fi + + - xctool -version + - xcrun simctl list + + # Workaround for https://github.com/golang/go/issues/23749 + - export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc' + - go run build/ci.go xcode -signer IOS_SIGNING_KEY -signify SIGNIFY_KEY -deploy trunk -upload gethstore/builds + + # These builders run the tests + - stage: build + os: linux + arch: amd64 + dist: bionic + go: 1.18.x + env: + - GO111MODULE=on + script: + - go run build/ci.go test -coverage $TEST_PACKAGES + + - stage: build + if: type = pull_request + os: linux + arch: arm64 + dist: bionic + go: 1.18.x + env: + - GO111MODULE=on + script: + - go run build/ci.go test -coverage $TEST_PACKAGES + + - stage: build + os: linux + dist: bionic + go: 1.17.x + env: + - GO111MODULE=on + script: + - go run build/ci.go test -coverage $TEST_PACKAGES + + # This builder does the Azure archive purges to avoid accumulating junk + - stage: build + if: type = cron + os: linux + dist: bionic + go: 1.18.x + env: + - azure-purge + - GO111MODULE=on + git: + submodules: false # avoid cloning ethereum/tests + script: + - go run build/ci.go purge -store gethstore/builds -days 14 + + # This builder executes race tests + - stage: build + if: type = cron + os: linux + dist: bionic + go: 1.18.x + env: + - GO111MODULE=on + script: + - go run build/ci.go test -race -coverage $TEST_PACKAGES + diff --git a/Dockerfile b/Dockerfile index 7badbc1320a4..ec46f60773e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.17-alpine as builder +FROM golang:1.18-alpine as builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 3ae5377e4f36..683f87a55e7e 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.17-alpine as builder +FROM golang:1.18-alpine as builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/README.md b/README.md index 81b7215ba85d..cddc619a20cb 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,22 @@ Going through all the possible command line flags is out of scope here (please c but we've enumerated a few common parameter combos to get you up to speed quickly on how you can run your own `geth` instance. +### Hardware Requirements + +Minimum: + +* CPU with 2+ cores +* 4GB RAM +* 500GB free storage space to sync the Mainnet +* 8 MBit/sec download Internet service + +Recommended: + +* Fast CPU with 4+ cores +* 16GB+ RAM +* High Performance SSD with at least 500GB free space +* 25+ MBit/sec download Internet service + ### Full node on the main Ethereum network By far the most common scenario is people wanting to simply interact with the Ethereum diff --git a/accounts/abi/selector_parser.go b/accounts/abi/selector_parser.go new file mode 100644 index 000000000000..75609b28a6cc --- /dev/null +++ b/accounts/abi/selector_parser.go @@ -0,0 +1,152 @@ +package abi + +import ( + "fmt" +) + +type SelectorMarshaling struct { + Name string `json:"name"` + Type string `json:"type"` + Inputs []ArgumentMarshaling `json:"inputs"` +} + +func isDigit(c byte) bool { + return c >= '0' && c <= '9' +} + +func isAlpha(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +func isIdentifierSymbol(c byte) bool { + return c == '$' || c == '_' +} + +func parseToken(unescapedSelector string, isIdent bool) (string, string, error) { + if len(unescapedSelector) == 0 { + return "", "", fmt.Errorf("empty token") + } + firstChar := unescapedSelector[0] + position := 1 + if !(isAlpha(firstChar) || (isIdent && isIdentifierSymbol(firstChar))) { + return "", "", fmt.Errorf("invalid token start: %c", firstChar) + } + for position < len(unescapedSelector) { + char := unescapedSelector[position] + if !(isAlpha(char) || isDigit(char) || (isIdent && isIdentifierSymbol(char))) { + break + } + position++ + } + return unescapedSelector[:position], unescapedSelector[position:], nil +} + +func parseIdentifier(unescapedSelector string) (string, string, error) { + return parseToken(unescapedSelector, true) +} + +func parseElementaryType(unescapedSelector string) (string, string, error) { + parsedType, rest, err := parseToken(unescapedSelector, false) + if err != nil { + return "", "", fmt.Errorf("failed to parse elementary type: %v", err) + } + // handle arrays + for len(rest) > 0 && rest[0] == '[' { + parsedType = parsedType + string(rest[0]) + rest = rest[1:] + for len(rest) > 0 && isDigit(rest[0]) { + parsedType = parsedType + string(rest[0]) + rest = rest[1:] + } + if len(rest) == 0 || rest[0] != ']' { + return "", "", fmt.Errorf("failed to parse array: expected ']', got %c", unescapedSelector[0]) + } + parsedType = parsedType + string(rest[0]) + rest = rest[1:] + } + return parsedType, rest, nil +} + +func parseCompositeType(unescapedSelector string) ([]interface{}, string, error) { + if len(unescapedSelector) == 0 || unescapedSelector[0] != '(' { + return nil, "", fmt.Errorf("expected '(', got %c", unescapedSelector[0]) + } + parsedType, rest, err := parseType(unescapedSelector[1:]) + if err != nil { + return nil, "", fmt.Errorf("failed to parse type: %v", err) + } + result := []interface{}{parsedType} + for len(rest) > 0 && rest[0] != ')' { + parsedType, rest, err = parseType(rest[1:]) + if err != nil { + return nil, "", fmt.Errorf("failed to parse type: %v", err) + } + result = append(result, parsedType) + } + if len(rest) == 0 || rest[0] != ')' { + return nil, "", fmt.Errorf("expected ')', got '%s'", rest) + } + return result, rest[1:], nil +} + +func parseType(unescapedSelector string) (interface{}, string, error) { + if len(unescapedSelector) == 0 { + return nil, "", fmt.Errorf("empty type") + } + if unescapedSelector[0] == '(' { + return parseCompositeType(unescapedSelector) + } else { + return parseElementaryType(unescapedSelector) + } +} + +func assembleArgs(args []interface{}) ([]ArgumentMarshaling, error) { + arguments := make([]ArgumentMarshaling, 0) + for i, arg := range args { + // generate dummy name to avoid unmarshal issues + name := fmt.Sprintf("name%d", i) + if s, ok := arg.(string); ok { + arguments = append(arguments, ArgumentMarshaling{name, s, s, nil, false}) + } else if components, ok := arg.([]interface{}); ok { + subArgs, err := assembleArgs(components) + if err != nil { + return nil, fmt.Errorf("failed to assemble components: %v", err) + } + arguments = append(arguments, ArgumentMarshaling{name, "tuple", "tuple", subArgs, false}) + } else { + return nil, fmt.Errorf("failed to assemble args: unexpected type %T", arg) + } + } + return arguments, nil +} + +// ParseSelector converts a method selector into a struct that can be JSON encoded +// and consumed by other functions in this package. +// Note, although uppercase letters are not part of the ABI spec, this function +// still accepts it as the general format is valid. +func ParseSelector(unescapedSelector string) (SelectorMarshaling, error) { + name, rest, err := parseIdentifier(unescapedSelector) + if err != nil { + return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': %v", unescapedSelector, err) + } + args := []interface{}{} + if len(rest) >= 2 && rest[0] == '(' && rest[1] == ')' { + rest = rest[2:] + } else { + args, rest, err = parseCompositeType(rest) + if err != nil { + return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': %v", unescapedSelector, err) + } + } + if len(rest) > 0 { + return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': unexpected string '%s'", unescapedSelector, rest) + } + + // Reassemble the fake ABI and constuct the JSON + fakeArgs, err := assembleArgs(args) + if err != nil { + return SelectorMarshaling{}, fmt.Errorf("failed to parse selector: %v", err) + } + + return SelectorMarshaling{name, "function", fakeArgs}, nil +} diff --git a/accounts/abi/selector_parser_test.go b/accounts/abi/selector_parser_test.go new file mode 100644 index 000000000000..9720c9d5308e --- /dev/null +++ b/accounts/abi/selector_parser_test.go @@ -0,0 +1,54 @@ +package abi + +import ( + "fmt" + "log" + "reflect" + "testing" +) + +func TestParseSelector(t *testing.T) { + mkType := func(types ...interface{}) []ArgumentMarshaling { + var result []ArgumentMarshaling + for i, typeOrComponents := range types { + name := fmt.Sprintf("name%d", i) + if typeName, ok := typeOrComponents.(string); ok { + result = append(result, ArgumentMarshaling{name, typeName, typeName, nil, false}) + } else if components, ok := typeOrComponents.([]ArgumentMarshaling); ok { + result = append(result, ArgumentMarshaling{name, "tuple", "tuple", components, false}) + } else { + log.Fatalf("unexpected type %T", typeOrComponents) + } + } + return result + } + tests := []struct { + input string + name string + args []ArgumentMarshaling + }{ + {"noargs()", "noargs", []ArgumentMarshaling{}}, + {"simple(uint256,uint256,uint256)", "simple", mkType("uint256", "uint256", "uint256")}, + {"other(uint256,address)", "other", mkType("uint256", "address")}, + {"withArray(uint256[],address[2],uint8[4][][5])", "withArray", mkType("uint256[]", "address[2]", "uint8[4][][5]")}, + {"singleNest(bytes32,uint8,(uint256,uint256),address)", "singleNest", mkType("bytes32", "uint8", mkType("uint256", "uint256"), "address")}, + {"multiNest(address,(uint256[],uint256),((address,bytes32),uint256))", "multiNest", + mkType("address", mkType("uint256[]", "uint256"), mkType(mkType("address", "bytes32"), "uint256"))}, + } + for i, tt := range tests { + selector, err := ParseSelector(tt.input) + if err != nil { + t.Errorf("test %d: failed to parse selector '%v': %v", i, tt.input, err) + } + if selector.Name != tt.name { + t.Errorf("test %d: unexpected function name: '%s' != '%s'", i, selector.Name, tt.name) + } + + if selector.Type != "function" { + t.Errorf("test %d: unexpected type: '%s' != '%s'", i, selector.Type, "function") + } + if !reflect.DeepEqual(selector.Inputs, tt.args) { + t.Errorf("test %d: unexpected args: '%v' != '%v'", i, selector.Inputs, tt.args) + } + } +} diff --git a/appveyor.yml b/appveyor.yml index 65b5f96841e2..d477e6db9f56 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,7 +13,7 @@ environment: GETH_MINGW: 'C:\msys64\mingw32' install: - - git submodule update --init --depth 1 + - git submodule update --init --depth 1 --recursive - go version for: diff --git a/build/checksums.txt b/build/checksums.txt index 5df27bbf6173..9d83c9ebba9f 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,37 +1,58 @@ # This file contains sha256 checksums of optional build dependencies. -3defb9a09bed042403195e872dcbc8c6fae1485963332279668ec52e80a95a2d go1.17.5.src.tar.gz -2db6a5d25815b56072465a2cacc8ed426c18f1d5fc26c1fc8c4f5a7188658264 go1.17.5.darwin-amd64.tar.gz -111f71166de0cb8089bb3e8f9f5b02d76e1bf1309256824d4062a47b0e5f98e0 go1.17.5.darwin-arm64.tar.gz -443c1cd9768df02085014f1eb034ebc7dbe032ffc8a9bb9f2e6617d037eee23c go1.17.5.freebsd-386.tar.gz -17180bdc4126acffd0ebf86d66ef5cbc3488b6734e93374fb00eb09494e006d3 go1.17.5.freebsd-amd64.tar.gz -4f4914303bc18f24fd137a97e595735308f5ce81323c7224c12466fd763fc59f go1.17.5.linux-386.tar.gz -bd78114b0d441b029c8fe0341f4910370925a4d270a6a590668840675b0c653e go1.17.5.linux-amd64.tar.gz -6f95ce3da40d9ce1355e48f31f4eb6508382415ca4d7413b1e7a3314e6430e7e go1.17.5.linux-arm64.tar.gz -aa1fb6c53b4fe72f159333362a10aca37ae938bde8adc9c6eaf2a8e87d1e47de go1.17.5.linux-armv6l.tar.gz -3d4be616e568f0a02cb7f7769bcaafda4b0969ed0f9bb4277619930b96847e70 go1.17.5.linux-ppc64le.tar.gz -8087d4fe991e82804e6485c26568c2e0ee0bfde00ceb9015dc86cb6bf84ef40b go1.17.5.linux-s390x.tar.gz -6d7b9948ee14a906b14f5cbebdfab63cd6828b0b618160847ecd3cc3470a26fe go1.17.5.windows-386.zip -671faf99cd5d81cd7e40936c0a94363c64d654faa0148d2af4bbc262555620b9 go1.17.5.windows-amd64.zip -45e88676b68e9cf364be469b5a27965397f4e339aa622c2f52c10433c56e5030 go1.17.5.windows-arm64.zip +38f423db4cc834883f2b52344282fa7a39fbb93650dc62a11fdf0be6409bdad6 go1.18.src.tar.gz +70bb4a066997535e346c8bfa3e0dfe250d61100b17ccc5676274642447834969 go1.18.darwin-amd64.tar.gz +9cab6123af9ffade905525d79fc9ee76651e716c85f1f215872b5f2976782480 go1.18.darwin-arm64.tar.gz +e63492d4f38487331518eb4b50e670d853bb8d67e88596269af84bb9aca0b381 go1.18.freebsd-386.tar.gz +01cd67bbc12e659ff236ecebde1806f76452f7ca145c172d5ecdbf4f4803daae go1.18.freebsd-amd64.tar.gz +1c04cf4440b323a66328e0df95d409f955b9b475e58eae235fdd3d1f1cf02f4f go1.18.linux-386.tar.gz +e85278e98f57cdb150fe8409e6e5df5343ecb13cebf03a5d5ff12bd55a80264f go1.18.linux-amd64.tar.gz +7ac7b396a691e588c5fb57687759e6c4db84a2a3bbebb0765f4b38e5b1c5b00e go1.18.linux-arm64.tar.gz +a80fa43d1f4575fb030adbfbaa94acd860c6847820764eecb06c63b7c103612b go1.18.linux-armv6l.tar.gz +070351edac192483c074b38d08ec19251a83f8210765a532a84c3dcf8aec04d8 go1.18.linux-ppc64le.tar.gz +ea265f5e62fcaf941d53f0cdb81222d9668e1672a0d39d992f16ff0e87c0ee6b go1.18.linux-s390x.tar.gz +e23fd2a0509690fe7e63b2b1bcd4c39ed57b46ccde76f35dc0d16ca7fdbc5aaa go1.18.windows-386.zip +65c5c0c709a7ca1b357091b10b795b439d8b50e579d3893edab4c7e9b384f435 go1.18.windows-amd64.zip +1c454eb60c64d481965a165c623ff1ed6cf32d68c6b31f36069c8768d908f093 go1.18.windows-arm64.zip -d4bd25b9814eeaa2134197dd2c7671bb791eae786d42010d9d788af20dee4bfa golangci-lint-1.42.0-darwin-amd64.tar.gz -e56859c04a2ad5390c6a497b1acb1cc9329ecb1010260c6faae9b5a4c35b35ea golangci-lint-1.42.0-darwin-arm64.tar.gz -14d912a3fa856830339472fc4dc341933adf15f37bdb7130bbbfcf960ecf4809 golangci-lint-1.42.0-freebsd-386.tar.gz -337257fccc9baeb5ee1cd7e70c153e9d9f59d3afde46d631659500048afbdf80 golangci-lint-1.42.0-freebsd-amd64.tar.gz -6debcc266b629359fdd8eef4f4abb05a621604079d27016265afb5b4593b0eff golangci-lint-1.42.0-freebsd-armv6.tar.gz -878f0e190169db2ce9dde8cefbd99adc4fe28b90b68686bbfcfcc2085e6d693e golangci-lint-1.42.0-freebsd-armv7.tar.gz -42c78e31faf62b225363eff1b1d2aa74f9dbcb75686c8914aa3e90d6af65cece golangci-lint-1.42.0-linux-386.tar.gz -6937f62f8e2329e94822dc11c10b871ace5557ae1fcc4ee2f9980cd6aecbc159 golangci-lint-1.42.0-linux-amd64.tar.gz -2cf8d23d96cd854a537b355dab2962b960b88a06b615232599f066afd233f246 golangci-lint-1.42.0-linux-arm64.tar.gz -08b003d1ed61367473886defc957af5301066e62338e5d96a319c34dadc4c1d1 golangci-lint-1.42.0-linux-armv6.tar.gz -c7c00ec4845e806a1f32685f5b150219e180bd6d6a9d584be8d27f0c41d7a1bf golangci-lint-1.42.0-linux-armv7.tar.gz -3650fcf29eb3d8ee326d77791a896b15259eb2d5bf77437dc72e7efe5af6bd40 golangci-lint-1.42.0-linux-mips64.tar.gz -f51ae003fdbca4fef78ba73e2eb736a939c8eaa178cd452234213b489da5a420 golangci-lint-1.42.0-linux-mips64le.tar.gz -1b0bb7b8b22cc4ea7da44fd5ad5faaf6111d0677e01cc6f961b62a96537de2c6 golangci-lint-1.42.0-linux-ppc64le.tar.gz -8cb56927eb75e572450efbe0ff0f9cf3f56dc9faa81d9e8d30d6559fc1d06e6d golangci-lint-1.42.0-linux-riscv64.tar.gz -5ac41cd31825a176b21505a371a7b307cd9cdf17df0f35bbb3bf1466f9356ccc golangci-lint-1.42.0-linux-s390x.tar.gz -e1cebd2af621ac4b64c20937df92c3819264f2174c92f51e196db1e64ae097e0 golangci-lint-1.42.0-windows-386.zip -7e70fcde8e87a17cae0455df07d257ebc86669f3968d568e12727fa24bbe9883 golangci-lint-1.42.0-windows-amd64.zip -59da7ce1bda432616bfc28ae663e52c3675adee8d9bf5959fafd657c159576ab golangci-lint-1.42.0-windows-armv6.zip -65f62dda937bfcede0326ac77abe947ce1548931e6e13298ca036cb31f224db5 golangci-lint-1.42.0-windows-armv7.zip +03c181fc1bb29ea3e73cbb23399c43b081063833a7cf7554b94e5a98308df53e golangci-lint-1.45.2-linux-riscv64.deb +08a50bbbf451ede6d5354179eb3e14a5634e156dfa92cb9a2606f855a637e35b golangci-lint-1.45.2-linux-ppc64le.rpm +0d12f6ec1296b5a70e392aa88cd2295cceef266165eb7028e675f455515dd1c9 golangci-lint-1.45.2-linux-armv7.deb +10f2846e2e50e4ea8ae426ee62dcd2227b23adddd8e991aa3c065927ac948735 golangci-lint-1.45.2-linux-ppc64le.deb +1463049b744871168095e3e8f687247d6040eeb895955b869889ea151e0603ab golangci-lint-1.45.2-linux-arm64.tar.gz +15720f9c4c6f9324af695f081dc189adc7751b255759e78d7b2df1d7e9192533 golangci-lint-1.45.2-linux-amd64.deb +166d922e4d3cfe3d47786c590154a9c8ea689dff0aa92b73d2f5fc74fc570c29 golangci-lint-1.45.2-linux-arm64.rpm +1a3754c69f7cc19ab89cbdcc2550da4cf9abb3120383c6b3bd440c1ec22da2e6 golangci-lint-1.45.2-freebsd-386.tar.gz +1dec0aa46d4f0d241863b573f70129bdf1de9c595cf51172a840a588a4cd9fc5 golangci-lint-1.45.2-windows-amd64.zip +3198453806517c1ad988229f5e758ef850e671203f46d6905509df5bdf4dc24b golangci-lint-1.45.2-freebsd-armv7.tar.gz +46a3cd1749d7b98adc2dc01510ddbe21abe42689c8a53fb0e81662713629f215 golangci-lint-1.45.2-linux-386.deb +4e28bfb593d464b9e160f2acd5b71993836a183270bf8299b78ad31f7a168c0d golangci-lint-1.45.2-linux-arm64.deb +5157a58c8f9ab85c33af2e46f0d7c57a3b1e8953b81d61130e292e09f545cfab golangci-lint-1.45.2-linux-mips64le.tar.gz +518cd027644129fbf8ec4f02bd6f9ad7278aae826f92b63c80d4d0819ddde49a golangci-lint-1.45.2-linux-armv6.rpm +595ad6c6dade4c064351bc309f411703e457f8ffbb7a1806b3d8ee713333427f golangci-lint-1.45.2-linux-amd64.tar.gz +6994d6c80f0730751090986184a3481b4be2e6b6e84416238a2b857910045a4f golangci-lint-1.45.2-windows-arm64.zip +6c81652fc340118811b487f713c441fc6f527800bf5fd11b8929d08124efa015 golangci-lint-1.45.2-linux-armv7.tar.gz +726cb045559b7518bafdd3459de70a0647c087eb1b4634627a4b2e95b1258580 golangci-lint-1.45.2-freebsd-amd64.tar.gz +77df3774cdfda49b956d4a0e676da9a9b883f496ee37293c530770fef6b1d24e golangci-lint-1.45.2-linux-mips64.deb +7a9840f279a7d5d405bb434e101c2290964b3729630ac2add29280b962b7b9a5 golangci-lint-1.45.2-windows-armv6.zip +7d4bf9a5d80ec467aaaf66e78dbdcab567bbc6ba8151334c714eee58766aae32 golangci-lint-1.45.2-windows-armv7.zip +7e5f8821d39bb11d273b0841b34355f56bd5a45a2d5179f0d09e614e0efc0482 golangci-lint-1.45.2-linux-s390x.rpm +828de1bde796b23d8656b17a8885fbd879ef612795d62d1e4618126b419728b5 golangci-lint-1.45.2-linux-mips64.rpm +879a52107a797678a03c175cc7cf441411a14a01f66dc87f70bdfa304a4129a6 golangci-lint-1.45.2-windows-386.zip +87b6c7e3a3769f7d9abeb3bb82119b3c91e3c975300f6834fdeef8b2e37c98ff golangci-lint-1.45.2-linux-amd64.rpm +8b605c6d686c8af53ecc4ef39544541eeb1644d34cc10f9ffc5087808210c4ff golangci-lint-1.45.2-linux-s390x.deb +9427dbf51d0ac6f73a0f992838bd40c817470cc5bf6c8e2e2bea6fac46d7af6e golangci-lint-1.45.2-linux-ppc64le.tar.gz +995e509e895ca6a64ffc7395ac884d5961bdec98423cb896b17f345a9b4a19cf golangci-lint-1.45.2-darwin-amd64.tar.gz +a3f36278f2ea5516341e9071a2df6e65df272be80230b5406a12b72c6d425bee golangci-lint-1.45.2-linux-armv7.rpm +a5e12c50c23e87ac1deffc872f92ae85427b1198604969399805ae47cfe43f08 golangci-lint-1.45.2-linux-riscv64.tar.gz +aa8fa1be0729dbc2fbc4e01e82027097613eee74bd686ebef20f860b01fff8b3 golangci-lint-1.45.2-freebsd-armv6.tar.gz +c2b9669decc1b638cf2ee9060571af4e255f6dfcbb225c293e3a7ee4bb2c7217 golangci-lint-1.45.2-darwin-arm64.tar.gz +dfa8bdaf0387aec1cd5c1aa8857f67b2bbdfc2e42efce540c8fb9bbe3e8af302 golangci-lint-1.45.2-linux-armv6.tar.gz +eb8b8539dd017eee5c131ea9b875893ab2cebeeca41e8c6624907fb02224d643 golangci-lint-1.45.2-linux-386.rpm +ed6c7e17a857f30d715c5302fa250d95936936b277024bffea201187a257d7a7 golangci-lint-1.45.2-linux-armv6.deb +ef4d0154ace4001f01b288baeb118176242efb4fd163e178763e3213b77ef30b golangci-lint-1.45.2-linux-mips64le.deb +ef7002a2229f5ff5ba201a715fcf877664ea88decbe58e69d163293913024955 golangci-lint-1.45.2-linux-s390x.tar.gz +f13ecbd09228632e6bbe91a8324bd675c406eed22eb6d2c1e8192eed9ec4f914 golangci-lint-1.45.2-linux-386.tar.gz +f4cd9cfb09252f51699407277512263cae8409b665dd764f55a34738d0e89edc golangci-lint-1.45.2-linux-riscv64.rpm +fb1945dc59d37c9d14bf0a4aea11ea8651fa0e1d582ea80c4c44d0a536c08893 golangci-lint-1.45.2-linux-mips64.tar.gz +fe542c22738010f453c735a3c410decfd3784d1bd394b395c298ee298fc4c606 golangci-lint-1.45.2-linux-mips64le.rpm \ No newline at end of file diff --git a/build/ci.go b/build/ci.go index 8a5f78cf9944..4b941cd62d7d 100644 --- a/build/ci.go +++ b/build/ci.go @@ -131,13 +131,14 @@ var ( // Distros for which packages are created. // Note: vivid is unsupported because there is no golang-1.6 package for it. // Note: the following Ubuntu releases have been officially deprecated on Launchpad: - // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy + // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite debDistroGoBoots = map[string]string{ - "trusty": "golang-1.11", - "xenial": "golang-go", - "bionic": "golang-go", - "focal": "golang-go", - "hirsute": "golang-go", + "trusty": "golang-1.11", // EOL: 04/2024 + "xenial": "golang-go", // EOL: 04/2026 + "bionic": "golang-go", // EOL: 04/2028 + "focal": "golang-go", // EOL: 04/2030 + "impish": "golang-go", // EOL: 07/2022 + // "jammy": "golang-go", // EOL: 04/2027 } debGoBootPaths = map[string]string{ @@ -148,7 +149,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.17.5" + dlgoVersion = "1.18" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -332,16 +333,21 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.42.0" + const version = "1.45.2" csdb := build.MustLoadChecksums("build/checksums.txt") arch := runtime.GOARCH + ext := ".tar.gz" + + if runtime.GOOS == "windows" { + ext = ".zip" + } if arch == "arm" { arch += "v" + os.Getenv("GOARM") } base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) - url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base) - archivePath := filepath.Join(cachedir, base+".tar.gz") + url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext) + archivePath := filepath.Join(cachedir, base+ext) if err := csdb.DownloadFile(url, archivePath); err != nil { log.Fatal(err) } @@ -1238,21 +1244,21 @@ func doPurge(cmdline []string) { // Iterate over the blobs, collect and sort all unstable builds for i := 0; i < len(blobs); i++ { - if !strings.Contains(blobs[i].Name, "unstable") { + if !strings.Contains(*blobs[i].Name, "unstable") { blobs = append(blobs[:i], blobs[i+1:]...) i-- } } for i := 0; i < len(blobs); i++ { for j := i + 1; j < len(blobs); j++ { - if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) { + if blobs[i].Properties.LastModified.After(*blobs[j].Properties.LastModified) { blobs[i], blobs[j] = blobs[j], blobs[i] } } } // Filter out all archives more recent that the given threshold for i, blob := range blobs { - if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { + if time.Since(*blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { blobs = blobs[:i] break } diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 3aaf898db2e6..f7c3adebc44a 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -661,7 +661,7 @@ func signer(c *cli.Context) error { if err != nil { utils.Fatalf("Could not register API: %w", err) } - handler := node.NewHTTPHandlerStack(srv, cors, vhosts) + handler := node.NewHTTPHandlerStack(srv, cors, vhosts, nil) // set port port := c.Int(rpcPortFlag.Name) diff --git a/cmd/evm/testdata/15/exp3.json b/cmd/evm/testdata/15/exp3.json index 6c46d267cf37..d7606a207361 100644 --- a/cmd/evm/testdata/15/exp3.json +++ b/cmd/evm/testdata/15/exp3.json @@ -21,19 +21,19 @@ "error": "transaction type not supported" }, { - "error": "rlp: expected List" + "error": "typed transaction too short" }, { - "error": "rlp: expected List" + "error": "typed transaction too short" }, { - "error": "rlp: expected List" + "error": "typed transaction too short" }, { - "error": "rlp: expected List" + "error": "typed transaction too short" }, { - "error": "rlp: expected List" + "error": "typed transaction too short" }, { "error": "rlp: expected input list for types.AccessListTx" diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 05d0dd19f392..d36ffb241d9a 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" @@ -174,6 +175,22 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } backend, eth := utils.RegisterEthService(stack, &cfg.Eth) + // Warn users to migrate if they have a legacy freezer format. + if eth != nil { + firstIdx := uint64(0) + // Hack to speed up check for mainnet because we know + // the first non-empty block. + ghash := rawdb.ReadCanonicalHash(eth.ChainDb(), 0) + if cfg.Eth.NetworkId == 1 && ghash == params.MainnetGenesisHash { + firstIdx = 46147 + } + isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx) + if err != nil { + log.Error("Failed to check db for legacy receipts", "err", err) + } else if isLegacy { + log.Warn("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") + } + } if ctx.GlobalBool(utils.StateDiffFlag.Name) { var indexerConfig interfaces.Config @@ -194,7 +211,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { switch dbType { case shared.FILE: indexerConfig = file.Config{ - FilePath: ctx.GlobalString(utils.StateDiffFilePath.Name), + FilePath: ctx.GlobalString(utils.StateDiffFilePath.Name), + WatchedAddressesFilePath: ctx.GlobalString(utils.StateDiffWatchedAddressesFilePath.Name), } case shared.POSTGRES: driverTypeStr := ctx.GlobalString(utils.StateDiffDBDriverTypeFlag.Name) @@ -252,14 +270,16 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } } p := statediff.Config{ - IndexerConfig: indexerConfig, - ID: nodeID, - ClientName: clientName, - Context: context.Background(), - EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name), - NumWorkers: ctx.GlobalUint(utils.StateDiffWorkersFlag.Name), + IndexerConfig: indexerConfig, + KnownGapsFilePath: ctx.GlobalString(utils.StateDiffKnownGapsFilePath.Name), + ID: nodeID, + ClientName: clientName, + Context: context.Background(), + EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name), + NumWorkers: ctx.GlobalUint(utils.StateDiffWorkersFlag.Name), + WaitForSync: ctx.GlobalBool(utils.StateDiffWaitForSync.Name), } - utils.RegisterStateDiffService(stack, eth, &cfg.Eth, p) + utils.RegisterStateDiffService(stack, eth, &cfg.Eth, p, backend) } // Configure GraphQL if requested diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 4799a6388a9d..33a7becfcd00 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" @@ -72,6 +73,7 @@ Remove blockchain and state databases`, dbImportCmd, dbExportCmd, dbMetadataCmd, + dbMigrateFreezerCmd, }, } dbInspectCmd = cli.Command{ @@ -251,6 +253,23 @@ WARNING: This is a low-level operation which may cause database corruption!`, }, Description: "Shows metadata about the chain status.", } + dbMigrateFreezerCmd = cli.Command{ + Action: utils.MigrateFlags(freezerMigrate), + Name: "freezer-migrate", + Usage: "Migrate legacy parts of the freezer. (WARNING: may take a long time)", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.SepoliaFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: `The freezer-migrate command checks your database for receipts in a legacy format and updates those. +WARNING: please back-up the receipt files in your ancients before running this command.`, + } ) func removeDB(ctx *cli.Context) error { @@ -728,6 +747,9 @@ func showMetaData(ctx *cli.Context) error { data = append(data, []string{"headBlock.Root", fmt.Sprintf("%v", b.Root())}) data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (0x%x)", b.Number(), b.Number())}) } + if b := rawdb.ReadSkeletonSyncStatus(db); b != nil { + data = append(data, []string{"SkeletonSyncStatus", string(b)}) + } if h := rawdb.ReadHeadHeader(db); h != nil { data = append(data, []string{"headHeader.Hash", fmt.Sprintf("%v", h.Hash())}) data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) @@ -750,3 +772,88 @@ func showMetaData(ctx *cli.Context) error { table.Render() return nil } + +func freezerMigrate(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, false) + defer db.Close() + + // Check first block for legacy receipt format + numAncients, err := db.Ancients() + if err != nil { + return err + } + if numAncients < 1 { + log.Info("No receipts in freezer to migrate") + return nil + } + + isFirstLegacy, firstIdx, err := dbHasLegacyReceipts(db, 0) + if err != nil { + return err + } + if !isFirstLegacy { + log.Info("No legacy receipts to migrate") + return nil + } + + log.Info("Starting migration", "ancients", numAncients, "firstLegacy", firstIdx) + start := time.Now() + if err := db.MigrateTable("receipts", types.ConvertLegacyStoredReceipts); err != nil { + return err + } + if err := db.Close(); err != nil { + return err + } + log.Info("Migration finished", "duration", time.Since(start)) + + return nil +} + +// dbHasLegacyReceipts checks freezer entries for legacy receipts. It stops at the first +// non-empty receipt and checks its format. The index of this first non-empty element is +// the second return parameter. +func dbHasLegacyReceipts(db ethdb.Database, firstIdx uint64) (bool, uint64, error) { + // Check first block for legacy receipt format + numAncients, err := db.Ancients() + if err != nil { + return false, 0, err + } + if numAncients < 1 { + return false, 0, nil + } + if firstIdx >= numAncients { + return false, firstIdx, nil + } + var ( + legacy bool + blob []byte + emptyRLPList = []byte{192} + ) + // Find first block with non-empty receipt, only if + // the index is not already provided. + if firstIdx == 0 { + for i := uint64(0); i < numAncients; i++ { + blob, err = db.Ancient("receipts", i) + if err != nil { + return false, 0, err + } + if len(blob) == 0 { + continue + } + if !bytes.Equal(blob, emptyRLPList) { + firstIdx = i + break + } + } + } + // Is first non-empty receipt legacy? + first, err := db.Ancient("receipts", firstIdx) + if err != nil { + return false, 0, err + } + legacy, err = types.IsLegacyStoredReceipts(first) + return legacy, firstIdx, err +} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 6471d618a14e..8309beaa59a1 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -107,7 +107,8 @@ var ( utils.UltraLightFractionFlag, utils.UltraLightOnlyAnnounceFlag, utils.LightNoSyncServeFlag, - utils.WhitelistFlag, + utils.EthPeerRequiredBlocksFlag, + utils.LegacyWhitelistFlag, utils.BloomFilterSizeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, @@ -118,6 +119,7 @@ var ( utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, + utils.FDLimitFlag, utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, @@ -146,6 +148,7 @@ var ( utils.SepoliaFlag, utils.RinkebyFlag, utils.GoerliFlag, + utils.KilnFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, @@ -176,6 +179,9 @@ var ( utils.StateDiffWritingFlag, utils.StateDiffWorkersFlag, utils.StateDiffFilePath, + utils.StateDiffKnownGapsFilePath, + utils.StateDiffWaitForSync, + utils.StateDiffWatchedAddressesFilePath, configFileFlag, } @@ -184,6 +190,10 @@ var ( utils.HTTPListenAddrFlag, utils.HTTPPortFlag, utils.HTTPCORSDomainFlag, + utils.AuthListenFlag, + utils.AuthPortFlag, + utils.AuthVirtualHostsFlag, + utils.JWTSecretFlag, utils.HTTPVirtualHostsFlag, utils.GraphQLEnabledFlag, utils.GraphQLCORSDomainFlag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index fdd46d9445e6..d0539eeff74c 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -314,8 +314,7 @@ func traverseState(ctx *cli.Context) error { } } if !bytes.Equal(acc.CodeHash, emptyCode) { - code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash)) - if len(code) == 0 { + if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { log.Error("Code is missing", "hash", common.BytesToHash(acc.CodeHash)) return errors.New("missing code") } @@ -386,11 +385,10 @@ func traverseRawState(ctx *cli.Context) error { nodes += 1 node := accIter.Hash() + // Check the present for non-empty hash node(embedded node doesn't + // have their own hash). if node != (common.Hash{}) { - // Check the present for non-empty hash node(embedded node doesn't - // have their own hash). - blob := rawdb.ReadTrieNode(chaindb, node) - if len(blob) == 0 { + if !rawdb.HasTrieNode(chaindb, node) { log.Error("Missing trie node(account)", "hash", node) return errors.New("missing account") } @@ -434,8 +432,7 @@ func traverseRawState(ctx *cli.Context) error { } } if !bytes.Equal(acc.CodeHash, emptyCode) { - code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash)) - if len(code) == 0 { + if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { log.Error("Code is missing", "account", common.BytesToHash(accIter.LeafKey())) return errors.New("missing code") } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 0700b0495fa7..660f0edf9b37 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -46,6 +46,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.RinkebyFlag, utils.RopstenFlag, utils.SepoliaFlag, + utils.KilnFlag, utils.SyncModeFlag, utils.ExitWhenSyncedFlag, utils.GCModeFlag, @@ -53,7 +54,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.EthStatsURLFlag, utils.IdentityFlag, utils.LightKDFFlag, - utils.WhitelistFlag, + utils.EthPeerRequiredBlocksFlag, }, }, { @@ -119,6 +120,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, + utils.FDLimitFlag, }, }, { @@ -148,6 +150,10 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.WSApiFlag, utils.WSPathPrefixFlag, utils.WSAllowedOriginsFlag, + utils.JWTSecretFlag, + utils.AuthListenFlag, + utils.AuthPortFlag, + utils.AuthVirtualHostsFlag, utils.GraphQLEnabledFlag, utils.GraphQLCORSDomainFlag, utils.GraphQLVirtualHostsFlag, @@ -221,6 +227,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ Name: "ALIASED (deprecated)", Flags: []cli.Flag{ utils.NoUSBFlag, + utils.LegacyWhitelistFlag, }, }, { @@ -246,6 +253,9 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.StateDiffWritingFlag, utils.StateDiffWorkersFlag, utils.StateDiffFilePath, + utils.StateDiffKnownGapsFilePath, + utils.StateDiffWaitForSync, + utils.StateDiffWatchedAddressesFilePath, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 705a964c3424..92599a67f27a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -163,6 +163,10 @@ var ( Name: "sepolia", Usage: "Sepolia network: pre-configured proof-of-work test network", } + KilnFlag = cli.BoolFlag{ + Name: "kiln", + Usage: "Kiln network: pre-configured proof-of-work to proof-of-stake test network", + } DeveloperFlag = cli.BoolFlag{ Name: "dev", Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled", @@ -239,9 +243,13 @@ var ( Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } - WhitelistFlag = cli.StringFlag{ + EthPeerRequiredBlocksFlag = cli.StringFlag{ + Name: "eth.requiredblocks", + Usage: "Comma separated block number-to-hash mappings to require for peering (=)", + } + LegacyWhitelistFlag = cli.StringFlag{ Name: "whitelist", - Usage: "Comma separated block number-to-hash mappings to enforce (=)", + Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --peer.requiredblocks)", } BloomFilterSizeFlag = cli.Uint64Flag{ Name: "bloomfilter.size", @@ -435,6 +443,10 @@ var ( Name: "cache.preimages", Usage: "Enable recording the SHA3/keccak preimages of trie keys", } + FDLimitFlag = cli.IntFlag{ + Name: "fdlimit", + Usage: "Raise the open file descriptor resource limit (default = system fd limit)", + } // Miner settings MiningEnabledFlag = cli.BoolFlag{ Name: "mine", @@ -520,6 +532,26 @@ var ( Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)", Value: ethconfig.Defaults.RPCTxFeeCap, } + // Authenticated RPC HTTP settings + AuthListenFlag = cli.StringFlag{ + Name: "authrpc.addr", + Usage: "Listening address for authenticated APIs", + Value: node.DefaultConfig.AuthAddr, + } + AuthPortFlag = cli.IntFlag{ + Name: "authrpc.port", + Usage: "Listening port for authenticated APIs", + Value: node.DefaultConfig.AuthPort, + } + AuthVirtualHostsFlag = cli.StringFlag{ + Name: "authrpc.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: strings.Join(node.DefaultConfig.AuthVirtualHosts, ","), + } + JWTSecretFlag = cli.StringFlag{ + Name: "authrpc.jwtsecret", + Usage: "Path to a JWT secret to use for authenticated RPC endpoints", + } // Logging and debug settings EthStatsURLFlag = cli.StringFlag{ Name: "ethstats", @@ -867,6 +899,15 @@ var ( Name: "statediff.file.path", Usage: "Full path (including filename) to write statediff data out to when operating in file mode", } + StateDiffKnownGapsFilePath = cli.StringFlag{ + Name: "statediff.knowngapsfile.path", + Usage: "Full path (including filename) to write knownGaps statements when the DB is unavailable.", + Value: "./known_gaps.sql", + } + StateDiffWatchedAddressesFilePath = cli.StringFlag{ + Name: "statediff.file.wapath", + Usage: "Full path (including filename) to write statediff watched addresses out to when operating in file mode", + } StateDiffDBClientNameFlag = cli.StringFlag{ Name: "statediff.db.clientname", Usage: "Client name to use when writing state diffs to database", @@ -881,6 +922,10 @@ var ( Usage: "Number of concurrent workers to use during statediff processing (default 1)", Value: 1, } + StateDiffWaitForSync = cli.BoolFlag{ + Name: "statediff.waitforsync", + Usage: "Should the statediff service wait for geth to catch up to the head of the chain?", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -902,6 +947,9 @@ func MakeDataDir(ctx *cli.Context) string { if ctx.GlobalBool(SepoliaFlag.Name) { return filepath.Join(path, "sepolia") } + if ctx.GlobalBool(KilnFlag.Name) { + return filepath.Join(path, "kiln") + } return path } Fatalf("Cannot determine default data directory, please set manually (--datadir)") @@ -956,6 +1004,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls = params.RinkebyBootnodes case ctx.GlobalBool(GoerliFlag.Name): urls = params.GoerliBootnodes + case ctx.GlobalBool(KilnFlag.Name): + urls = params.KilnBootnodes case cfg.BootstrapNodes != nil: return // already set, don't apply defaults. } @@ -1042,6 +1092,18 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { cfg.HTTPPort = ctx.GlobalInt(HTTPPortFlag.Name) } + if ctx.GlobalIsSet(AuthListenFlag.Name) { + cfg.AuthAddr = ctx.GlobalString(AuthListenFlag.Name) + } + + if ctx.GlobalIsSet(AuthPortFlag.Name) { + cfg.AuthPort = ctx.GlobalInt(AuthPortFlag.Name) + } + + if ctx.GlobalIsSet(AuthVirtualHostsFlag.Name) { + cfg.AuthVirtualHosts = SplitAndTrim(ctx.GlobalString(AuthVirtualHostsFlag.Name)) + } + if ctx.GlobalIsSet(HTTPCORSDomainFlag.Name) { cfg.HTTPCors = SplitAndTrim(ctx.GlobalString(HTTPCORSDomainFlag.Name)) } @@ -1152,11 +1214,24 @@ func setLes(ctx *cli.Context, cfg *ethconfig.Config) { // MakeDatabaseHandles raises out the number of allowed file handles per process // for Geth and returns half of the allowance to assign to the database. -func MakeDatabaseHandles() int { +func MakeDatabaseHandles(max int) int { limit, err := fdlimit.Maximum() if err != nil { Fatalf("Failed to retrieve file descriptor allowance: %v", err) } + switch { + case max == 0: + // User didn't specify a meaningful value, use system limits + case max < 128: + // User specified something unhealthy, just use system defaults + log.Error("File descriptor limit invalid (<128)", "had", max, "updated", limit) + case max > limit: + // User requested more than the OS allows, notify that we can't allocate it + log.Warn("Requested file descriptors denied by OS", "req", max, "limit", limit) + default: + // User limit is meaningful and within allowed range, use that + limit = max + } raised, err := fdlimit.Raise(uint64(limit)) if err != nil { Fatalf("Failed to raise file descriptor allowance: %v", err) @@ -1313,6 +1388,10 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setDataDir(ctx, cfg) setSmartCard(ctx, cfg) + if ctx.GlobalIsSet(JWTSecretFlag.Name) { + cfg.JWTSecret = ctx.GlobalString(JWTSecretFlag.Name) + } + if ctx.GlobalIsSet(ExternalSignerFlag.Name) { cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) } @@ -1381,6 +1460,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") case ctx.GlobalBool(SepoliaFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "sepolia") + case ctx.GlobalBool(KilnFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "kiln") } } @@ -1499,26 +1580,33 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { } } -func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) { - whitelist := ctx.GlobalString(WhitelistFlag.Name) - if whitelist == "" { - return +func setPeerRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { + peerRequiredBlocks := ctx.GlobalString(EthPeerRequiredBlocksFlag.Name) + + if peerRequiredBlocks == "" { + if ctx.GlobalIsSet(LegacyWhitelistFlag.Name) { + log.Warn("The flag --rpc is deprecated and will be removed, please use --peer.requiredblocks") + peerRequiredBlocks = ctx.GlobalString(LegacyWhitelistFlag.Name) + } else { + return + } } - cfg.Whitelist = make(map[uint64]common.Hash) - for _, entry := range strings.Split(whitelist, ",") { + + cfg.PeerRequiredBlocks = make(map[uint64]common.Hash) + for _, entry := range strings.Split(peerRequiredBlocks, ",") { parts := strings.Split(entry, "=") if len(parts) != 2 { - Fatalf("Invalid whitelist entry: %s", entry) + Fatalf("Invalid peer required block entry: %s", entry) } number, err := strconv.ParseUint(parts[0], 0, 64) if err != nil { - Fatalf("Invalid whitelist block number %s: %v", parts[0], err) + Fatalf("Invalid peer required block number %s: %v", parts[0], err) } var hash common.Hash if err = hash.UnmarshalText([]byte(parts[1])); err != nil { - Fatalf("Invalid whitelist hash %s: %v", parts[1], err) + Fatalf("Invalid peer required block hash %s: %v", parts[1], err) } - cfg.Whitelist[number] = hash + cfg.PeerRequiredBlocks[number] = hash } } @@ -1566,7 +1654,7 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Avoid conflicting network flags - CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag) + CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag, KilnFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { @@ -1585,7 +1673,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) setMiner(ctx, &cfg.Miner) - setWhitelist(ctx, cfg) + setPeerRequiredBlocks(ctx, cfg) setLes(ctx, cfg) // Cap the cache allowance and tune the garbage collector @@ -1617,7 +1705,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) { cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 } - cfg.DatabaseHandles = MakeDatabaseHandles() + cfg.DatabaseHandles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name)) if ctx.GlobalIsSet(AncientFlag.Name) { cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name) } @@ -1728,6 +1816,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.Genesis = core.DefaultGoerliGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash) + case ctx.GlobalBool(KilnFlag.Name): + if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + cfg.NetworkId = 1337802 + } + cfg.Genesis = core.DefaultKilnGenesisBlock() + SetDNSDiscoveryDefaults(cfg, params.KilnGenesisHash) case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1865,8 +1959,8 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, cfg node.C } // RegisterStateDiffService configures and registers a service to stream state diff data over RPC -func RegisterStateDiffService(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params statediff.Config) { - if err := statediff.New(stack, ethServ, cfg, params); err != nil { +func RegisterStateDiffService(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params statediff.Config, backend ethapi.Backend) { + if err := statediff.New(stack, ethServ, cfg, params, backend); err != nil { Fatalf("Failed to register the Statediff service: %v", err) } } @@ -1951,7 +2045,7 @@ func SplitTagsFlag(tagsFlag string) map[string]string { func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database { var ( cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 - handles = MakeDatabaseHandles() + handles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name)) err error chainDb ethdb.Database @@ -1982,6 +2076,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { genesis = core.DefaultRinkebyGenesisBlock() case ctx.GlobalBool(GoerliFlag.Name): genesis = core.DefaultGoerliGenesisBlock() + case ctx.GlobalBool(KilnFlag.Name): + genesis = core.DefaultKilnGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 4e33d99c8dde..c196ad062170 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -549,6 +549,11 @@ func NewShared() *Ethash { // Close closes the exit channel to notify all backend threads exiting. func (ethash *Ethash) Close() error { + return ethash.StopRemoteSealer() +} + +// StopRemoteSealer stops the remote sealer +func (ethash *Ethash) StopRemoteSealer() error { ethash.closeOnce.Do(func() { // Short circuit if the exit channel is not allocated. if ethash.remote == nil { diff --git a/core/beacon/errors.go b/core/beacon/errors.go index 5b95c38a23ba..83d5eebd5fa0 100644 --- a/core/beacon/errors.go +++ b/core/beacon/errors.go @@ -19,11 +19,32 @@ package beacon import "github.com/ethereum/go-ethereum/rpc" var ( - VALID = GenericStringResponse{"VALID"} - SUCCESS = GenericStringResponse{"SUCCESS"} - INVALID = ForkChoiceResponse{Status: "INVALID", PayloadID: nil} - SYNCING = ForkChoiceResponse{Status: "SYNCING", PayloadID: nil} + // VALID is returned by the engine API in the following calls: + // - newPayloadV1: if the payload was already known or was just validated and executed + // - forkchoiceUpdateV1: if the chain accepted the reorg (might ignore if it's stale) + VALID = "VALID" + + // INVALID is returned by the engine API in the following calls: + // - newPayloadV1: if the payload failed to execute on top of the local chain + // - forkchoiceUpdateV1: if the new head is unknown, pre-merge, or reorg to it fails + INVALID = "INVALID" + + // SYNCING is returned by the engine API in the following calls: + // - newPayloadV1: if the payload was accepted on top of an active sync + // - forkchoiceUpdateV1: if the new head was seen before, but not part of the chain + SYNCING = "SYNCING" + + // ACCEPTED is returned by the engine API in the following calls: + // - newPayloadV1: if the payload was accepted, but not processed (side chain) + ACCEPTED = "ACCEPTED" + + INVALIDBLOCKHASH = "INVALID_BLOCK_HASH" + INVALIDTERMINALBLOCK = "INVALID_TERMINAL_BLOCK" + GenericServerError = rpc.CustomError{Code: -32000, ValidationError: "Server error"} UnknownPayload = rpc.CustomError{Code: -32001, ValidationError: "Unknown payload"} InvalidTB = rpc.CustomError{Code: -32002, ValidationError: "Invalid terminal block"} + + STATUS_INVALID = ForkChoiceResponse{PayloadStatus: PayloadStatusV1{Status: INVALID}, PayloadID: nil} + STATUS_SYNCING = ForkChoiceResponse{PayloadStatus: PayloadStatusV1{Status: SYNCING}, PayloadID: nil} ) diff --git a/core/beacon/gen_blockparams.go b/core/beacon/gen_blockparams.go index d3d569b7da75..0e2ea4bb1338 100644 --- a/core/beacon/gen_blockparams.go +++ b/core/beacon/gen_blockparams.go @@ -16,7 +16,7 @@ var _ = (*payloadAttributesMarshaling)(nil) func (p PayloadAttributesV1) MarshalJSON() ([]byte, error) { type PayloadAttributesV1 struct { Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` - Random common.Hash `json:"random" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient" gencodec:"required"` } var enc PayloadAttributesV1 @@ -30,7 +30,7 @@ func (p PayloadAttributesV1) MarshalJSON() ([]byte, error) { func (p *PayloadAttributesV1) UnmarshalJSON(input []byte) error { type PayloadAttributesV1 struct { Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` - Random *common.Hash `json:"random" gencodec:"required"` + Random *common.Hash `json:"prevRandao" gencodec:"required"` SuggestedFeeRecipient *common.Address `json:"suggestedFeeRecipient" gencodec:"required"` } var dec PayloadAttributesV1 @@ -42,7 +42,7 @@ func (p *PayloadAttributesV1) UnmarshalJSON(input []byte) error { } p.Timestamp = uint64(*dec.Timestamp) if dec.Random == nil { - return errors.New("missing required field 'random' for PayloadAttributesV1") + return errors.New("missing required field 'prevRandao' for PayloadAttributesV1") } p.Random = *dec.Random if dec.SuggestedFeeRecipient == nil { diff --git a/core/beacon/gen_ed.go b/core/beacon/gen_ed.go index ac94f49a562a..dcee3bf18c79 100644 --- a/core/beacon/gen_ed.go +++ b/core/beacon/gen_ed.go @@ -19,9 +19,9 @@ func (e ExecutableDataV1) MarshalJSON() ([]byte, error) { ParentHash common.Hash `json:"parentHash" gencodec:"required"` FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Random common.Hash `json:"random" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` @@ -60,9 +60,9 @@ func (e *ExecutableDataV1) UnmarshalJSON(input []byte) error { ParentHash *common.Hash `json:"parentHash" gencodec:"required"` FeeRecipient *common.Address `json:"feeRecipient" gencodec:"required"` StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` + ReceiptsRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Random *common.Hash `json:"random" gencodec:"required"` + Random *common.Hash `json:"prevRandao" gencodec:"required"` Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` @@ -97,7 +97,7 @@ func (e *ExecutableDataV1) UnmarshalJSON(input []byte) error { } e.LogsBloom = *dec.LogsBloom if dec.Random == nil { - return errors.New("missing required field 'random' for ExecutableDataV1") + return errors.New("missing required field 'prevRandao' for ExecutableDataV1") } e.Random = *dec.Random if dec.Number == nil { diff --git a/core/beacon/types.go b/core/beacon/types.go index d7f6ba535e5f..18d5d2ab78b4 100644 --- a/core/beacon/types.go +++ b/core/beacon/types.go @@ -31,7 +31,7 @@ import ( // PayloadAttributesV1 structure described at https://github.com/ethereum/execution-apis/pull/74 type PayloadAttributesV1 struct { Timestamp uint64 `json:"timestamp" gencodec:"required"` - Random common.Hash `json:"random" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient" gencodec:"required"` } @@ -47,9 +47,9 @@ type ExecutableDataV1 struct { ParentHash common.Hash `json:"parentHash" gencodec:"required"` FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` LogsBloom []byte `json:"logsBloom" gencodec:"required"` - Random common.Hash `json:"random" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` Number uint64 `json:"blockNumber" gencodec:"required"` GasLimit uint64 `json:"gasLimit" gencodec:"required"` GasUsed uint64 `json:"gasUsed" gencodec:"required"` @@ -72,26 +72,16 @@ type executableDataMarshaling struct { Transactions []hexutil.Bytes } -type NewBlockResponse struct { - Valid bool `json:"valid"` +type PayloadStatusV1 struct { + Status string `json:"status"` + LatestValidHash *common.Hash `json:"latestValidHash"` + ValidationError *string `json:"validationError"` } -type GenericResponse struct { - Success bool `json:"success"` -} - -type GenericStringResponse struct { - Status string `json:"status"` -} - -type ExecutePayloadResponse struct { - Status string `json:"status"` - LatestValidHash common.Hash `json:"latestValidHash"` -} - -type ConsensusValidatedParams struct { - BlockHash common.Hash `json:"blockHash"` - Status string `json:"status"` +type TransitionConfigurationV1 struct { + TerminalTotalDifficulty *hexutil.Big `json:"terminalTotalDifficulty"` + TerminalBlockHash common.Hash `json:"terminalBlockHash"` + TerminalBlockNumber hexutil.Uint64 `json:"terminalBlockNumber"` } // PayloadID is an identifier of the payload build process @@ -114,8 +104,8 @@ func (b *PayloadID) UnmarshalText(input []byte) error { } type ForkChoiceResponse struct { - Status string `json:"status"` - PayloadID *PayloadID `json:"payloadId"` + PayloadStatus PayloadStatusV1 `json:"payloadStatus"` + PayloadID *PayloadID `json:"payloadId"` } type ForkchoiceStateV1 struct { diff --git a/core/blockchain.go b/core/blockchain.go index fe8d772b71c4..0d47693df4de 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -549,6 +549,19 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo } } if beyondRoot || newHeadBlock.NumberU64() == 0 { + if newHeadBlock.NumberU64() == 0 { + // Recommit the genesis state into disk in case the rewinding destination + // is genesis block and the relevant state is gone. In the future this + // rewinding destination can be the earliest block stored in the chain + // if the historical chain pruning is enabled. In that case the logic + // needs to be improved here. + if !bc.HasState(bc.genesisBlock.Root()) { + if err := CommitGenesisState(bc.db, bc.genesisBlock.Hash()); err != nil { + log.Crit("Failed to commit genesis state", "err", err) + } + log.Debug("Recommitted genesis state to disk") + } + } log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) break } @@ -599,7 +612,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo if num+1 <= frozen { // Truncate all relative data(header, total difficulty, body, receipt // and canonical hash) from ancient store. - if err := bc.db.TruncateAncients(num); err != nil { + if err := bc.db.TruncateHead(num); err != nil { log.Crit("Failed to truncate ancient data", "number", num, "err", err) } // Remove the hash <-> number mapping from the active store. @@ -1001,7 +1014,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ size += int64(batch.ValueSize()) if err = batch.Write(); err != nil { fastBlock := bc.CurrentFastBlock().NumberU64() - if err := bc.db.TruncateAncients(fastBlock + 1); err != nil { + if err := bc.db.TruncateHead(fastBlock + 1); err != nil { log.Error("Can't truncate ancient store after failed insert", "err", err) } return 0, err @@ -1019,7 +1032,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if !updateHead(blockChain[len(blockChain)-1]) { // We end up here if the header chain has reorg'ed, and the blocks/receipts // don't match the canonical chain. - if err := bc.db.TruncateAncients(previousFastBlock + 1); err != nil { + if err := bc.db.TruncateHead(previousFastBlock + 1); err != nil { log.Error("Can't truncate ancient store after failed insert", "err", err) } return 0, errSideChainReceipts @@ -1665,12 +1678,16 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits) blockInsertTimer.UpdateSince(start) + // Report the import stats before returning the various results + stats.processed++ + stats.usedGas += usedGas + + dirty, _ := bc.stateCache.TrieDB().Size() + stats.report(chain, it.index, dirty, setHead) + if !setHead { - // We did not setHead, so we don't have any stats to update - log.Info("Inserted block", "number", block.Number(), "hash", block.Hash(), "txs", len(block.Transactions()), "elapsed", common.PrettyDuration(time.Since(start))) - return it.index, nil + return it.index, nil // Direct block insertion of a single block } - switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), @@ -1697,11 +1714,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), "root", block.Root()) } - stats.processed++ - stats.usedGas += usedGas - - dirty, _ := bc.stateCache.TrieDB().Size() - stats.report(chain, it.index, dirty) } // Any blocks remaining here? The only ones we care about are the future ones @@ -2098,28 +2110,39 @@ func (bc *BlockChain) InsertBlockWithoutSetHead(block *types.Block) error { // block. It's possible that after the reorg the relevant state of head // is missing. It can be fixed by inserting a new block which triggers // the re-execution. -func (bc *BlockChain) SetChainHead(newBlock *types.Block) error { +func (bc *BlockChain) SetChainHead(head *types.Block) error { if !bc.chainmu.TryLock() { return errChainStopped } defer bc.chainmu.Unlock() // Run the reorg if necessary and set the given block as new head. - if newBlock.ParentHash() != bc.CurrentBlock().Hash() { - if err := bc.reorg(bc.CurrentBlock(), newBlock); err != nil { + start := time.Now() + if head.ParentHash() != bc.CurrentBlock().Hash() { + if err := bc.reorg(bc.CurrentBlock(), head); err != nil { return err } } - bc.writeHeadBlock(newBlock) + bc.writeHeadBlock(head) // Emit events - logs := bc.collectLogs(newBlock.Hash(), false) - bc.chainFeed.Send(ChainEvent{Block: newBlock, Hash: newBlock.Hash(), Logs: logs}) + logs := bc.collectLogs(head.Hash(), false) + bc.chainFeed.Send(ChainEvent{Block: head, Hash: head.Hash(), Logs: logs}) if len(logs) > 0 { bc.logsFeed.Send(logs) } - bc.chainHeadFeed.Send(ChainHeadEvent{Block: newBlock}) - log.Info("Set the chain head", "number", newBlock.Number(), "hash", newBlock.Hash()) + bc.chainHeadFeed.Send(ChainHeadEvent{Block: head}) + + context := []interface{}{ + "number", head.Number(), + "hash", head.Hash(), + "root", head.Root(), + "elapsed", time.Since(start), + } + if timestamp := time.Unix(int64(head.Time()), 0); time.Since(timestamp) > time.Minute { + context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...) + } + log.Info("Chain head was updated", context...) return nil } @@ -2305,6 +2328,9 @@ Error: %v // of the header retrieval mechanisms already need to verify nonces, as well as // because nonces can be verified sparsely, not needing to check each. func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) { + if len(chain) == 0 { + return 0, nil + } start := time.Now() if i, err := bc.hc.ValidateHeaderChain(chain, checkFreq); err != nil { return i, err diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index 446487027911..479eccc83e47 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -39,7 +39,7 @@ const statsReportLimit = 8 * time.Second // report prints statistics if some number of blocks have been processed // or more than a few seconds have passed since the last message. -func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize) { +func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize, setHead bool) { // Fetch the timings for the batch var ( now = mclock.Now() @@ -71,8 +71,11 @@ func (st *insertStats) report(chain []*types.Block, index int, dirty common.Stor if st.ignored > 0 { context = append(context, []interface{}{"ignored", st.ignored}...) } - log.Info("Imported new chain segment", context...) - + if setHead { + log.Info("Imported new chain segment", context...) + } else { + log.Info("Imported new potential chain segment", context...) + } // Bump the stats reported to the next section *st = insertStats{startTime: now, lastIndex: index + 1} } diff --git a/core/genesis.go b/core/genesis.go index 1d17f298a4fb..4949197da5cd 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -80,6 +80,81 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { return nil } +// flush adds allocated genesis accounts into a fresh new statedb and +// commit the state changes into the given database handler. +func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { + statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) + if err != nil { + return common.Hash{}, err + } + for addr, account := range *ga { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } + root, err := statedb.Commit(false) + if err != nil { + return common.Hash{}, err + } + err = statedb.Database().TrieDB().Commit(root, true, nil) + if err != nil { + return common.Hash{}, err + } + return root, nil +} + +// write writes the json marshaled genesis state into database +// with the given block hash as the unique identifier. +func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { + blob, err := json.Marshal(ga) + if err != nil { + return err + } + rawdb.WriteGenesisState(db, hash, blob) + return nil +} + +// CommitGenesisState loads the stored genesis state with the given block +// hash and commits them into the given database handler. +func CommitGenesisState(db ethdb.Database, hash common.Hash) error { + var alloc GenesisAlloc + blob := rawdb.ReadGenesisState(db, hash) + if len(blob) != 0 { + if err := alloc.UnmarshalJSON(blob); err != nil { + return err + } + } else { + // Genesis allocation is missing and there are several possibilities: + // the node is legacy which doesn't persist the genesis allocation or + // the persisted allocation is just lost. + // - supported networks(mainnet, testnets), recover with defined allocations + // - private network, can't recover + var genesis *Genesis + switch hash { + case params.MainnetGenesisHash: + genesis = DefaultGenesisBlock() + case params.RopstenGenesisHash: + genesis = DefaultRopstenGenesisBlock() + case params.RinkebyGenesisHash: + genesis = DefaultRinkebyGenesisBlock() + case params.GoerliGenesisHash: + genesis = DefaultGoerliGenesisBlock() + case params.SepoliaGenesisHash: + genesis = DefaultSepoliaGenesisBlock() + } + if genesis != nil { + alloc = genesis.Alloc + } else { + return errors.New("not found") + } + } + _, err := alloc.flush(db) + return err +} + // GenesisAccount is an account in the state of the genesis block. type GenesisAccount struct { Code []byte `json:"code,omitempty"` @@ -219,11 +294,19 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override rawdb.WriteChainConfig(db, stored, newcfg) return newcfg, stored, nil } - // Special case: don't change the existing config of a non-mainnet chain if no new - // config is supplied. These chains would get AllProtocolChanges (and a compat error) - // if we just continued here. + // Special case: if a private network is being used (no genesis and also no + // mainnet hash in the database), we must not apply the `configOrDefault` + // chain config as that would be AllProtocolChanges (applying any new fork + // on top of an existing private network genesis block). In that case, only + // apply the overrides. if genesis == nil && stored != params.MainnetGenesisHash { - return storedcfg, stored, nil + newcfg = storedcfg + if overrideArrowGlacier != nil { + newcfg.ArrowGlacierBlock = overrideArrowGlacier + } + if overrideTerminalTotalDifficulty != nil { + newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty + } } // Check config compatibility and write the config. Compatibility errors // are returned to the caller unless we're already at block zero. @@ -253,6 +336,8 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { return params.RinkebyChainConfig case ghash == params.GoerliGenesisHash: return params.GoerliChainConfig + case ghash == params.KilnGenesisHash: + return DefaultKilnGenesisBlock().Config default: return params.AllEthashProtocolChanges } @@ -264,19 +349,10 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if db == nil { db = rawdb.NewMemoryDatabase() } - statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) + root, err := g.Alloc.flush(db) if err != nil { panic(err) } - for addr, account := range g.Alloc { - statedb.AddBalance(addr, account.Balance) - statedb.SetCode(addr, account.Code) - statedb.SetNonce(addr, account.Nonce) - for key, value := range account.Storage { - statedb.SetState(addr, key, value) - } - } - root := statedb.IntermediateRoot(false) head := &types.Header{ Number: new(big.Int).SetUint64(g.Number), Nonce: types.EncodeNonce(g.Nonce), @@ -304,9 +380,6 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { head.BaseFee = new(big.Int).SetUint64(params.InitialBaseFee) } } - statedb.Commit(false) - statedb.Database().TrieDB().Commit(root, true, nil) - return types.NewBlock(head, nil, nil, nil, trie.NewStackTrie(nil)) } @@ -327,6 +400,9 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if config.Clique != nil && len(block.Extra()) == 0 { return nil, errors.New("can't start clique chain without signers") } + if err := g.Alloc.write(db, block.Hash()); err != nil { + return nil, err + } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) rawdb.WriteBlock(db, block) rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), nil) @@ -418,6 +494,15 @@ func DefaultSepoliaGenesisBlock() *Genesis { } } +func DefaultKilnGenesisBlock() *Genesis { + g := new(Genesis) + reader := strings.NewReader(KilnAllocData) + if err := json.NewDecoder(reader).Decode(g); err != nil { + panic(err) + } + return g +} + // DeveloperGenesisBlock returns the 'geth --dev' genesis block. func DeveloperGenesisBlock(period uint64, gasLimit uint64, faucet common.Address) *Genesis { // Override the default period to the user requested one diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index 3d053904e7af..041c55424238 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -27,3 +27,868 @@ const rinkebyAllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03 const goerliAllocData = "\xf9\x04\x06\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xe0\x94L*\xe4\x82Y5\x05\xf0\x16<\xde\xfc\a>\x81\xc6<\xdaA\a\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe0\x94\xa8\xe8\xf1G2e\x8eKQ\xe8q\x191\x05:\x8ai\xba\xf2\xb1\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe1\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\u08bdBX\xd2v\x887\xba\xa2j(\xfeq\xdc\a\x9f\x84\u01cbJG\xe3\xc1$H\xf4\xad\x00\x00\x00" const calaverasAllocData = "\xf9\x06\x14\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x10A\xaf\xbc\xb3Y\u0568\xdcX\xc1[/\xf5\x13T\xff\x8a!}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94#o\xf1\xe9t\x19\xae\x93\xad\x80\xca\xfb\xaa!\"\f]x\xfb}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xba\xdc\r\xe9\xe0yK\x04\x9b^\xa6<>\x1ei\x8a4v\xc1r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xf00\v\ue24a\xe2r\xeb4~\x83i\xac\fv\xdfB\xc9?\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xfe;U~\x8f\xb6+\x89\xf4\x91kr\x1b\xe5\\\ub08d\xbds\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const sepoliaAllocData = "\xf9\x01\xee\u0791i\x16\xa8{\x823?BE\x04f#\xb27\x94\xc6\\\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\x10\xf5\xd4XT\xe08\a\x14\x85\xac\x9e@#\b\u03c0\xd2\xd2\xfe\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\u0794y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\x88\r\u0db3\xa7d\x00\x00\xe0\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x8b\u007f\tw\xbbO\x0f\xbepv\xfa\"\xbc$\xac\xa0CX?^\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xa2\xa6\xd949\x14O\xfeM'\xc9\xe0\x88\xdc\u0637\x83\x94bc\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xaa\xec\x869DA\xf9\x15\xbc\xe3\xe6\xab9\x99w\xe9\x90o;i\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\u1532\x1c3\xde\x1f\xab?\xa1T\x99\xc6+Y\xfe\f\xc3%\x00 \u044bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xbc\x11)Y6\xaay\u0554\x13\x9d\xe1\xb2\xe1&)AO;\u06ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xbe\xef2\xca[\x9a\x19\x8d'\xb4\xe0/LpC\x9f\xe6\x03V\u03ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe1\x94\xd7\xd7lX\xb3\xa5\x19\xe9\xfal\xc4\xd2-\xc0\x17%\x9b\u011f\x1e\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xd7\xed\xdbx\xed)[<\x96)$\x0e\x89$\xfb\x8d\x88t\xdd\u060a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xe2\xe2e\x90(\x147\x84\xd5W\xbc\xeco\xf3\xa0r\x10H\x88\n\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xf4|\xae\x1c\xf7\x9c\xa6u\x8b\xfcx}\xbd!\u6f7eq\x12\xb8\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00" +const KilnAllocData = `{ + "config": { + "chainId": 1337802, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "mergeForkBlock": 1000, + "terminalTotalDifficulty": 20000000000000 + }, + "alloc": { + "0x0000000000000000000000000000000000000000": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000001": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000002": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000003": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000004": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000005": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000006": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000007": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000008": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000009": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000010": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000011": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000012": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000013": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000014": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000015": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000016": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000017": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000018": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000019": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000020": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000021": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000022": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000023": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000024": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000025": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000026": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000027": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000028": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000029": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000030": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000031": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000032": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000033": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000034": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000035": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000036": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000037": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000038": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000039": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000040": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000041": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000042": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000043": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000044": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000045": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000046": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000047": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000048": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000049": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000050": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000051": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000052": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000053": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000054": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000055": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000056": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000057": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000058": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000059": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000060": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000061": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000062": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000063": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000064": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000065": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000066": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000067": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000068": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000069": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000070": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000071": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000072": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000073": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000074": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000075": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000076": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000077": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000078": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000079": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000080": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000081": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000082": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000083": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000084": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000085": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000086": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000087": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000088": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000089": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000090": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000091": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000092": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000093": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000094": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000095": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000096": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000097": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000098": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000099": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009f": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000aa": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ab": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ac": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ad": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ae": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000af": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ba": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000be": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bf": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ca": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ce": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cf": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000da": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000db": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000dc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000dd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000de": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000df": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ea": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000eb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ec": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ed": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ee": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ef": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fa": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fe": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ff": { + "balance": "1" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + }, + "0xf97e180c050e5Ab072211Ad2C213Eb5AEE4DF134": { + "balance": "10000000000000000000000000" + }, + "0x2cA5F489CC1Fd1CEC24747B64E8dE0F4A6A850E1": { + "balance": "10000000000000000000000000" + }, + "0x7203bd333a874D9d329050ecE393820fCD501eaA": { + "balance": "10000000000000000000000000" + }, + "0xA51918aA40D78Ff8be939bf0E8404252875c6aDF": { + "balance": "10000000000000000000000000" + }, + "0xAA81078e6b2121dd7A846690DFdD6b10d7658d8B": { + "balance": "10000000000000000000000000" + }, + "0xFA2d31D8f21c1D1633E9BEB641dF77D21D63ccDd": { + "balance": "10000000000000000000000000" + }, + "0xf751C9c6d60614226fE57D2cAD6e10C856a2ddA3": { + "balance": "10000000000000000000000000" + }, + "0x9cD16887f6A808AEaa65D3c840f059EeA4ca1319": { + "balance": "10000000000000000000000000" + }, + "0x2E07043584F11BFF0AC39c927665DF6c6ebaffFB": { + "balance": "10000000000000000000000000" + }, + "0x60e771E5eCA8E26690920de669520Da210D64A9B": { + "balance": "10000000000000000000000000" + }, + "0xFC4db92C2Cf77CE02fBfd7Da0346d2CbFA66aD59": { + "balance": "10000000000000000000000000" + } + }, + "coinbase": "0x0000000000000000000000000000000000000000", + "difficulty": "0x01", + "extraData": "", + "gasLimit": "0x400000", + "nonce": "0x1234", + "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0" + }` diff --git a/core/genesis_test.go b/core/genesis_test.go index f3d6b23e5fe0..e8010e3d4ebd 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -213,3 +213,33 @@ func TestGenesis_Commit(t *testing.T) { t.Errorf("inequal difficulty; stored: %v, genesisBlock: %v", stored, genesisBlock.Difficulty()) } } + +func TestReadWriteGenesisAlloc(t *testing.T) { + var ( + db = rawdb.NewMemoryDatabase() + alloc = &GenesisAlloc{ + {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, + {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, + } + hash = common.HexToHash("0xdeadbeef") + ) + alloc.write(db, hash) + + var reload GenesisAlloc + err := reload.UnmarshalJSON(rawdb.ReadGenesisState(db, hash)) + if err != nil { + t.Fatalf("Failed to load genesis state %v", err) + } + if len(reload) != len(*alloc) { + t.Fatal("Unexpected genesis allocation") + } + for addr, account := range reload { + want, ok := (*alloc)[addr] + if !ok { + t.Fatal("Account is not found") + } + if !reflect.DeepEqual(want, account) { + t.Fatal("Unexpected account") + } + } +} diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 8e9706ea6fdb..f9c224dfa8f8 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -83,8 +83,8 @@ type NumberHash struct { Hash common.Hash } -// ReadAllHashes retrieves all the hashes assigned to blocks at a certain heights, -// both canonical and reorged forks included. +// ReadAllHashesInRange retrieves all the hashes assigned to blocks at certain +// heights, both canonical and reorged forks included. // This method considers both limits to be _inclusive_. func ReadAllHashesInRange(db ethdb.Iteratee, first, last uint64) []*NumberHash { var ( @@ -776,7 +776,7 @@ func WriteBlock(db ethdb.KeyValueWriter, block *types.Block) { WriteHeader(db, block.Header()) } -// WriteAncientBlock writes entire block data into ancient store and returns the total written size. +// WriteAncientBlocks writes entire block data into ancient store and returns the total written size. func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts []types.Receipts, td *big.Int) (int64, error) { var ( tdSum = new(big.Int).Set(td) diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 3b0fcf0f2d1f..f5a161adb688 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -81,6 +81,19 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha } } +// ReadGenesisState retrieves the genesis state based on the given genesis hash. +func ReadGenesisState(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(genesisKey(hash)) + return data +} + +// WriteGenesisState writes the genesis state into the disk. +func WriteGenesisState(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { + if err := db.Put(genesisKey(hash), data); err != nil { + log.Crit("Failed to store genesis state", "err", err) + } +} + // crashList is a list of unclean-shutdown-markers, for rlp-encoding to the // database type crashList struct { diff --git a/core/rawdb/accessors_snapshot.go b/core/rawdb/accessors_snapshot.go index 1c828662c150..3c82b3f73141 100644 --- a/core/rawdb/accessors_snapshot.go +++ b/core/rawdb/accessors_snapshot.go @@ -115,7 +115,7 @@ func DeleteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash com // IterateStorageSnapshots returns an iterator for walking the entire storage // space of a specific account. func IterateStorageSnapshots(db ethdb.Iteratee, accountHash common.Hash) ethdb.Iterator { - return db.NewIterator(storageSnapshotsKey(accountHash), nil) + return NewKeyLengthIterator(db.NewIterator(storageSnapshotsKey(accountHash), nil), len(SnapshotStoragePrefix)+2*common.HashLength) } // ReadSnapshotJournal retrieves the serialized in-memory diff layers saved at diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index a239d0766743..41e21b6ca40b 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -28,17 +28,6 @@ func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte { return data } -// WritePreimages writes the provided set of preimages to the database. -func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) { - for hash, preimage := range preimages { - if err := db.Put(preimageKey(hash), preimage); err != nil { - log.Crit("Failed to store trie preimage", "err", err) - } - } - preimageCounter.Inc(int64(len(preimages))) - preimageHitCounter.Inc(int64(len(preimages))) -} - // ReadCode retrieves the contract code of the provided code hash. func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte { // Try with the prefixed code scheme first, if not then try with legacy @@ -47,7 +36,7 @@ func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte { if len(data) != 0 { return data } - data, _ = db.Get(hash[:]) + data, _ = db.Get(hash.Bytes()) return data } @@ -59,6 +48,24 @@ func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte { return data } +// ReadTrieNode retrieves the trie node of the provided hash. +func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(hash.Bytes()) + return data +} + +// HasCode checks if the contract code corresponding to the +// provided code hash is present in the db. +func HasCode(db ethdb.KeyValueReader, hash common.Hash) bool { + // Try with the prefixed code scheme first, if not then try with legacy + // scheme. + if ok := HasCodeWithPrefix(db, hash); ok { + return true + } + ok, _ := db.Has(hash.Bytes()) + return ok +} + // HasCodeWithPrefix checks if the contract code corresponding to the // provided code hash is present in the db. This function will only check // presence using the prefix-scheme. @@ -67,30 +74,28 @@ func HasCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool { return ok } -// WriteCode writes the provided contract code database. -func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) { - if err := db.Put(codeKey(hash), code); err != nil { - log.Crit("Failed to store contract code", "err", err) - } +// HasTrieNode checks if the trie node with the provided hash is present in db. +func HasTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool { + ok, _ := db.Has(hash.Bytes()) + return ok } -// DeleteCode deletes the specified contract code from the database. -func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) { - if err := db.Delete(codeKey(hash)); err != nil { - log.Crit("Failed to delete contract code", "err", err) +// WritePreimages writes the provided set of preimages to the database. +func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) { + for hash, preimage := range preimages { + if err := db.Put(preimageKey(hash), preimage); err != nil { + log.Crit("Failed to store trie preimage", "err", err) + } } + preimageCounter.Inc(int64(len(preimages))) + preimageHitCounter.Inc(int64(len(preimages))) } -// ReadTrieNode retrieves the trie node of the provided hash. -func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte { - data, _ := db.Get(hash.Bytes()) - return data -} - -// HasTrieNode checks if the trie node with the provided hash is present in db. -func HasTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool { - ok, _ := db.Has(hash.Bytes()) - return ok +// WriteCode writes the provided contract code database. +func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) { + if err := db.Put(codeKey(hash), code); err != nil { + log.Crit("Failed to store contract code", "err", err) + } } // WriteTrieNode writes the provided trie node database. @@ -100,6 +105,13 @@ func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) { } } +// DeleteCode deletes the specified contract code from the database. +func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(codeKey(hash)); err != nil { + log.Crit("Failed to delete contract code", "err", err) + } +} + // DeleteTrieNode deletes the specified trie node from the database. func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) { if err := db.Delete(hash.Bytes()); err != nil { diff --git a/core/rawdb/accessors_sync.go b/core/rawdb/accessors_sync.go new file mode 100644 index 000000000000..50dfb848e4e0 --- /dev/null +++ b/core/rawdb/accessors_sync.go @@ -0,0 +1,80 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +// ReadSkeletonSyncStatus retrieves the serialized sync status saved at shutdown. +func ReadSkeletonSyncStatus(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(skeletonSyncStatusKey) + return data +} + +// WriteSkeletonSyncStatus stores the serialized sync status to save at shutdown. +func WriteSkeletonSyncStatus(db ethdb.KeyValueWriter, status []byte) { + if err := db.Put(skeletonSyncStatusKey, status); err != nil { + log.Crit("Failed to store skeleton sync status", "err", err) + } +} + +// DeleteSkeletonSyncStatus deletes the serialized sync status saved at the last +// shutdown +func DeleteSkeletonSyncStatus(db ethdb.KeyValueWriter) { + if err := db.Delete(skeletonSyncStatusKey); err != nil { + log.Crit("Failed to remove skeleton sync status", "err", err) + } +} + +// ReadSkeletonHeader retrieves a block header from the skeleton sync store, +func ReadSkeletonHeader(db ethdb.KeyValueReader, number uint64) *types.Header { + data, _ := db.Get(skeletonHeaderKey(number)) + if len(data) == 0 { + return nil + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + log.Error("Invalid skeleton header RLP", "number", number, "err", err) + return nil + } + return header +} + +// WriteSkeletonHeader stores a block header into the skeleton sync store. +func WriteSkeletonHeader(db ethdb.KeyValueWriter, header *types.Header) { + data, err := rlp.EncodeToBytes(header) + if err != nil { + log.Crit("Failed to RLP encode header", "err", err) + } + key := skeletonHeaderKey(header.Number.Uint64()) + if err := db.Put(key, data); err != nil { + log.Crit("Failed to store skeleton header", "err", err) + } +} + +// DeleteSkeletonHeader removes all block header data associated with a hash. +func DeleteSkeletonHeader(db ethdb.KeyValueWriter, number uint64) { + if err := db.Delete(skeletonHeaderKey(number)); err != nil { + log.Crit("Failed to delete skeleton header", "err", err) + } +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 5ef64d26a205..5d645b61dbee 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -99,6 +99,11 @@ func (db *nofreezedb) Ancients() (uint64, error) { return 0, errNotSupported } +// Tail returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Tail() (uint64, error) { + return 0, errNotSupported +} + // AncientSize returns an error as we don't have a backing chain freezer. func (db *nofreezedb) AncientSize(kind string) (uint64, error) { return 0, errNotSupported @@ -109,8 +114,13 @@ func (db *nofreezedb) ModifyAncients(func(ethdb.AncientWriteOp) error) (int64, e return 0, errNotSupported } -// TruncateAncients returns an error as we don't have a backing chain freezer. -func (db *nofreezedb) TruncateAncients(items uint64) error { +// TruncateHead returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) TruncateHead(items uint64) error { + return errNotSupported +} + +// TruncateTail returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) TruncateTail(items uint64) error { return errNotSupported } @@ -135,6 +145,12 @@ func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReader) error) (e return fn(db) } +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (db *nofreezedb) MigrateTable(kind string, convert convertLegacyFn) error { + return errNotSupported +} + // NewDatabase creates a high level database on top of a given key-value data // store without a freezer moving immutable chain segments into cold storage. func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { @@ -211,7 +227,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // Block #1 is still in the database, we're allowed to init a new feezer } // Otherwise, the head header is still the genesis, we're allowed to init a new - // feezer. + // freezer. } } // Freezer is consistent with the key-value database, permit combining the two @@ -321,6 +337,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { storageSnaps stat preimages stat bloomBits stat + beaconHeaders stat cliqueSnaps stat // Ancient store statistics @@ -375,10 +392,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { preimages.Add(size) case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): metadata.Add(size) + case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): + metadata.Add(size) case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): bloomBits.Add(size) case bytes.HasPrefix(key, BloomBitsIndexPrefix): bloomBits.Add(size) + case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): + beaconHeaders.Add(size) case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: cliqueSnaps.Add(size) case bytes.HasPrefix(key, []byte("cht-")) || @@ -395,7 +416,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, - uncleanShutdownKey, badBlockKey, transitionStatusKey, + uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, } { if bytes.Equal(key, meta) { metadata.Add(size) @@ -441,6 +462,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, + {"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, {"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()}, diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 88c72625eede..8266933ee655 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -19,6 +19,7 @@ package rawdb import ( "errors" "fmt" + "io/ioutil" "math" "os" "path/filepath" @@ -66,7 +67,7 @@ const ( freezerTableSize = 2 * 1000 * 1000 * 1000 ) -// freezer is an memory mapped append-only database to store immutable chain data +// freezer is a memory mapped append-only database to store immutable chain data // into flat files: // // - The append only nature ensures that disk writes are minimized. @@ -78,6 +79,7 @@ type freezer struct { // 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned, // so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG). frozen uint64 // Number of blocks already frozen + tail uint64 // Number of the first stored item in the freezer threshold uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) // This lock synchronizes writers and the truncate operation, as well as @@ -226,6 +228,11 @@ func (f *freezer) Ancients() (uint64, error) { return atomic.LoadUint64(&f.frozen), nil } +// Tail returns the number of first stored item in the freezer. +func (f *freezer) Tail() (uint64, error) { + return atomic.LoadUint64(&f.tail), nil +} + // AncientSize returns the ancient size of the specified category. func (f *freezer) AncientSize(kind string) (uint64, error) { // This needs the write lock to avoid data races on table fields. @@ -261,7 +268,7 @@ func (f *freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize if err != nil { // The write operation has failed. Go back to the previous item position. for name, table := range f.tables { - err := table.truncate(prevItem) + err := table.truncateHead(prevItem) if err != nil { log.Error("Freezer table roll-back failed", "table", name, "index", prevItem, "err", err) } @@ -281,8 +288,8 @@ func (f *freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize return writeSize, nil } -// TruncateAncients discards any recent data above the provided threshold number. -func (f *freezer) TruncateAncients(items uint64) error { +// TruncateHead discards any recent data above the provided threshold number. +func (f *freezer) TruncateHead(items uint64) error { if f.readonly { return errReadOnly } @@ -293,7 +300,7 @@ func (f *freezer) TruncateAncients(items uint64) error { return nil } for _, table := range f.tables { - if err := table.truncate(items); err != nil { + if err := table.truncateHead(items); err != nil { return err } } @@ -301,6 +308,26 @@ func (f *freezer) TruncateAncients(items uint64) error { return nil } +// TruncateTail discards any recent data below the provided threshold number. +func (f *freezer) TruncateTail(tail uint64) error { + if f.readonly { + return errReadOnly + } + f.writeLock.Lock() + defer f.writeLock.Unlock() + + if atomic.LoadUint64(&f.tail) >= tail { + return nil + } + for _, table := range f.tables { + if err := table.truncateTail(tail); err != nil { + return err + } + } + atomic.StoreUint64(&f.tail, tail) + return nil +} + // Sync flushes all data tables to disk. func (f *freezer) Sync() error { var errs []error @@ -344,19 +371,30 @@ func (f *freezer) validate() error { // repair truncates all data tables to the same length. func (f *freezer) repair() error { - min := uint64(math.MaxUint64) + var ( + head = uint64(math.MaxUint64) + tail = uint64(0) + ) for _, table := range f.tables { items := atomic.LoadUint64(&table.items) - if min > items { - min = items + if head > items { + head = items + } + hidden := atomic.LoadUint64(&table.itemHidden) + if hidden > tail { + tail = hidden } } for _, table := range f.tables { - if err := table.truncate(min); err != nil { + if err := table.truncateHead(head); err != nil { + return err + } + if err := table.truncateTail(tail); err != nil { return err } } - atomic.StoreUint64(&f.frozen, min) + atomic.StoreUint64(&f.frozen, head) + atomic.StoreUint64(&f.tail, tail) return nil } @@ -580,3 +618,116 @@ func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes [] return hashes, err } + +// convertLegacyFn takes a raw freezer entry in an older format and +// returns it in the new format. +type convertLegacyFn = func([]byte) ([]byte, error) + +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { + if f.readonly { + return errReadOnly + } + f.writeLock.Lock() + defer f.writeLock.Unlock() + + table, ok := f.tables[kind] + if !ok { + return errUnknownTable + } + // forEach iterates every entry in the table serially and in order, calling `fn` + // with the item as argument. If `fn` returns an error the iteration stops + // and that error will be returned. + forEach := func(t *freezerTable, offset uint64, fn func(uint64, []byte) error) error { + var ( + items = atomic.LoadUint64(&t.items) + batchSize = uint64(1024) + maxBytes = uint64(1024 * 1024) + ) + for i := offset; i < items; { + if i+batchSize > items { + batchSize = items - i + } + data, err := t.RetrieveItems(i, batchSize, maxBytes) + if err != nil { + return err + } + for j, item := range data { + if err := fn(i+uint64(j), item); err != nil { + return err + } + } + i += uint64(len(data)) + } + return nil + } + // TODO(s1na): This is a sanity-check since as of now no process does tail-deletion. But the migration + // process assumes no deletion at tail and needs to be modified to account for that. + if table.itemOffset > 0 || table.itemHidden > 0 { + return fmt.Errorf("migration not supported for tail-deleted freezers") + } + ancientsPath := filepath.Dir(table.index.Name()) + // Set up new dir for the migrated table, the content of which + // we'll at the end move over to the ancients dir. + migrationPath := filepath.Join(ancientsPath, "migration") + newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind], false) + if err != nil { + return err + } + var ( + batch = newTable.newBatch() + out []byte + start = time.Now() + logged = time.Now() + offset = newTable.items + ) + if offset > 0 { + log.Info("found previous migration attempt", "migrated", offset) + } + // Iterate through entries and transform them + if err := forEach(table, offset, func(i uint64, blob []byte) error { + if i%10000 == 0 && time.Since(logged) > 16*time.Second { + log.Info("Processing legacy elements", "count", i, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + out, err = convert(blob) + if err != nil { + return err + } + if err := batch.AppendRaw(i, out); err != nil { + return err + } + return nil + }); err != nil { + return err + } + if err := batch.commit(); err != nil { + return err + } + log.Info("Replacing old table files with migrated ones", "elapsed", common.PrettyDuration(time.Since(start))) + // Release and delete old table files. Note this won't + // delete the index file. + table.releaseFilesAfter(0, true) + + if err := newTable.Close(); err != nil { + return err + } + files, err := ioutil.ReadDir(migrationPath) + if err != nil { + return err + } + // Move migrated files to ancients dir. + for _, f := range files { + // This will replace the old index file as a side-effect. + if err := os.Rename(filepath.Join(migrationPath, f.Name()), filepath.Join(ancientsPath, f.Name())); err != nil { + return err + } + } + // Delete by now empty dir. + if err := os.Remove(migrationPath); err != nil { + return err + } + + return nil +} diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go index 762fa8f25f19..864a7f5e98bf 100644 --- a/core/rawdb/freezer_batch.go +++ b/core/rawdb/freezer_batch.go @@ -191,7 +191,7 @@ func (batch *freezerTableBatch) commit() error { dataSize := int64(len(batch.dataBuffer)) batch.dataBuffer = batch.dataBuffer[:0] - // Write index. + // Write indices. _, err = batch.t.index.Write(batch.indexBuffer) if err != nil { return err diff --git a/core/rawdb/freezer_meta.go b/core/rawdb/freezer_meta.go new file mode 100644 index 000000000000..d0bd2f954436 --- /dev/null +++ b/core/rawdb/freezer_meta.go @@ -0,0 +1,109 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package rawdb + +import ( + "io" + "os" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +const freezerVersion = 1 // The initial version tag of freezer table metadata + +// freezerTableMeta wraps all the metadata of the freezer table. +type freezerTableMeta struct { + // Version is the versioning descriptor of the freezer table. + Version uint16 + + // VirtualTail indicates how many items have been marked as deleted. + // Its value is equal to the number of items removed from the table + // plus the number of items hidden in the table, so it should never + // be lower than the "actual tail". + VirtualTail uint64 +} + +// newMetadata initializes the metadata object with the given virtual tail. +func newMetadata(tail uint64) *freezerTableMeta { + return &freezerTableMeta{ + Version: freezerVersion, + VirtualTail: tail, + } +} + +// readMetadata reads the metadata of the freezer table from the +// given metadata file. +func readMetadata(file *os.File) (*freezerTableMeta, error) { + _, err := file.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + var meta freezerTableMeta + if err := rlp.Decode(file, &meta); err != nil { + return nil, err + } + return &meta, nil +} + +// writeMetadata writes the metadata of the freezer table into the +// given metadata file. +func writeMetadata(file *os.File, meta *freezerTableMeta) error { + _, err := file.Seek(0, io.SeekStart) + if err != nil { + return err + } + return rlp.Encode(file, meta) +} + +// loadMetadata loads the metadata from the given metadata file. +// Initializes the metadata file with the given "actual tail" if +// it's empty. +func loadMetadata(file *os.File, tail uint64) (*freezerTableMeta, error) { + stat, err := file.Stat() + if err != nil { + return nil, err + } + // Write the metadata with the given actual tail into metadata file + // if it's non-existent. There are two possible scenarios here: + // - the freezer table is empty + // - the freezer table is legacy + // In both cases, write the meta into the file with the actual tail + // as the virtual tail. + if stat.Size() == 0 { + m := newMetadata(tail) + if err := writeMetadata(file, m); err != nil { + return nil, err + } + return m, nil + } + m, err := readMetadata(file) + if err != nil { + return nil, err + } + // Update the virtual tail with the given actual tail if it's even + // lower than it. Theoretically it shouldn't happen at all, print + // a warning here. + if m.VirtualTail < tail { + log.Warn("Updated virtual tail", "have", m.VirtualTail, "now", tail) + m.VirtualTail = tail + if err := writeMetadata(file, m); err != nil { + return nil, err + } + } + return m, nil +} diff --git a/core/rawdb/freezer_meta_test.go b/core/rawdb/freezer_meta_test.go new file mode 100644 index 000000000000..191744a75410 --- /dev/null +++ b/core/rawdb/freezer_meta_test.go @@ -0,0 +1,61 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package rawdb + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestReadWriteFreezerTableMeta(t *testing.T) { + f, err := ioutil.TempFile(os.TempDir(), "*") + if err != nil { + t.Fatalf("Failed to create file %v", err) + } + err = writeMetadata(f, newMetadata(100)) + if err != nil { + t.Fatalf("Failed to write metadata %v", err) + } + meta, err := readMetadata(f) + if err != nil { + t.Fatalf("Failed to read metadata %v", err) + } + if meta.Version != freezerVersion { + t.Fatalf("Unexpected version field") + } + if meta.VirtualTail != uint64(100) { + t.Fatalf("Unexpected virtual tail field") + } +} + +func TestInitializeFreezerTableMeta(t *testing.T) { + f, err := ioutil.TempFile(os.TempDir(), "*") + if err != nil { + t.Fatalf("Failed to create file %v", err) + } + meta, err := loadMetadata(f, uint64(100)) + if err != nil { + t.Fatalf("Failed to read metadata %v", err) + } + if meta.Version != freezerVersion { + t.Fatalf("Unexpected version field") + } + if meta.VirtualTail != uint64(100) { + t.Fatalf("Unexpected virtual tail field") + } +} diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 7cfba70c5004..01867ee8c242 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -47,20 +47,19 @@ var ( ) // indexEntry contains the number/id of the file that the data resides in, aswell as the -// offset within the file to the end of the data +// offset within the file to the end of the data. // In serialized form, the filenum is stored as uint16. type indexEntry struct { - filenum uint32 // stored as uint16 ( 2 bytes) - offset uint32 // stored as uint32 ( 4 bytes) + filenum uint32 // stored as uint16 ( 2 bytes ) + offset uint32 // stored as uint32 ( 4 bytes ) } const indexEntrySize = 6 // unmarshalBinary deserializes binary b into the rawIndex entry. -func (i *indexEntry) unmarshalBinary(b []byte) error { +func (i *indexEntry) unmarshalBinary(b []byte) { i.filenum = uint32(binary.BigEndian.Uint16(b[:2])) i.offset = binary.BigEndian.Uint32(b[2:6]) - return nil } // append adds the encoded entry to the end of b. @@ -75,14 +74,14 @@ func (i *indexEntry) append(b []byte) []byte { // bounds returns the start- and end- offsets, and the file number of where to // read there data item marked by the two index entries. The two entries are // assumed to be sequential. -func (start *indexEntry) bounds(end *indexEntry) (startOffset, endOffset, fileId uint32) { - if start.filenum != end.filenum { +func (i *indexEntry) bounds(end *indexEntry) (startOffset, endOffset, fileId uint32) { + if i.filenum != end.filenum { // If a piece of data 'crosses' a data-file, // it's actually in one piece on the second data-file. // We return a zero-indexEntry for the second file as start return 0, end.offset, end.filenum } - return start.offset, end.offset, end.filenum + return i.offset, end.offset, end.filenum } // freezerTable represents a single chained data table within the freezer (e.g. blocks). @@ -92,7 +91,15 @@ type freezerTable struct { // WARNING: The `items` field is accessed atomically. On 32 bit platforms, only // 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned, // so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG). - items uint64 // Number of items stored in the table (including items removed from tail) + items uint64 // Number of items stored in the table (including items removed from tail) + itemOffset uint64 // Number of items removed from the table + + // itemHidden is the number of items marked as deleted. Tail deletion is + // only supported at file level which means the actual deletion will be + // delayed until the entire data file is marked as deleted. Before that + // these items will be hidden to prevent being visited again. The value + // should never be lower than itemOffset. + itemHidden uint64 noCompression bool // if true, disables snappy compression. Note: does not work retroactively readonly bool @@ -101,14 +108,11 @@ type freezerTable struct { path string head *os.File // File descriptor for the data head of the table + index *os.File // File descriptor for the indexEntry file of the table + meta *os.File // File descriptor for metadata of the table files map[uint32]*os.File // open files headId uint32 // number of the currently active head file tailId uint32 // number of the earliest file - index *os.File // File descriptor for the indexEntry file of the table - - // In the case that old items are deleted (from the tail), we use itemOffset - // to count how many historic items have gone missing. - itemOffset uint32 // Offset (number of discarded items) headBytes int64 // Number of bytes written to the head file readMeter metrics.Meter // Meter for measuring the effective amount of data read @@ -124,46 +128,8 @@ func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerT return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly) } -// openFreezerFileForAppend opens a freezer table file and seeks to the end -func openFreezerFileForAppend(filename string) (*os.File, error) { - // Open the file without the O_APPEND flag - // because it has differing behaviour during Truncate operations - // on different OS's - file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return nil, err - } - // Seek to end for append - if _, err = file.Seek(0, io.SeekEnd); err != nil { - return nil, err - } - return file, nil -} - -// openFreezerFileForReadOnly opens a freezer table file for read only access -func openFreezerFileForReadOnly(filename string) (*os.File, error) { - return os.OpenFile(filename, os.O_RDONLY, 0644) -} - -// openFreezerFileTruncated opens a freezer table making sure it is truncated -func openFreezerFileTruncated(filename string) (*os.File, error) { - return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) -} - -// truncateFreezerFile resizes a freezer table file and seeks to the end -func truncateFreezerFile(file *os.File, size int64) error { - if err := file.Truncate(size); err != nil { - return err - } - // Seek to end for append - if _, err := file.Seek(0, io.SeekEnd); err != nil { - return err - } - return nil -} - // newTable opens a freezer table, creating the data and index files if they are -// non existent. Both files are truncated to the shortest common length to ensure +// non-existent. Both files are truncated to the shortest common length to ensure // they don't go out of sync. func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression, readonly bool) (*freezerTable, error) { // Ensure the containing directory exists and open the indexEntry file @@ -172,28 +138,47 @@ func newTable(path string, name string, readMeter metrics.Meter, writeMeter metr } var idxName string if noCompression { - // Raw idx - idxName = fmt.Sprintf("%s.ridx", name) + idxName = fmt.Sprintf("%s.ridx", name) // raw index file } else { - // Compressed idx - idxName = fmt.Sprintf("%s.cidx", name) + idxName = fmt.Sprintf("%s.cidx", name) // compressed index file } var ( - err error - offsets *os.File + err error + index *os.File + meta *os.File ) if readonly { // Will fail if table doesn't exist - offsets, err = openFreezerFileForReadOnly(filepath.Join(path, idxName)) + index, err = openFreezerFileForReadOnly(filepath.Join(path, idxName)) + if err != nil { + return nil, err + } + // TODO(rjl493456442) change it to read-only mode. Open the metadata file + // in rw mode. It's a temporary solution for now and should be changed + // whenever the tail deletion is actually used. The reason for this hack is + // the additional meta file for each freezer table is added in order to support + // tail deletion, but for most legacy nodes this file is missing. This check + // will suddenly break lots of database relevant commands. So the metadata file + // is always opened for mutation and nothing else will be written except + // the initialization. + meta, err = openFreezerFileForAppend(filepath.Join(path, fmt.Sprintf("%s.meta", name))) + if err != nil { + return nil, err + } } else { - offsets, err = openFreezerFileForAppend(filepath.Join(path, idxName)) - } - if err != nil { - return nil, err + index, err = openFreezerFileForAppend(filepath.Join(path, idxName)) + if err != nil { + return nil, err + } + meta, err = openFreezerFileForAppend(filepath.Join(path, fmt.Sprintf("%s.meta", name))) + if err != nil { + return nil, err + } } // Create the table and repair any past inconsistency tab := &freezerTable{ - index: offsets, + index: index, + meta: meta, files: make(map[uint32]*os.File), readMeter: readMeter, writeMeter: writeMeter, @@ -220,7 +205,7 @@ func newTable(path string, name string, readMeter metrics.Meter, writeMeter metr return tab, nil } -// repair cross checks the head and the index file and truncates them to +// repair cross-checks the head and the index file and truncates them to // be in sync with each other after a potential crash / data loss. func (t *freezerTable) repair() error { // Create a temporary offset buffer to init files with and read indexEntry into @@ -258,11 +243,27 @@ func (t *freezerTable) repair() error { t.index.ReadAt(buffer, 0) firstIndex.unmarshalBinary(buffer) + // Assign the tail fields with the first stored index. + // The total removed items is represented with an uint32, + // which is not enough in theory but enough in practice. + // TODO: use uint64 to represent total removed items. t.tailId = firstIndex.filenum - t.itemOffset = firstIndex.offset + t.itemOffset = uint64(firstIndex.offset) + + // Load metadata from the file + meta, err := loadMetadata(t.meta, t.itemOffset) + if err != nil { + return err + } + t.itemHidden = meta.VirtualTail - t.index.ReadAt(buffer, offsetsSize-indexEntrySize) - lastIndex.unmarshalBinary(buffer) + // Read the last index, use the default value in case the freezer is empty + if offsetsSize == indexEntrySize { + lastIndex = indexEntry{filenum: t.tailId, offset: 0} + } else { + t.index.ReadAt(buffer, offsetsSize-indexEntrySize) + lastIndex.unmarshalBinary(buffer) + } if t.readonly { t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForReadOnly) } else { @@ -278,7 +279,6 @@ func (t *freezerTable) repair() error { // Keep truncating both files until they come in sync contentExp = int64(lastIndex.offset) - for contentExp != contentSize { // Truncate the head file to the last offset pointer if contentExp < contentSize { @@ -295,9 +295,16 @@ func (t *freezerTable) repair() error { return err } offsetsSize -= indexEntrySize - t.index.ReadAt(buffer, offsetsSize-indexEntrySize) + + // Read the new head index, use the default value in case + // the freezer is already empty. var newLastIndex indexEntry - newLastIndex.unmarshalBinary(buffer) + if offsetsSize == indexEntrySize { + newLastIndex = indexEntry{filenum: t.tailId, offset: 0} + } else { + t.index.ReadAt(buffer, offsetsSize-indexEntrySize) + newLastIndex.unmarshalBinary(buffer) + } // We might have slipped back into an earlier head-file here if newLastIndex.filenum != lastIndex.filenum { // Release earlier opened file @@ -325,12 +332,21 @@ func (t *freezerTable) repair() error { if err := t.head.Sync(); err != nil { return err } + if err := t.meta.Sync(); err != nil { + return err + } } // Update the item and byte counters and return - t.items = uint64(t.itemOffset) + uint64(offsetsSize/indexEntrySize-1) // last indexEntry points to the end of the data file + t.items = t.itemOffset + uint64(offsetsSize/indexEntrySize-1) // last indexEntry points to the end of the data file t.headBytes = contentSize t.headId = lastIndex.filenum + // Delete the leftover files because of head deletion + t.releaseFilesAfter(t.headId, true) + + // Delete the leftover files because of tail deletion + t.releaseFilesBefore(t.tailId, true) + // Close opened files and preopen all files if err := t.preopen(); err != nil { return err @@ -346,6 +362,7 @@ func (t *freezerTable) repair() error { func (t *freezerTable) preopen() (err error) { // The repair might have already opened (some) files t.releaseFilesAfter(0, false) + // Open all except head in RDONLY for i := t.tailId; i < t.headId; i++ { if _, err = t.openFile(i, openFreezerFileForReadOnly); err != nil { @@ -361,16 +378,19 @@ func (t *freezerTable) preopen() (err error) { return err } -// truncate discards any recent data above the provided threshold number. -func (t *freezerTable) truncate(items uint64) error { +// truncateHead discards any recent data above the provided threshold number. +func (t *freezerTable) truncateHead(items uint64) error { t.lock.Lock() defer t.lock.Unlock() - // If our item count is correct, don't do anything + // Ensure the given truncate target falls in the correct range existing := atomic.LoadUint64(&t.items) if existing <= items { return nil } + if items < atomic.LoadUint64(&t.itemHidden) { + return errors.New("truncation below tail") + } // We need to truncate, save the old size for metrics tracking oldSize, err := t.sizeNolock() if err != nil { @@ -382,17 +402,24 @@ func (t *freezerTable) truncate(items uint64) error { log = t.logger.Warn // Only loud warn if we delete multiple items } log("Truncating freezer table", "items", existing, "limit", items) - if err := truncateFreezerFile(t.index, int64(items+1)*indexEntrySize); err != nil { + + // Truncate the index file first, the tail position is also considered + // when calculating the new freezer table length. + length := items - atomic.LoadUint64(&t.itemOffset) + if err := truncateFreezerFile(t.index, int64(length+1)*indexEntrySize); err != nil { return err } // Calculate the new expected size of the data file and truncate it - buffer := make([]byte, indexEntrySize) - if _, err := t.index.ReadAt(buffer, int64(items*indexEntrySize)); err != nil { - return err - } var expected indexEntry - expected.unmarshalBinary(buffer) - + if length == 0 { + expected = indexEntry{filenum: t.tailId, offset: 0} + } else { + buffer := make([]byte, indexEntrySize) + if _, err := t.index.ReadAt(buffer, int64(length*indexEntrySize)); err != nil { + return err + } + expected.unmarshalBinary(buffer) + } // We might need to truncate back to older files if expected.filenum != t.headId { // If already open for reading, force-reopen for writing @@ -421,7 +448,110 @@ func (t *freezerTable) truncate(items uint64) error { return err } t.sizeGauge.Dec(int64(oldSize - newSize)) + return nil +} + +// truncateTail discards any recent data before the provided threshold number. +func (t *freezerTable) truncateTail(items uint64) error { + t.lock.Lock() + defer t.lock.Unlock() + + // Ensure the given truncate target falls in the correct range + if atomic.LoadUint64(&t.itemHidden) >= items { + return nil + } + if atomic.LoadUint64(&t.items) < items { + return errors.New("truncation above head") + } + // Load the new tail index by the given new tail position + var ( + newTailId uint32 + buffer = make([]byte, indexEntrySize) + ) + if atomic.LoadUint64(&t.items) == items { + newTailId = t.headId + } else { + offset := items - atomic.LoadUint64(&t.itemOffset) + if _, err := t.index.ReadAt(buffer, int64((offset+1)*indexEntrySize)); err != nil { + return err + } + var newTail indexEntry + newTail.unmarshalBinary(buffer) + newTailId = newTail.filenum + } + // Update the virtual tail marker and hidden these entries in table. + atomic.StoreUint64(&t.itemHidden, items) + if err := writeMetadata(t.meta, newMetadata(items)); err != nil { + return err + } + // Hidden items still fall in the current tail file, no data file + // can be dropped. + if t.tailId == newTailId { + return nil + } + // Hidden items fall in the incorrect range, returns the error. + if t.tailId > newTailId { + return fmt.Errorf("invalid index, tail-file %d, item-file %d", t.tailId, newTailId) + } + // Hidden items exceed the current tail file, drop the relevant + // data files. We need to truncate, save the old size for metrics + // tracking. + oldSize, err := t.sizeNolock() + if err != nil { + return err + } + // Count how many items can be deleted from the file. + var ( + newDeleted = items + deleted = atomic.LoadUint64(&t.itemOffset) + ) + for current := items - 1; current >= deleted; current -= 1 { + if _, err := t.index.ReadAt(buffer, int64((current-deleted+1)*indexEntrySize)); err != nil { + return err + } + var pre indexEntry + pre.unmarshalBinary(buffer) + if pre.filenum != newTailId { + break + } + newDeleted = current + } + // Commit the changes of metadata file first before manipulating + // the indexes file. + if err := t.meta.Sync(); err != nil { + return err + } + // Truncate the deleted index entries from the index file. + err = copyFrom(t.index.Name(), t.index.Name(), indexEntrySize*(newDeleted-deleted+1), func(f *os.File) error { + tailIndex := indexEntry{ + filenum: newTailId, + offset: uint32(newDeleted), + } + _, err := f.Write(tailIndex.append(nil)) + return err + }) + if err != nil { + return err + } + // Reopen the modified index file to load the changes + if err := t.index.Close(); err != nil { + return err + } + t.index, err = openFreezerFileForAppend(t.index.Name()) + if err != nil { + return err + } + // Release any files before the current tail + t.tailId = newTailId + atomic.StoreUint64(&t.itemOffset, newDeleted) + t.releaseFilesBefore(t.tailId, true) + // Retrieve the new size and update the total size counter + newSize, err := t.sizeNolock() + if err != nil { + return err + } + t.sizeGauge.Dec(int64(oldSize - newSize)) return nil } @@ -436,6 +566,11 @@ func (t *freezerTable) Close() error { } t.index = nil + if err := t.meta.Close(); err != nil { + errs = append(errs, err) + } + t.meta = nil + for _, f := range t.files { if err := f.Close(); err != nil { errs = append(errs, err) @@ -490,6 +625,19 @@ func (t *freezerTable) releaseFilesAfter(num uint32, remove bool) { } } +// releaseFilesBefore closes all open files with a lower number, and optionally also deletes the files +func (t *freezerTable) releaseFilesBefore(num uint32, remove bool) { + for fnum, f := range t.files { + if fnum < num { + delete(t.files, fnum) + f.Close() + if remove { + os.Remove(f.Name()) + } + } + } +} + // getIndices returns the index entries for the given from-item, covering 'count' items. // N.B: The actual number of returned indices for N items will always be N+1 (unless an // error is returned). @@ -498,7 +646,7 @@ func (t *freezerTable) releaseFilesAfter(num uint32, remove bool) { // it will return error. func (t *freezerTable) getIndices(from, count uint64) ([]*indexEntry, error) { // Apply the table-offset - from = from - uint64(t.itemOffset) + from = from - t.itemOffset // For reading N items, we need N+1 indices. buffer := make([]byte, (count+1)*indexEntrySize) if _, err := t.index.ReadAt(buffer, int64(from*indexEntrySize)); err != nil { @@ -583,18 +731,21 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i t.lock.RLock() defer t.lock.RUnlock() - // Ensure the table and the item is accessible + // Ensure the table and the item are accessible if t.index == nil || t.head == nil { return nil, nil, errClosed } - itemCount := atomic.LoadUint64(&t.items) // max number + var ( + items = atomic.LoadUint64(&t.items) // the total items(head + 1) + hidden = atomic.LoadUint64(&t.itemHidden) // the number of hidden items + ) // Ensure the start is written, not deleted from the tail, and that the // caller actually wants something - if itemCount <= start || uint64(t.itemOffset) > start || count == 0 { + if items <= start || hidden > start || count == 0 { return nil, nil, errOutOfBounds } - if start+count > itemCount { - count = itemCount - start + if start+count > items { + count = items - start } var ( output = make([]byte, maxBytes) // Buffer to read data into @@ -670,10 +821,10 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i return output[:outputSize], sizes, nil } -// has returns an indicator whether the specified number data -// exists in the freezer table. +// has returns an indicator whether the specified number data is still accessible +// in the freezer table. func (t *freezerTable) has(number uint64) bool { - return atomic.LoadUint64(&t.items) > number + return atomic.LoadUint64(&t.items) > number && atomic.LoadUint64(&t.itemHidden) <= number } // size returns the total data size in the freezer table. @@ -727,6 +878,9 @@ func (t *freezerTable) Sync() error { if err := t.index.Sync(); err != nil { return err } + if err := t.meta.Sync(); err != nil { + return err + } return t.head.Sync() } @@ -744,13 +898,20 @@ func (t *freezerTable) dumpIndexString(start, stop int64) string { } func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { + meta, err := readMetadata(t.meta) + if err != nil { + fmt.Fprintf(w, "Failed to decode freezer table %v\n", err) + return + } + fmt.Fprintf(w, "Version %d deleted %d, hidden %d\n", meta.Version, atomic.LoadUint64(&t.itemOffset), atomic.LoadUint64(&t.itemHidden)) + buf := make([]byte, indexEntrySize) fmt.Fprintf(w, "| number | fileno | offset |\n") fmt.Fprintf(w, "|--------|--------|--------|\n") for i := uint64(start); ; i++ { - if _, err := t.index.ReadAt(buf, int64(i*indexEntrySize)); err != nil { + if _, err := t.index.ReadAt(buf, int64((i+1)*indexEntrySize)); err != nil { break } var entry indexEntry diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 15464e1bd768..0bddcf721136 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -18,13 +18,18 @@ package rawdb import ( "bytes" + "encoding/binary" "fmt" "math/rand" "os" "path/filepath" + "reflect" + "sync/atomic" "testing" + "testing/quick" "time" + "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/metrics" "github.com/stretchr/testify/require" ) @@ -204,7 +209,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) { } // Remove everything but the first item, and leave data unaligned // 0-indexEntry, 1-indexEntry, corrupt-indexEntry - idxFile.Truncate(indexEntrySize + indexEntrySize + indexEntrySize/2) + idxFile.Truncate(2*indexEntrySize + indexEntrySize/2) idxFile.Close() // Now open it again @@ -387,7 +392,7 @@ func TestFreezerTruncate(t *testing.T) { t.Fatal(err) } defer f.Close() - f.truncate(10) // 150 bytes + f.truncateHead(10) // 150 bytes if f.items != 10 { t.Fatalf("expected %d items, got %d", 10, f.items) } @@ -504,7 +509,7 @@ func TestFreezerReadAndTruncate(t *testing.T) { } // Now, truncate back to zero - f.truncate(0) + f.truncateHead(0) // Write the data again batch := f.newBatch() @@ -565,18 +570,19 @@ func TestFreezerOffset(t *testing.T) { // Update the index file, so that we store // [ file = 2, offset = 4 ] at index zero - tailId := uint32(2) // First file is 2 - itemOffset := uint32(4) // We have removed four items zeroIndex := indexEntry{ - filenum: tailId, - offset: itemOffset, + filenum: uint32(2), // First file is 2 + offset: uint32(4), // We have removed four items } buf := zeroIndex.append(nil) + // Overwrite index zero copy(indexBuf, buf) + // Remove the four next indices by overwriting copy(indexBuf[indexEntrySize:], indexBuf[indexEntrySize*5:]) indexFile.WriteAt(indexBuf, 0) + // Need to truncate the moved index items indexFile.Truncate(indexEntrySize * (1 + 2)) indexFile.Close() @@ -623,13 +629,12 @@ func TestFreezerOffset(t *testing.T) { // Update the index file, so that we store // [ file = 2, offset = 1M ] at index zero - tailId := uint32(2) // First file is 2 - itemOffset := uint32(1000000) // We have removed 1M items zeroIndex := indexEntry{ - offset: itemOffset, - filenum: tailId, + offset: uint32(1000000), // We have removed 1M items + filenum: uint32(2), // First file is 2 } buf := zeroIndex.append(nil) + // Overwrite index zero copy(indexBuf, buf) indexFile.WriteAt(indexBuf, 0) @@ -659,6 +664,171 @@ func TestFreezerOffset(t *testing.T) { } } +func TestTruncateTail(t *testing.T) { + t.Parallel() + rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() + fname := fmt.Sprintf("truncate-tail-%d", rand.Uint64()) + + // Fill table + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + if err != nil { + t.Fatal(err) + } + + // Write 7 x 20 bytes, splitting out into four files + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) + require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) + require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd))) + require.NoError(t, batch.AppendRaw(3, getChunk(20, 0xcc))) + require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb))) + require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa))) + require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x11))) + require.NoError(t, batch.commit()) + + // nothing to do, all the items should still be there. + f.truncateTail(0) + fmt.Println(f.dumpIndexString(0, 1000)) + checkRetrieve(t, f, map[uint64][]byte{ + 0: getChunk(20, 0xFF), + 1: getChunk(20, 0xEE), + 2: getChunk(20, 0xdd), + 3: getChunk(20, 0xcc), + 4: getChunk(20, 0xbb), + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x11), + }) + + // truncate single element( item 0 ), deletion is only supported at file level + f.truncateTail(1) + fmt.Println(f.dumpIndexString(0, 1000)) + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + }) + checkRetrieve(t, f, map[uint64][]byte{ + 1: getChunk(20, 0xEE), + 2: getChunk(20, 0xdd), + 3: getChunk(20, 0xcc), + 4: getChunk(20, 0xbb), + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x11), + }) + + // Reopen the table, the deletion information should be persisted as well + f.Close() + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + if err != nil { + t.Fatal(err) + } + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + }) + checkRetrieve(t, f, map[uint64][]byte{ + 1: getChunk(20, 0xEE), + 2: getChunk(20, 0xdd), + 3: getChunk(20, 0xcc), + 4: getChunk(20, 0xbb), + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x11), + }) + + // truncate two elements( item 0, item 1 ), the file 0 should be deleted + f.truncateTail(2) + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + 1: errOutOfBounds, + }) + checkRetrieve(t, f, map[uint64][]byte{ + 2: getChunk(20, 0xdd), + 3: getChunk(20, 0xcc), + 4: getChunk(20, 0xbb), + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x11), + }) + + // Reopen the table, the above testing should still pass + f.Close() + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + 1: errOutOfBounds, + }) + checkRetrieve(t, f, map[uint64][]byte{ + 2: getChunk(20, 0xdd), + 3: getChunk(20, 0xcc), + 4: getChunk(20, 0xbb), + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x11), + }) + + // truncate all, the entire freezer should be deleted + f.truncateTail(7) + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + 1: errOutOfBounds, + 2: errOutOfBounds, + 3: errOutOfBounds, + 4: errOutOfBounds, + 5: errOutOfBounds, + 6: errOutOfBounds, + }) +} + +func TestTruncateHead(t *testing.T) { + t.Parallel() + rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() + fname := fmt.Sprintf("truncate-head-blow-tail-%d", rand.Uint64()) + + // Fill table + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + if err != nil { + t.Fatal(err) + } + + // Write 7 x 20 bytes, splitting out into four files + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) + require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) + require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd))) + require.NoError(t, batch.AppendRaw(3, getChunk(20, 0xcc))) + require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb))) + require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa))) + require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x11))) + require.NoError(t, batch.commit()) + + f.truncateTail(4) // Tail = 4 + + // NewHead is required to be 3, the entire table should be truncated + f.truncateHead(4) + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, // Deleted by tail + 1: errOutOfBounds, // Deleted by tail + 2: errOutOfBounds, // Deleted by tail + 3: errOutOfBounds, // Deleted by tail + 4: errOutOfBounds, // Deleted by Head + 5: errOutOfBounds, // Deleted by Head + 6: errOutOfBounds, // Deleted by Head + }) + + // Append new items + batch = f.newBatch() + require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb))) + require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa))) + require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x11))) + require.NoError(t, batch.commit()) + + checkRetrieve(t, f, map[uint64][]byte{ + 4: getChunk(20, 0xbb), + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x11), + }) +} + func checkRetrieve(t *testing.T, f *freezerTable, items map[uint64][]byte) { t.Helper() @@ -915,3 +1085,212 @@ func TestFreezerReadonly(t *testing.T) { t.Fatalf("Writing to readonly table should fail") } } + +// randTest performs random freezer table operations. +// Instances of this test are created by Generate. +type randTest []randTestStep + +type randTestStep struct { + op int + items []uint64 // for append and retrieve + blobs [][]byte // for append + target uint64 // for truncate(head/tail) + err error // for debugging +} + +const ( + opReload = iota + opAppend + opRetrieve + opTruncateHead + opTruncateHeadAll + opTruncateTail + opTruncateTailAll + opCheckAll + opMax // boundary value, not an actual op +) + +func getVals(first uint64, n int) [][]byte { + var ret [][]byte + for i := 0; i < n; i++ { + val := make([]byte, 8) + binary.BigEndian.PutUint64(val, first+uint64(i)) + ret = append(ret, val) + } + return ret +} + +func (randTest) Generate(r *rand.Rand, size int) reflect.Value { + var ( + deleted uint64 // The number of deleted items from tail + items []uint64 // The index of entries in table + + // getItems retrieves the indexes for items in table. + getItems = func(n int) []uint64 { + length := len(items) + if length == 0 { + return nil + } + var ret []uint64 + index := rand.Intn(length) + for i := index; len(ret) < n && i < length; i++ { + ret = append(ret, items[i]) + } + return ret + } + + // addItems appends the given length items into the table. + addItems = func(n int) []uint64 { + var first = deleted + if len(items) != 0 { + first = items[len(items)-1] + 1 + } + var ret []uint64 + for i := 0; i < n; i++ { + ret = append(ret, first+uint64(i)) + } + items = append(items, ret...) + return ret + } + ) + + var steps randTest + for i := 0; i < size; i++ { + step := randTestStep{op: r.Intn(opMax)} + switch step.op { + case opReload, opCheckAll: + case opAppend: + num := r.Intn(3) + step.items = addItems(num) + if len(step.items) == 0 { + step.blobs = nil + } else { + step.blobs = getVals(step.items[0], num) + } + case opRetrieve: + step.items = getItems(r.Intn(3)) + case opTruncateHead: + if len(items) == 0 { + step.target = deleted + } else { + index := r.Intn(len(items)) + items = items[:index] + step.target = deleted + uint64(index) + } + case opTruncateHeadAll: + step.target = deleted + items = items[:0] + case opTruncateTail: + if len(items) == 0 { + step.target = deleted + } else { + index := r.Intn(len(items)) + items = items[index:] + deleted += uint64(index) + step.target = deleted + } + case opTruncateTailAll: + step.target = deleted + uint64(len(items)) + items = items[:0] + deleted = step.target + } + steps = append(steps, step) + } + return reflect.ValueOf(steps) +} + +func runRandTest(rt randTest) bool { + fname := fmt.Sprintf("randtest-%d", rand.Uint64()) + f, err := newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false) + if err != nil { + panic("failed to initialize table") + } + var values [][]byte + for i, step := range rt { + switch step.op { + case opReload: + f.Close() + f, err = newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false) + if err != nil { + rt[i].err = fmt.Errorf("failed to reload table %v", err) + } + case opCheckAll: + tail := atomic.LoadUint64(&f.itemHidden) + head := atomic.LoadUint64(&f.items) + + if tail == head { + continue + } + got, err := f.RetrieveItems(atomic.LoadUint64(&f.itemHidden), head-tail, 100000) + if err != nil { + rt[i].err = err + } else { + if !reflect.DeepEqual(got, values) { + rt[i].err = fmt.Errorf("mismatch on retrieved values %v %v", got, values) + } + } + + case opAppend: + batch := f.newBatch() + for i := 0; i < len(step.items); i++ { + batch.AppendRaw(step.items[i], step.blobs[i]) + } + batch.commit() + values = append(values, step.blobs...) + + case opRetrieve: + var blobs [][]byte + if len(step.items) == 0 { + continue + } + tail := atomic.LoadUint64(&f.itemHidden) + for i := 0; i < len(step.items); i++ { + blobs = append(blobs, values[step.items[i]-tail]) + } + got, err := f.RetrieveItems(step.items[0], uint64(len(step.items)), 100000) + if err != nil { + rt[i].err = err + } else { + if !reflect.DeepEqual(got, blobs) { + rt[i].err = fmt.Errorf("mismatch on retrieved values %v %v %v", got, blobs, step.items) + } + } + + case opTruncateHead: + f.truncateHead(step.target) + + length := atomic.LoadUint64(&f.items) - atomic.LoadUint64(&f.itemHidden) + values = values[:length] + + case opTruncateHeadAll: + f.truncateHead(step.target) + values = nil + + case opTruncateTail: + prev := atomic.LoadUint64(&f.itemHidden) + f.truncateTail(step.target) + + truncated := atomic.LoadUint64(&f.itemHidden) - prev + values = values[truncated:] + + case opTruncateTailAll: + f.truncateTail(step.target) + values = nil + } + // Abort the test on error. + if rt[i].err != nil { + return false + } + } + f.Close() + return true +} + +func TestRandom(t *testing.T) { + if err := quick.Check(runRandTest, nil); err != nil { + if cerr, ok := err.(*quick.CheckError); ok { + t.Fatalf("random test iteration %d failed: %s", cerr.Count, spew.Sdump(cerr.In)) + } + t.Fatal(err) + } +} diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index d5c3749e5d21..b3fd3059e750 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -24,6 +24,7 @@ import ( "math/big" "math/rand" "os" + "path" "sync" "testing" @@ -186,7 +187,7 @@ func TestFreezerConcurrentModifyRetrieve(t *testing.T) { wg.Wait() } -// This test runs ModifyAncients and TruncateAncients concurrently with each other. +// This test runs ModifyAncients and TruncateHead concurrently with each other. func TestFreezerConcurrentModifyTruncate(t *testing.T) { f, dir := newFreezerForTesting(t, freezerTestTableDef) defer os.RemoveAll(dir) @@ -196,7 +197,7 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) { for i := 0; i < 1000; i++ { // First reset and write 100 items. - if err := f.TruncateAncients(0); err != nil { + if err := f.TruncateHead(0); err != nil { t.Fatal("truncate failed:", err) } _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { @@ -231,7 +232,7 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) { wg.Done() }() go func() { - truncateErr = f.TruncateAncients(10) + truncateErr = f.TruncateHead(10) wg.Done() }() go func() { @@ -337,3 +338,92 @@ func checkAncientCount(t *testing.T, f *freezer, kind string, n uint64) { t.Errorf("Ancient(%q, %d) returned unexpected error %q", kind, index, err) } } + +func TestRenameWindows(t *testing.T) { + var ( + fname = "file.bin" + fname2 = "file2.bin" + data = []byte{1, 2, 3, 4} + data2 = []byte{2, 3, 4, 5} + data3 = []byte{3, 5, 6, 7} + dataLen = 4 + ) + + // Create 2 temp dirs + dir1, err := os.MkdirTemp("", "rename-test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(dir1) + dir2, err := os.MkdirTemp("", "rename-test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(dir2) + + // Create file in dir1 and fill with data + f, err := os.Create(path.Join(dir1, fname)) + if err != nil { + t.Fatal(err) + } + f2, err := os.Create(path.Join(dir1, fname2)) + if err != nil { + t.Fatal(err) + } + f3, err := os.Create(path.Join(dir2, fname2)) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write(data); err != nil { + t.Fatal(err) + } + if _, err := f2.Write(data2); err != nil { + t.Fatal(err) + } + if _, err := f3.Write(data3); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + if err := f2.Close(); err != nil { + t.Fatal(err) + } + if err := f3.Close(); err != nil { + t.Fatal(err) + } + if err := os.Rename(f.Name(), path.Join(dir2, fname)); err != nil { + t.Fatal(err) + } + if err := os.Rename(f2.Name(), path.Join(dir2, fname2)); err != nil { + t.Fatal(err) + } + + // Check file contents + f, err = os.Open(path.Join(dir2, fname)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + buf := make([]byte, dataLen) + if _, err := f.Read(buf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf, data) { + t.Errorf("unexpected file contents. Got %v\n", buf) + } + + f, err = os.Open(path.Join(dir2, fname2)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + if _, err := f.Read(buf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf, data2) { + t.Errorf("unexpected file contents. Got %v\n", buf) + } +} diff --git a/core/rawdb/freezer_utils.go b/core/rawdb/freezer_utils.go new file mode 100644 index 000000000000..5695fc0fa891 --- /dev/null +++ b/core/rawdb/freezer_utils.go @@ -0,0 +1,120 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// copyFrom copies data from 'srcPath' at offset 'offset' into 'destPath'. +// The 'destPath' is created if it doesn't exist, otherwise it is overwritten. +// Before the copy is executed, there is a callback can be registered to +// manipulate the dest file. +// It is perfectly valid to have destPath == srcPath. +func copyFrom(srcPath, destPath string, offset uint64, before func(f *os.File) error) error { + // Create a temp file in the same dir where we want it to wind up + f, err := ioutil.TempFile(filepath.Dir(destPath), "*") + if err != nil { + return err + } + fname := f.Name() + + // Clean up the leftover file + defer func() { + if f != nil { + f.Close() + } + os.Remove(fname) + }() + // Apply the given function if it's not nil before we copy + // the content from the src. + if before != nil { + if err := before(f); err != nil { + return err + } + } + // Open the source file + src, err := os.Open(srcPath) + if err != nil { + return err + } + if _, err = src.Seek(int64(offset), 0); err != nil { + src.Close() + return err + } + // io.Copy uses 32K buffer internally. + _, err = io.Copy(f, src) + if err != nil { + src.Close() + return err + } + // Rename the temporary file to the specified dest name. + // src may be same as dest, so needs to be closed before + // we do the final move. + src.Close() + + if err := f.Close(); err != nil { + return err + } + f = nil + + if err := os.Rename(fname, destPath); err != nil { + return err + } + return nil +} + +// openFreezerFileForAppend opens a freezer table file and seeks to the end +func openFreezerFileForAppend(filename string) (*os.File, error) { + // Open the file without the O_APPEND flag + // because it has differing behaviour during Truncate operations + // on different OS's + file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return nil, err + } + // Seek to end for append + if _, err = file.Seek(0, io.SeekEnd); err != nil { + return nil, err + } + return file, nil +} + +// openFreezerFileForReadOnly opens a freezer table file for read only access +func openFreezerFileForReadOnly(filename string) (*os.File, error) { + return os.OpenFile(filename, os.O_RDONLY, 0644) +} + +// openFreezerFileTruncated opens a freezer table making sure it is truncated +func openFreezerFileTruncated(filename string) (*os.File, error) { + return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) +} + +// truncateFreezerFile resizes a freezer table file and seeks to the end +func truncateFreezerFile(file *os.File, size int64) error { + if err := file.Truncate(size); err != nil { + return err + } + // Seek to end for append + if _, err := file.Seek(0, io.SeekEnd); err != nil { + return err + } + return nil +} diff --git a/core/rawdb/freezer_utils_test.go b/core/rawdb/freezer_utils_test.go new file mode 100644 index 000000000000..de8087f9b936 --- /dev/null +++ b/core/rawdb/freezer_utils_test.go @@ -0,0 +1,76 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +func TestCopyFrom(t *testing.T) { + var ( + content = []byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8} + prefix = []byte{0x9, 0xa, 0xb, 0xc, 0xd, 0xf} + ) + var cases = []struct { + src, dest string + offset uint64 + writePrefix bool + }{ + {"foo", "bar", 0, false}, + {"foo", "bar", 1, false}, + {"foo", "bar", 8, false}, + {"foo", "foo", 0, false}, + {"foo", "foo", 1, false}, + {"foo", "foo", 8, false}, + {"foo", "bar", 0, true}, + {"foo", "bar", 1, true}, + {"foo", "bar", 8, true}, + } + for _, c := range cases { + ioutil.WriteFile(c.src, content, 0644) + + if err := copyFrom(c.src, c.dest, c.offset, func(f *os.File) error { + if !c.writePrefix { + return nil + } + f.Write(prefix) + return nil + }); err != nil { + os.Remove(c.src) + t.Fatalf("Failed to copy %v", err) + } + + blob, err := ioutil.ReadFile(c.dest) + if err != nil { + os.Remove(c.src) + os.Remove(c.dest) + t.Fatalf("Failed to read %v", err) + } + want := content[c.offset:] + if c.writePrefix { + want = append(prefix, want...) + } + if !bytes.Equal(blob, want) { + t.Fatal("Unexpected value") + } + os.Remove(c.src) + os.Remove(c.dest) + } +} diff --git a/core/rawdb/key_length_iterator.go b/core/rawdb/key_length_iterator.go new file mode 100644 index 000000000000..d1c5af269a31 --- /dev/null +++ b/core/rawdb/key_length_iterator.go @@ -0,0 +1,47 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import "github.com/ethereum/go-ethereum/ethdb" + +// KeyLengthIterator is a wrapper for a database iterator that ensures only key-value pairs +// with a specific key length will be returned. +type KeyLengthIterator struct { + requiredKeyLength int + ethdb.Iterator +} + +// NewKeyLengthIterator returns a wrapped version of the iterator that will only return key-value +// pairs where keys with a specific key length will be returned. +func NewKeyLengthIterator(it ethdb.Iterator, keyLen int) ethdb.Iterator { + return &KeyLengthIterator{ + Iterator: it, + requiredKeyLength: keyLen, + } +} + +func (it *KeyLengthIterator) Next() bool { + // Return true as soon as a key with the required key length is discovered + for it.Iterator.Next() { + if len(it.Iterator.Key()) == it.requiredKeyLength { + return true + } + } + + // Return false when we exhaust the keys in the underlying iterator. + return false +} diff --git a/core/rawdb/key_length_iterator_test.go b/core/rawdb/key_length_iterator_test.go new file mode 100644 index 000000000000..654efc5b55b5 --- /dev/null +++ b/core/rawdb/key_length_iterator_test.go @@ -0,0 +1,60 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "encoding/binary" + "testing" +) + +func TestKeyLengthIterator(t *testing.T) { + db := NewMemoryDatabase() + + keyLen := 8 + expectedKeys := make(map[string]struct{}) + for i := 0; i < 100; i++ { + key := make([]byte, keyLen) + binary.BigEndian.PutUint64(key, uint64(i)) + if err := db.Put(key, []byte{0x1}); err != nil { + t.Fatal(err) + } + expectedKeys[string(key)] = struct{}{} + + longerKey := make([]byte, keyLen*2) + binary.BigEndian.PutUint64(longerKey, uint64(i)) + if err := db.Put(longerKey, []byte{0x1}); err != nil { + t.Fatal(err) + } + } + + it := NewKeyLengthIterator(db.NewIterator(nil, nil), keyLen) + for it.Next() { + key := it.Key() + _, exists := expectedKeys[string(key)] + if !exists { + t.Fatalf("Found unexpected key %d", binary.BigEndian.Uint64(key)) + } + delete(expectedKeys, string(key)) + if len(key) != keyLen { + t.Fatalf("Found unexpected key in key length iterator with length %d", len(key)) + } + } + + if len(expectedKeys) != 0 { + t.Fatalf("Expected all keys of length %d to be removed from expected keys during iteration", keyLen) + } +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index b35fcba45f79..08f373488056 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -63,6 +63,9 @@ var ( // snapshotSyncStatusKey tracks the snapshot sync status across restarts. snapshotSyncStatusKey = []byte("SnapshotSyncStatus") + // skeletonSyncStatusKey tracks the skeleton sync status across restarts. + skeletonSyncStatusKey = []byte("SkeletonSyncStatus") + // txIndexTailKey tracks the oldest block whose transactions have been indexed. txIndexTailKey = []byte("TransactionIndexTail") @@ -92,9 +95,11 @@ var ( SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value CodePrefix = []byte("c") // CodePrefix + code hash -> account code + skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header - PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage - configPrefix = []byte("ethereum-config-") // config prefix for the db + PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage + configPrefix = []byte("ethereum-config-") // config prefix for the db + genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db // Chain index prefixes (use `i` + single byte to avoid mixing data types). BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress @@ -210,6 +215,11 @@ func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte { return key } +// skeletonHeaderKey = skeletonHeaderPrefix + num (uint64 big endian) +func skeletonHeaderKey(number uint64) []byte { + return append(skeletonHeaderPrefix, encodeBlockNumber(number)...) +} + // preimageKey = PreimagePrefix + hash func preimageKey(hash common.Hash) []byte { return append(PreimagePrefix, hash.Bytes()...) @@ -233,3 +243,8 @@ func IsCodeKey(key []byte) (bool, []byte) { func configKey(hash common.Hash) []byte { return append(configPrefix, hash.Bytes()...) } + +// genesisKey = genesisPrefix + hash +func genesisKey(hash common.Hash) []byte { + return append(genesisPrefix, hash.Bytes()...) +} diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 91fc31b660d6..5eadf5f7c159 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -74,6 +74,12 @@ func (t *table) Ancients() (uint64, error) { return t.db.Ancients() } +// Tail is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Tail() (uint64, error) { + return t.db.Tail() +} + // AncientSize is a noop passthrough that just forwards the request to the underlying // database. func (t *table) AncientSize(kind string) (uint64, error) { @@ -89,10 +95,16 @@ func (t *table) ReadAncients(fn func(reader ethdb.AncientReader) error) (err err return t.db.ReadAncients(fn) } -// TruncateAncients is a noop passthrough that just forwards the request to the underlying +// TruncateHead is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) TruncateHead(items uint64) error { + return t.db.TruncateHead(items) +} + +// TruncateTail is a noop passthrough that just forwards the request to the underlying // database. -func (t *table) TruncateAncients(items uint64) error { - return t.db.TruncateAncients(items) +func (t *table) TruncateTail(items uint64) error { + return t.db.TruncateTail(items) } // Sync is a noop passthrough that just forwards the request to the underlying @@ -101,6 +113,12 @@ func (t *table) Sync() error { return t.db.Sync() } +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (t *table) MigrateTable(kind string, convert convertLegacyFn) error { + return t.db.MigrateTable(kind, convert) +} + // Put inserts the given value into the database at a prefixed version of the // provided key. func (t *table) Put(key []byte, value []byte) error { @@ -172,6 +190,18 @@ func (t *table) NewBatch() ethdb.Batch { return &tableBatch{t.db.NewBatch(), t.prefix} } +// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. +func (t *table) NewBatchWithSize(size int) ethdb.Batch { + return &tableBatch{t.db.NewBatchWithSize(size), t.prefix} +} + +// NewSnapshot creates a database snapshot based on the current state. +// The created snapshot will not be affected by all following mutations +// happened on the database. +func (t *table) NewSnapshot() (ethdb.Snapshot, error) { + return t.db.NewSnapshot() +} + // tableBatch is a wrapper around a database batch that prefixes each key access // with a pre-configured string. type tableBatch struct { diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 37772ca35c55..4e3daac669b4 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -265,7 +265,7 @@ func (p *Pruner) Prune(root common.Hash) error { // Ensure the root is really present. The weak assumption // is the presence of root can indicate the presence of the // entire trie. - if blob := rawdb.ReadTrieNode(p.db, root); len(blob) == 0 { + if !rawdb.HasTrieNode(p.db, root) { // The special case is for clique based networks(rinkeby, goerli // and some other private networks), it's possible that two // consecutive blocks will have same root. In this case snapshot @@ -279,7 +279,7 @@ func (p *Pruner) Prune(root common.Hash) error { // as the pruning target. var found bool for i := len(layers) - 2; i >= 2; i-- { - if blob := rawdb.ReadTrieNode(p.db, layers[i].Root()); len(blob) != 0 { + if rawdb.HasTrieNode(p.db, layers[i].Root()) { root = layers[i].Root() found = true log.Info("Selecting middle-layer as the pruning target", "root", root, "depth", i) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 6ee6b06bb5f2..76200851e469 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -546,20 +546,19 @@ func diffToDisk(bottom *diffLayer) *diskLayer { it := rawdb.IterateStorageSnapshots(base.diskdb, hash) for it.Next() { - if key := it.Key(); len(key) == 65 { // TODO(karalabe): Yuck, we should move this into the iterator - batch.Delete(key) - base.cache.Del(key[1:]) - snapshotFlushStorageItemMeter.Mark(1) - - // Ensure we don't delete too much data blindly (contract can be - // huge). It's ok to flush, the root will go missing in case of a - // crash and we'll detect and regenerate the snapshot. - if batch.ValueSize() > ethdb.IdealBatchSize { - if err := batch.Write(); err != nil { - log.Crit("Failed to write storage deletions", "err", err) - } - batch.Reset() + key := it.Key() + batch.Delete(key) + base.cache.Del(key[1:]) + snapshotFlushStorageItemMeter.Mark(1) + + // Ensure we don't delete too much data blindly (contract can be + // huge). It's ok to flush, the root will go missing in case of a + // crash and we'll detect and regenerate the snapshot. + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + log.Crit("Failed to write storage deletions", "err", err) } + batch.Reset() } } it.Release() diff --git a/core/state/statedb.go b/core/state/statedb.go index 4ff2c3e48da0..1d31cf470be0 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -884,7 +884,6 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { func (s *StateDB) Prepare(thash common.Hash, ti int) { s.thash = thash s.txIndex = ti - s.accessList = newAccessList() } func (s *StateDB) clearJournalAndRefund() { @@ -994,6 +993,9 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // // This method should only be called if Berlin/2929+2930 is applicable at the current number. func (s *StateDB) PrepareAccessList(sender common.Address, dst *common.Address, precompiles []common.Address, list types.AccessList) { + // Clear out any leftover from previous executions + s.accessList = newAccessList() + s.AddAddressToAccessList(sender) if dst != nil { s.AddAddressToAccessList(*dst) diff --git a/core/types/access_list_tx.go b/core/types/access_list_tx.go index ee5f194b77b8..8ad5e739e9eb 100644 --- a/core/types/access_list_tx.go +++ b/core/types/access_list_tx.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" ) -//go:generate gencodec -type AccessTuple -out gen_access_tuple.go +//go:generate go run github.com/fjl/gencodec@latest -type AccessTuple -out gen_access_tuple.go // AccessList is an EIP-2930 access list. type AccessList []AccessTuple diff --git a/core/types/block.go b/core/types/block.go index f38c55c1ffd7..314990dc99af 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -63,7 +63,8 @@ func (n *BlockNonce) UnmarshalText(input []byte) error { return hexutil.UnmarshalFixedText("BlockNonce", input, n[:]) } -//go:generate gencodec -type Header -field-override headerMarshaling -out gen_header_json.go +//go:generate go run github.com/fjl/gencodec@latest -type Header -field-override headerMarshaling -out gen_header_json.go +//go:generate go run ../../rlp/rlpgen -type Header -out gen_header_rlp.go // Header represents a block header in the Ethereum blockchain. type Header struct { diff --git a/core/types/block_test.go b/core/types/block_test.go index 5cdea3fc06e0..aa1db2f4faad 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -285,7 +285,7 @@ func makeBenchBlock() *Block { func TestRlpDecodeParentHash(t *testing.T) { // A minimum one want := common.HexToHash("0x112233445566778899001122334455667788990011223344556677889900aabb") - if rlpData, err := rlp.EncodeToBytes(Header{ParentHash: want}); err != nil { + if rlpData, err := rlp.EncodeToBytes(&Header{ParentHash: want}); err != nil { t.Fatal(err) } else { if have := HeaderParentHashFromRLP(rlpData); have != want { @@ -299,7 +299,7 @@ func TestRlpDecodeParentHash(t *testing.T) { // | BaseFee | dynamic| *big.Int | 64 bits | mainnetTd := new(big.Int) mainnetTd.SetString("5ad3c2c71bbff854908", 16) - if rlpData, err := rlp.EncodeToBytes(Header{ + if rlpData, err := rlp.EncodeToBytes(&Header{ ParentHash: want, Difficulty: mainnetTd, Number: new(big.Int).SetUint64(math.MaxUint64), @@ -316,7 +316,7 @@ func TestRlpDecodeParentHash(t *testing.T) { { // The rlp-encoding of the heder belowCauses _total_ length of 65540, // which is the first to blow the fast-path. - h := Header{ + h := &Header{ ParentHash: want, Extra: make([]byte, 65041), } diff --git a/core/types/gen_account_rlp.go b/core/types/gen_account_rlp.go new file mode 100644 index 000000000000..5181d884112f --- /dev/null +++ b/core/types/gen_account_rlp.go @@ -0,0 +1,27 @@ +// Code generated by rlpgen. DO NOT EDIT. + +//go:build !norlpgen +// +build !norlpgen + +package types + +import "github.com/ethereum/go-ethereum/rlp" +import "io" + +func (obj *StateAccount) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteUint64(obj.Nonce) + if obj.Balance == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Balance.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Balance) + } + w.WriteBytes(obj.Root[:]) + w.WriteBytes(obj.CodeHash) + w.ListEnd(_tmp0) + return w.Flush() +} diff --git a/core/types/gen_header_rlp.go b/core/types/gen_header_rlp.go new file mode 100644 index 000000000000..e1a687331853 --- /dev/null +++ b/core/types/gen_header_rlp.go @@ -0,0 +1,56 @@ +// Code generated by rlpgen. DO NOT EDIT. + +//go:build !norlpgen +// +build !norlpgen + +package types + +import "github.com/ethereum/go-ethereum/rlp" +import "io" + +func (obj *Header) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteBytes(obj.ParentHash[:]) + w.WriteBytes(obj.UncleHash[:]) + w.WriteBytes(obj.Coinbase[:]) + w.WriteBytes(obj.Root[:]) + w.WriteBytes(obj.TxHash[:]) + w.WriteBytes(obj.ReceiptHash[:]) + w.WriteBytes(obj.Bloom[:]) + if obj.Difficulty == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Difficulty.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Difficulty) + } + if obj.Number == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Number.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Number) + } + w.WriteUint64(obj.GasLimit) + w.WriteUint64(obj.GasUsed) + w.WriteUint64(obj.Time) + w.WriteBytes(obj.Extra) + w.WriteBytes(obj.MixDigest[:]) + w.WriteBytes(obj.Nonce[:]) + _tmp1 := obj.BaseFee != nil + if _tmp1 { + if obj.BaseFee == nil { + w.Write(rlp.EmptyString) + } else { + if obj.BaseFee.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.BaseFee) + } + } + w.ListEnd(_tmp0) + return w.Flush() +} diff --git a/core/types/gen_log_rlp.go b/core/types/gen_log_rlp.go new file mode 100644 index 000000000000..4a6c6b0094f8 --- /dev/null +++ b/core/types/gen_log_rlp.go @@ -0,0 +1,23 @@ +// Code generated by rlpgen. DO NOT EDIT. + +//go:build !norlpgen +// +build !norlpgen + +package types + +import "github.com/ethereum/go-ethereum/rlp" +import "io" + +func (obj *rlpLog) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteBytes(obj.Address[:]) + _tmp1 := w.List() + for _, _tmp2 := range obj.Topics { + w.WriteBytes(_tmp2[:]) + } + w.ListEnd(_tmp1) + w.WriteBytes(obj.Data) + w.ListEnd(_tmp0) + return w.Flush() +} diff --git a/core/types/legacy.go b/core/types/legacy.go new file mode 100644 index 000000000000..9254381b1e11 --- /dev/null +++ b/core/types/legacy.go @@ -0,0 +1,53 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "errors" + + "github.com/ethereum/go-ethereum/rlp" +) + +// IsLegacyStoredReceipts tries to parse the RLP-encoded blob +// first as an array of v3 stored receipt, then v4 stored receipt and +// returns true if successful. +func IsLegacyStoredReceipts(raw []byte) (bool, error) { + var v3 []v3StoredReceiptRLP + if err := rlp.DecodeBytes(raw, &v3); err == nil { + return true, nil + } + var v4 []v4StoredReceiptRLP + if err := rlp.DecodeBytes(raw, &v4); err == nil { + return true, nil + } + var v5 []storedReceiptRLP + // Check to see valid fresh stored receipt + if err := rlp.DecodeBytes(raw, &v5); err == nil { + return false, nil + } + return false, errors.New("value is not a valid receipt encoding") +} + +// ConvertLegacyStoredReceipts takes the RLP encoding of an array of legacy +// stored receipts and returns a fresh RLP-encoded stored receipt. +func ConvertLegacyStoredReceipts(raw []byte) ([]byte, error) { + var receipts []ReceiptForStorage + if err := rlp.DecodeBytes(raw, &receipts); err != nil { + return nil, err + } + return rlp.EncodeToBytes(&receipts) +} diff --git a/core/types/log.go b/core/types/log.go index 88274e39dae0..b27c7ccbd3a9 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -//go:generate gencodec -type Log -field-override logMarshaling -out gen_log_json.go +//go:generate go run github.com/fjl/gencodec@latest -type Log -field-override logMarshaling -out gen_log_json.go // Log represents a contract log event. These events are generated by the LOG opcode and // stored/indexed by the node. @@ -62,15 +62,14 @@ type logMarshaling struct { Index hexutil.Uint } +//go:generate go run ../../rlp/rlpgen -type rlpLog -out gen_log_rlp.go + type rlpLog struct { Address common.Address Topics []common.Hash Data []byte } -// rlpStorageLog is the storage encoding of a log. -type rlpStorageLog rlpLog - // legacyRlpStorageLog is the previous storage encoding of a log including some redundant fields. type legacyRlpStorageLog struct { Address common.Address @@ -85,7 +84,8 @@ type legacyRlpStorageLog struct { // EncodeRLP implements rlp.Encoder. func (l *Log) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}) + rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data} + return rlp.Encode(w, &rl) } // DecodeRLP implements rlp.Decoder. @@ -104,11 +104,8 @@ type LogForStorage Log // EncodeRLP implements rlp.Encoder. func (l *LogForStorage) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, rlpStorageLog{ - Address: l.Address, - Topics: l.Topics, - Data: l.Data, - }) + rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data} + return rlp.Encode(w, &rl) } // DecodeRLP implements rlp.Decoder. @@ -119,7 +116,7 @@ func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error { if err != nil { return err } - var dec rlpStorageLog + var dec rlpLog err = rlp.DecodeBytes(blob, &dec) if err == nil { *l = LogForStorage{ diff --git a/core/types/receipt.go b/core/types/receipt.go index 517e76283946..475fcf4ebe9b 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -31,15 +31,14 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -//go:generate gencodec -type Receipt -field-override receiptMarshaling -out gen_receipt_json.go +//go:generate go run github.com/fjl/gencodec@latest -type Receipt -field-override receiptMarshaling -out gen_receipt_json.go var ( receiptStatusFailedRLP = []byte{} receiptStatusSuccessfulRLP = []byte{0x01} ) -// This error is returned when a typed receipt is decoded, but the string is empty. -var errEmptyTypedReceipt = errors.New("empty typed receipt bytes") +var errShortTypedReceipt = errors.New("typed receipt too short") const ( // ReceiptStatusFailed is the status code of a transaction if execution failed. @@ -186,26 +185,13 @@ func (r *Receipt) DecodeRLP(s *rlp.Stream) error { } r.Type = LegacyTxType return r.setFromRLP(dec) - case kind == rlp.String: + default: // It's an EIP-2718 typed tx receipt. b, err := s.Bytes() if err != nil { return err } - if len(b) == 0 { - return errEmptyTypedReceipt - } - r.Type = b[0] - if r.Type == AccessListTxType || r.Type == DynamicFeeTxType { - var dec receiptRLP - if err := rlp.DecodeBytes(b[1:], &dec); err != nil { - return err - } - return r.setFromRLP(dec) - } - return ErrTxTypeNotSupported - default: - return rlp.ErrExpectedList + return r.decodeTyped(b) } } @@ -228,8 +214,8 @@ func (r *Receipt) UnmarshalBinary(b []byte) error { // decodeTyped decodes a typed receipt from the canonical format. func (r *Receipt) decodeTyped(b []byte) error { - if len(b) == 0 { - return errEmptyTypedReceipt + if len(b) <= 1 { + return errShortTypedReceipt } switch b[0] { case DynamicFeeTxType, AccessListTxType: @@ -291,16 +277,20 @@ type ReceiptForStorage Receipt // EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt // into an RLP stream. -func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error { - enc := &storedReceiptRLP{ - PostStateOrStatus: (*Receipt)(r).statusEncoding(), - CumulativeGasUsed: r.CumulativeGasUsed, - Logs: make([]*LogForStorage, len(r.Logs)), - } - for i, log := range r.Logs { - enc.Logs[i] = (*LogForStorage)(log) +func (r *ReceiptForStorage) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + outerList := w.List() + w.WriteBytes((*Receipt)(r).statusEncoding()) + w.WriteUint64(r.CumulativeGasUsed) + logList := w.List() + for _, log := range r.Logs { + if err := rlp.Encode(w, log); err != nil { + return err + } } - return rlp.Encode(w, enc) + w.ListEnd(logList) + w.ListEnd(outerList) + return w.Flush() } // DecodeRLP implements rlp.Decoder, and loads both consensus and implementation diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 613559a6586c..bba18d2a7bf3 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -86,7 +86,7 @@ func TestDecodeEmptyTypedReceipt(t *testing.T) { input := []byte{0x80} var r Receipt err := rlp.DecodeBytes(input, &r) - if err != errEmptyTypedReceipt { + if err != errShortTypedReceipt { t.Fatal("wrong error:", err) } } diff --git a/core/types/state_account.go b/core/types/state_account.go index 68804bf311f4..3b01be45198f 100644 --- a/core/types/state_account.go +++ b/core/types/state_account.go @@ -22,6 +22,8 @@ import ( "github.com/ethereum/go-ethereum/common" ) +//go:generate go run ../../rlp/rlpgen -type StateAccount -out gen_account_rlp.go + // StateAccount is the Ethereum consensus representation of accounts. // These objects are stored in the main account trie. type StateAccount struct { diff --git a/core/types/transaction.go b/core/types/transaction.go index 515e63971381..d9d610c45f61 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -37,7 +37,7 @@ var ( ErrInvalidTxType = errors.New("transaction type not valid in this context") ErrTxTypeNotSupported = errors.New("transaction type not supported") ErrGasFeeCapTooLow = errors.New("fee cap less than base fee") - errEmptyTypedTx = errors.New("empty typed transaction bytes") + errShortTypedTx = errors.New("typed transaction too short") ) // Transaction types. @@ -138,7 +138,7 @@ func (tx *Transaction) DecodeRLP(s *rlp.Stream) error { tx.setDecoded(&inner, int(rlp.ListSize(size))) } return err - case kind == rlp.String: + default: // It's an EIP-2718 typed TX envelope. var b []byte if b, err = s.Bytes(); err != nil { @@ -149,8 +149,6 @@ func (tx *Transaction) DecodeRLP(s *rlp.Stream) error { tx.setDecoded(inner, len(b)) } return err - default: - return rlp.ErrExpectedList } } @@ -178,8 +176,8 @@ func (tx *Transaction) UnmarshalBinary(b []byte) error { // decodeTyped decodes a typed transaction from the canonical format. func (tx *Transaction) decodeTyped(b []byte) (TxData, error) { - if len(b) == 0 { - return nil, errEmptyTypedTx + if len(b) <= 1 { + return nil, errShortTypedTx } switch b[0] { case AccessListTxType: diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 58c95071b288..a4755675cd25 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -76,7 +76,7 @@ func TestDecodeEmptyTypedTx(t *testing.T) { input := []byte{0x80} var tx Transaction err := rlp.DecodeBytes(input, &tx) - if err != errEmptyTypedTx { + if err != errShortTypedTx { t.Fatal("wrong error:", err) } } diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index 483226eefad8..551e1f5f1188 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -214,7 +214,7 @@ var ( // see gasSStoreEIP2200(...) in core/vm/gas_table.go for more info about how EIP 2200 is specified gasSStoreEIP2929 = makeGasSStoreFunc(params.SstoreClearsScheduleRefundEIP2200) - // gasSStoreEIP2539 implements gas cost for SSTORE according to EPI-2539 + // gasSStoreEIP2539 implements gas cost for SSTORE according to EIP-2539 // Replace `SSTORE_CLEARS_SCHEDULE` with `SSTORE_RESET_GAS + ACCESS_LIST_STORAGE_KEY_COST` (4,800) gasSStoreEIP3529 = makeGasSStoreFunc(params.SstoreClearsScheduleRefundEIP3529) ) diff --git a/crypto/bls12381/bls12_381.go b/crypto/bls12381/bls12_381.go index e204a927d1a9..1c1c97765f47 100644 --- a/crypto/bls12381/bls12_381.go +++ b/crypto/bls12381/bls12_381.go @@ -119,105 +119,105 @@ var g2One = PointG2{ */ var frobeniusCoeffs61 = [6]fe2{ - fe2{ + { fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, }, - fe2{ + { fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, }, - fe2{ + { fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, }, } var frobeniusCoeffs62 = [6]fe2{ - fe2{ + { fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x890dc9e4867545c3, 0x2af322533285a5d5, 0x50880866309b7e2c, 0xa20d1b8c7e881024, 0x14e4f04fe2db9068, 0x14e56d3f1564853a}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0xecfb361b798dba3a, 0xc100ddb891865a2c, 0x0ec08ff1232bda8e, 0xd5c13cc6f1ca4721, 0x47222a47bf7b5c04, 0x0110f184e51c5f59}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, } var frobeniusCoeffs12 = [12]fe2{ - fe2{ + { fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x07089552b319d465, 0xc6695f92b50a8313, 0x97e83cccd117228f, 0xa35baecab2dc29ee, 0x1ce393ea5daace4d, 0x08f2220fb0fb66eb}, fe{0xb2f66aad4ce5d646, 0x5842a06bfc497cec, 0xcf4895d42599d394, 0xc11b9cba40a8e8d0, 0x2e3813cbe5a0de89, 0x110eefda88847faf}, }, - fe2{ + { fe{0xecfb361b798dba3a, 0xc100ddb891865a2c, 0x0ec08ff1232bda8e, 0xd5c13cc6f1ca4721, 0x47222a47bf7b5c04, 0x0110f184e51c5f59}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x3e2f585da55c9ad1, 0x4294213d86c18183, 0x382844c88b623732, 0x92ad2afd19103e18, 0x1d794e4fac7cf0b9, 0x0bd592fc7d825ec8}, fe{0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, }, - fe2{ + { fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x3726c30af242c66c, 0x7c2ac1aad1b6fe70, 0xa04007fbba4b14a2, 0xef517c3266341429, 0x0095ba654ed2226b, 0x02e370eccc86f7dd}, fe{0x82d83cf50dbce43f, 0xa2813e53df9d018f, 0xc6f0caa53c65e181, 0x7525cf528d50fe95, 0x4a85ed50f4798a6b, 0x171da0fd6cf8eebd}, }, - fe2{ + { fe{0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0xb2f66aad4ce5d646, 0x5842a06bfc497cec, 0xcf4895d42599d394, 0xc11b9cba40a8e8d0, 0x2e3813cbe5a0de89, 0x110eefda88847faf}, fe{0x07089552b319d465, 0xc6695f92b50a8313, 0x97e83cccd117228f, 0xa35baecab2dc29ee, 0x1ce393ea5daace4d, 0x08f2220fb0fb66eb}, }, - fe2{ + { fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, fe{0x3e2f585da55c9ad1, 0x4294213d86c18183, 0x382844c88b623732, 0x92ad2afd19103e18, 0x1d794e4fac7cf0b9, 0x0bd592fc7d825ec8}, }, - fe2{ + { fe{0x890dc9e4867545c3, 0x2af322533285a5d5, 0x50880866309b7e2c, 0xa20d1b8c7e881024, 0x14e4f04fe2db9068, 0x14e56d3f1564853a}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, - fe2{ + { fe{0x82d83cf50dbce43f, 0xa2813e53df9d018f, 0xc6f0caa53c65e181, 0x7525cf528d50fe95, 0x4a85ed50f4798a6b, 0x171da0fd6cf8eebd}, fe{0x3726c30af242c66c, 0x7c2ac1aad1b6fe70, 0xa04007fbba4b14a2, 0xef517c3266341429, 0x0095ba654ed2226b, 0x02e370eccc86f7dd}, }, diff --git a/crypto/bls12381/isogeny.go b/crypto/bls12381/isogeny.go index 91e03936d57d..c3cb0a6f7bf0 100644 --- a/crypto/bls12381/isogeny.go +++ b/crypto/bls12381/isogeny.go @@ -77,149 +77,149 @@ func isogenyMapG2(e *fp2, x, y *fe2) { } var isogenyConstansG1 = [4][16]*fe{ - [16]*fe{ - &fe{0x4d18b6f3af00131c, 0x19fa219793fee28c, 0x3f2885f1467f19ae, 0x23dcea34f2ffb304, 0xd15b58d2ffc00054, 0x0913be200a20bef4}, - &fe{0x898985385cdbbd8b, 0x3c79e43cc7d966aa, 0x1597e193f4cd233a, 0x8637ef1e4d6623ad, 0x11b22deed20d827b, 0x07097bc5998784ad}, - &fe{0xa542583a480b664b, 0xfc7169c026e568c6, 0x5ba2ef314ed8b5a6, 0x5b5491c05102f0e7, 0xdf6e99707d2a0079, 0x0784151ed7605524}, - &fe{0x494e212870f72741, 0xab9be52fbda43021, 0x26f5577994e34c3d, 0x049dfee82aefbd60, 0x65dadd7828505289, 0x0e93d431ea011aeb}, - &fe{0x90ee774bd6a74d45, 0x7ada1c8a41bfb185, 0x0f1a8953b325f464, 0x104c24211be4805c, 0x169139d319ea7a8f, 0x09f20ead8e532bf6}, - &fe{0x6ddd93e2f43626b7, 0xa5482c9aa1ccd7bd, 0x143245631883f4bd, 0x2e0a94ccf77ec0db, 0xb0282d480e56489f, 0x18f4bfcbb4368929}, - &fe{0x23c5f0c953402dfd, 0x7a43ff6958ce4fe9, 0x2c390d3d2da5df63, 0xd0df5c98e1f9d70f, 0xffd89869a572b297, 0x1277ffc72f25e8fe}, - &fe{0x79f4f0490f06a8a6, 0x85f894a88030fd81, 0x12da3054b18b6410, 0xe2a57f6505880d65, 0xbba074f260e400f1, 0x08b76279f621d028}, - &fe{0xe67245ba78d5b00b, 0x8456ba9a1f186475, 0x7888bff6e6b33bb4, 0xe21585b9a30f86cb, 0x05a69cdcef55feee, 0x09e699dd9adfa5ac}, - &fe{0x0de5c357bff57107, 0x0a0db4ae6b1a10b2, 0xe256bb67b3b3cd8d, 0x8ad456574e9db24f, 0x0443915f50fd4179, 0x098c4bf7de8b6375}, - &fe{0xe6b0617e7dd929c7, 0xfe6e37d442537375, 0x1dafdeda137a489e, 0xe4efd1ad3f767ceb, 0x4a51d8667f0fe1cf, 0x054fdf4bbf1d821c}, - &fe{0x72db2a50658d767b, 0x8abf91faa257b3d5, 0xe969d6833764ab47, 0x464170142a1009eb, 0xb14f01aadb30be2f, 0x18ae6a856f40715d}, - &fe{0, 0, 0, 0, 0, 0}, - &fe{0, 0, 0, 0, 0, 0}, - &fe{0, 0, 0, 0, 0, 0}, - &fe{0, 0, 0, 0, 0, 0}, + { + {0x4d18b6f3af00131c, 0x19fa219793fee28c, 0x3f2885f1467f19ae, 0x23dcea34f2ffb304, 0xd15b58d2ffc00054, 0x0913be200a20bef4}, + {0x898985385cdbbd8b, 0x3c79e43cc7d966aa, 0x1597e193f4cd233a, 0x8637ef1e4d6623ad, 0x11b22deed20d827b, 0x07097bc5998784ad}, + {0xa542583a480b664b, 0xfc7169c026e568c6, 0x5ba2ef314ed8b5a6, 0x5b5491c05102f0e7, 0xdf6e99707d2a0079, 0x0784151ed7605524}, + {0x494e212870f72741, 0xab9be52fbda43021, 0x26f5577994e34c3d, 0x049dfee82aefbd60, 0x65dadd7828505289, 0x0e93d431ea011aeb}, + {0x90ee774bd6a74d45, 0x7ada1c8a41bfb185, 0x0f1a8953b325f464, 0x104c24211be4805c, 0x169139d319ea7a8f, 0x09f20ead8e532bf6}, + {0x6ddd93e2f43626b7, 0xa5482c9aa1ccd7bd, 0x143245631883f4bd, 0x2e0a94ccf77ec0db, 0xb0282d480e56489f, 0x18f4bfcbb4368929}, + {0x23c5f0c953402dfd, 0x7a43ff6958ce4fe9, 0x2c390d3d2da5df63, 0xd0df5c98e1f9d70f, 0xffd89869a572b297, 0x1277ffc72f25e8fe}, + {0x79f4f0490f06a8a6, 0x85f894a88030fd81, 0x12da3054b18b6410, 0xe2a57f6505880d65, 0xbba074f260e400f1, 0x08b76279f621d028}, + {0xe67245ba78d5b00b, 0x8456ba9a1f186475, 0x7888bff6e6b33bb4, 0xe21585b9a30f86cb, 0x05a69cdcef55feee, 0x09e699dd9adfa5ac}, + {0x0de5c357bff57107, 0x0a0db4ae6b1a10b2, 0xe256bb67b3b3cd8d, 0x8ad456574e9db24f, 0x0443915f50fd4179, 0x098c4bf7de8b6375}, + {0xe6b0617e7dd929c7, 0xfe6e37d442537375, 0x1dafdeda137a489e, 0xe4efd1ad3f767ceb, 0x4a51d8667f0fe1cf, 0x054fdf4bbf1d821c}, + {0x72db2a50658d767b, 0x8abf91faa257b3d5, 0xe969d6833764ab47, 0x464170142a1009eb, 0xb14f01aadb30be2f, 0x18ae6a856f40715d}, + {0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, }, - [16]*fe{ - &fe{0xb962a077fdb0f945, 0xa6a9740fefda13a0, 0xc14d568c3ed6c544, 0xb43fc37b908b133e, 0x9c0b3ac929599016, 0x0165aa6c93ad115f}, - &fe{0x23279a3ba506c1d9, 0x92cfca0a9465176a, 0x3b294ab13755f0ff, 0x116dda1c5070ae93, 0xed4530924cec2045, 0x083383d6ed81f1ce}, - &fe{0x9885c2a6449fecfc, 0x4a2b54ccd37733f0, 0x17da9ffd8738c142, 0xa0fba72732b3fafd, 0xff364f36e54b6812, 0x0f29c13c660523e2}, - &fe{0xe349cc118278f041, 0xd487228f2f3204fb, 0xc9d325849ade5150, 0x43a92bd69c15c2df, 0x1c2c7844bc417be4, 0x12025184f407440c}, - &fe{0x587f65ae6acb057b, 0x1444ef325140201f, 0xfbf995e71270da49, 0xccda066072436a42, 0x7408904f0f186bb2, 0x13b93c63edf6c015}, - &fe{0xfb918622cd141920, 0x4a4c64423ecaddb4, 0x0beb232927f7fb26, 0x30f94df6f83a3dc2, 0xaeedd424d780f388, 0x06cc402dd594bbeb}, - &fe{0xd41f761151b23f8f, 0x32a92465435719b3, 0x64f436e888c62cb9, 0xdf70a9a1f757c6e4, 0x6933a38d5b594c81, 0x0c6f7f7237b46606}, - &fe{0x693c08747876c8f7, 0x22c9850bf9cf80f0, 0x8e9071dab950c124, 0x89bc62d61c7baf23, 0xbc6be2d8dad57c23, 0x17916987aa14a122}, - &fe{0x1be3ff439c1316fd, 0x9965243a7571dfa7, 0xc7f7f62962f5cd81, 0x32c6aa9af394361c, 0xbbc2ee18e1c227f4, 0x0c102cbac531bb34}, - &fe{0x997614c97bacbf07, 0x61f86372b99192c0, 0x5b8c95fc14353fc3, 0xca2b066c2a87492f, 0x16178f5bbf698711, 0x12a6dcd7f0f4e0e8}, - &fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, - &fe{0, 0, 0, 0, 0, 0}, - &fe{0, 0, 0, 0, 0, 0}, - &fe{0, 0, 0, 0, 0, 0}, - &fe{0, 0, 0, 0, 0, 0}, - &fe{0, 0, 0, 0, 0, 0}, + { + {0xb962a077fdb0f945, 0xa6a9740fefda13a0, 0xc14d568c3ed6c544, 0xb43fc37b908b133e, 0x9c0b3ac929599016, 0x0165aa6c93ad115f}, + {0x23279a3ba506c1d9, 0x92cfca0a9465176a, 0x3b294ab13755f0ff, 0x116dda1c5070ae93, 0xed4530924cec2045, 0x083383d6ed81f1ce}, + {0x9885c2a6449fecfc, 0x4a2b54ccd37733f0, 0x17da9ffd8738c142, 0xa0fba72732b3fafd, 0xff364f36e54b6812, 0x0f29c13c660523e2}, + {0xe349cc118278f041, 0xd487228f2f3204fb, 0xc9d325849ade5150, 0x43a92bd69c15c2df, 0x1c2c7844bc417be4, 0x12025184f407440c}, + {0x587f65ae6acb057b, 0x1444ef325140201f, 0xfbf995e71270da49, 0xccda066072436a42, 0x7408904f0f186bb2, 0x13b93c63edf6c015}, + {0xfb918622cd141920, 0x4a4c64423ecaddb4, 0x0beb232927f7fb26, 0x30f94df6f83a3dc2, 0xaeedd424d780f388, 0x06cc402dd594bbeb}, + {0xd41f761151b23f8f, 0x32a92465435719b3, 0x64f436e888c62cb9, 0xdf70a9a1f757c6e4, 0x6933a38d5b594c81, 0x0c6f7f7237b46606}, + {0x693c08747876c8f7, 0x22c9850bf9cf80f0, 0x8e9071dab950c124, 0x89bc62d61c7baf23, 0xbc6be2d8dad57c23, 0x17916987aa14a122}, + {0x1be3ff439c1316fd, 0x9965243a7571dfa7, 0xc7f7f62962f5cd81, 0x32c6aa9af394361c, 0xbbc2ee18e1c227f4, 0x0c102cbac531bb34}, + {0x997614c97bacbf07, 0x61f86372b99192c0, 0x5b8c95fc14353fc3, 0xca2b066c2a87492f, 0x16178f5bbf698711, 0x12a6dcd7f0f4e0e8}, + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + {0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, }, - [16]*fe{ - &fe{0x2b567ff3e2837267, 0x1d4d9e57b958a767, 0xce028fea04bd7373, 0xcc31a30a0b6cd3df, 0x7d7b18a682692693, 0x0d300744d42a0310}, - &fe{0x99c2555fa542493f, 0xfe7f53cc4874f878, 0x5df0608b8f97608a, 0x14e03832052b49c8, 0x706326a6957dd5a4, 0x0a8dadd9c2414555}, - &fe{0x13d942922a5cf63a, 0x357e33e36e261e7d, 0xcf05a27c8456088d, 0x0000bd1de7ba50f0, 0x83d0c7532f8c1fde, 0x13f70bf38bbf2905}, - &fe{0x5c57fd95bfafbdbb, 0x28a359a65e541707, 0x3983ceb4f6360b6d, 0xafe19ff6f97e6d53, 0xb3468f4550192bf7, 0x0bb6cde49d8ba257}, - &fe{0x590b62c7ff8a513f, 0x314b4ce372cacefd, 0x6bef32ce94b8a800, 0x6ddf84a095713d5f, 0x64eace4cb0982191, 0x0386213c651b888d}, - &fe{0xa5310a31111bbcdd, 0xa14ac0f5da148982, 0xf9ad9cc95423d2e9, 0xaa6ec095283ee4a7, 0xcf5b1f022e1c9107, 0x01fddf5aed881793}, - &fe{0x65a572b0d7a7d950, 0xe25c2d8183473a19, 0xc2fcebe7cb877dbd, 0x05b2d36c769a89b0, 0xba12961be86e9efb, 0x07eb1b29c1dfde1f}, - &fe{0x93e09572f7c4cd24, 0x364e929076795091, 0x8569467e68af51b5, 0xa47da89439f5340f, 0xf4fa918082e44d64, 0x0ad52ba3e6695a79}, - &fe{0x911429844e0d5f54, 0xd03f51a3516bb233, 0x3d587e5640536e66, 0xfa86d2a3a9a73482, 0xa90ed5adf1ed5537, 0x149c9c326a5e7393}, - &fe{0x462bbeb03c12921a, 0xdc9af5fa0a274a17, 0x9a558ebde836ebed, 0x649ef8f11a4fae46, 0x8100e1652b3cdc62, 0x1862bd62c291dacb}, - &fe{0x05c9b8ca89f12c26, 0x0194160fa9b9ac4f, 0x6a643d5a6879fa2c, 0x14665bdd8846e19d, 0xbb1d0d53af3ff6bf, 0x12c7e1c3b28962e5}, - &fe{0xb55ebf900b8a3e17, 0xfedc77ec1a9201c4, 0x1f07db10ea1a4df4, 0x0dfbd15dc41a594d, 0x389547f2334a5391, 0x02419f98165871a4}, - &fe{0xb416af000745fc20, 0x8e563e9d1ea6d0f5, 0x7c763e17763a0652, 0x01458ef0159ebbef, 0x8346fe421f96bb13, 0x0d2d7b829ce324d2}, - &fe{0x93096bb538d64615, 0x6f2a2619951d823a, 0x8f66b3ea59514fa4, 0xf563e63704f7092f, 0x724b136c4cf2d9fa, 0x046959cfcfd0bf49}, - &fe{0xea748d4b6e405346, 0x91e9079c2c02d58f, 0x41064965946d9b59, 0xa06731f1d2bbe1ee, 0x07f897e267a33f1b, 0x1017290919210e5f}, - &fe{0x872aa6c17d985097, 0xeecc53161264562a, 0x07afe37afff55002, 0x54759078e5be6838, 0xc4b92d15db8acca8, 0x106d87d1b51d13b9}, + { + {0x2b567ff3e2837267, 0x1d4d9e57b958a767, 0xce028fea04bd7373, 0xcc31a30a0b6cd3df, 0x7d7b18a682692693, 0x0d300744d42a0310}, + {0x99c2555fa542493f, 0xfe7f53cc4874f878, 0x5df0608b8f97608a, 0x14e03832052b49c8, 0x706326a6957dd5a4, 0x0a8dadd9c2414555}, + {0x13d942922a5cf63a, 0x357e33e36e261e7d, 0xcf05a27c8456088d, 0x0000bd1de7ba50f0, 0x83d0c7532f8c1fde, 0x13f70bf38bbf2905}, + {0x5c57fd95bfafbdbb, 0x28a359a65e541707, 0x3983ceb4f6360b6d, 0xafe19ff6f97e6d53, 0xb3468f4550192bf7, 0x0bb6cde49d8ba257}, + {0x590b62c7ff8a513f, 0x314b4ce372cacefd, 0x6bef32ce94b8a800, 0x6ddf84a095713d5f, 0x64eace4cb0982191, 0x0386213c651b888d}, + {0xa5310a31111bbcdd, 0xa14ac0f5da148982, 0xf9ad9cc95423d2e9, 0xaa6ec095283ee4a7, 0xcf5b1f022e1c9107, 0x01fddf5aed881793}, + {0x65a572b0d7a7d950, 0xe25c2d8183473a19, 0xc2fcebe7cb877dbd, 0x05b2d36c769a89b0, 0xba12961be86e9efb, 0x07eb1b29c1dfde1f}, + {0x93e09572f7c4cd24, 0x364e929076795091, 0x8569467e68af51b5, 0xa47da89439f5340f, 0xf4fa918082e44d64, 0x0ad52ba3e6695a79}, + {0x911429844e0d5f54, 0xd03f51a3516bb233, 0x3d587e5640536e66, 0xfa86d2a3a9a73482, 0xa90ed5adf1ed5537, 0x149c9c326a5e7393}, + {0x462bbeb03c12921a, 0xdc9af5fa0a274a17, 0x9a558ebde836ebed, 0x649ef8f11a4fae46, 0x8100e1652b3cdc62, 0x1862bd62c291dacb}, + {0x05c9b8ca89f12c26, 0x0194160fa9b9ac4f, 0x6a643d5a6879fa2c, 0x14665bdd8846e19d, 0xbb1d0d53af3ff6bf, 0x12c7e1c3b28962e5}, + {0xb55ebf900b8a3e17, 0xfedc77ec1a9201c4, 0x1f07db10ea1a4df4, 0x0dfbd15dc41a594d, 0x389547f2334a5391, 0x02419f98165871a4}, + {0xb416af000745fc20, 0x8e563e9d1ea6d0f5, 0x7c763e17763a0652, 0x01458ef0159ebbef, 0x8346fe421f96bb13, 0x0d2d7b829ce324d2}, + {0x93096bb538d64615, 0x6f2a2619951d823a, 0x8f66b3ea59514fa4, 0xf563e63704f7092f, 0x724b136c4cf2d9fa, 0x046959cfcfd0bf49}, + {0xea748d4b6e405346, 0x91e9079c2c02d58f, 0x41064965946d9b59, 0xa06731f1d2bbe1ee, 0x07f897e267a33f1b, 0x1017290919210e5f}, + {0x872aa6c17d985097, 0xeecc53161264562a, 0x07afe37afff55002, 0x54759078e5be6838, 0xc4b92d15db8acca8, 0x106d87d1b51d13b9}, }, - [16]*fe{ - &fe{0xeb6c359d47e52b1c, 0x18ef5f8a10634d60, 0xddfa71a0889d5b7e, 0x723e71dcc5fc1323, 0x52f45700b70d5c69, 0x0a8b981ee47691f1}, - &fe{0x616a3c4f5535b9fb, 0x6f5f037395dbd911, 0xf25f4cc5e35c65da, 0x3e50dffea3c62658, 0x6a33dca523560776, 0x0fadeff77b6bfe3e}, - &fe{0x2be9b66df470059c, 0x24a2c159a3d36742, 0x115dbe7ad10c2a37, 0xb6634a652ee5884d, 0x04fe8bb2b8d81af4, 0x01c2a7a256fe9c41}, - &fe{0xf27bf8ef3b75a386, 0x898b367476c9073f, 0x24482e6b8c2f4e5f, 0xc8e0bbd6fe110806, 0x59b0c17f7631448a, 0x11037cd58b3dbfbd}, - &fe{0x31c7912ea267eec6, 0x1dbf6f1c5fcdb700, 0xd30d4fe3ba86fdb1, 0x3cae528fbee9a2a4, 0xb1cce69b6aa9ad9a, 0x044393bb632d94fb}, - &fe{0xc66ef6efeeb5c7e8, 0x9824c289dd72bb55, 0x71b1a4d2f119981d, 0x104fc1aafb0919cc, 0x0e49df01d942a628, 0x096c3a09773272d4}, - &fe{0x9abc11eb5fadeff4, 0x32dca50a885728f0, 0xfb1fa3721569734c, 0xc4b76271ea6506b3, 0xd466a75599ce728e, 0x0c81d4645f4cb6ed}, - &fe{0x4199f10e5b8be45b, 0xda64e495b1e87930, 0xcb353efe9b33e4ff, 0x9e9efb24aa6424c6, 0xf08d33680a237465, 0x0d3378023e4c7406}, - &fe{0x7eb4ae92ec74d3a5, 0xc341b4aa9fac3497, 0x5be603899e907687, 0x03bfd9cca75cbdeb, 0x564c2935a96bfa93, 0x0ef3c33371e2fdb5}, - &fe{0x7ee91fd449f6ac2e, 0xe5d5bd5cb9357a30, 0x773a8ca5196b1380, 0xd0fda172174ed023, 0x6cb95e0fa776aead, 0x0d22d5a40cec7cff}, - &fe{0xf727e09285fd8519, 0xdc9d55a83017897b, 0x7549d8bd057894ae, 0x178419613d90d8f8, 0xfce95ebdeb5b490a, 0x0467ffaef23fc49e}, - &fe{0xc1769e6a7c385f1b, 0x79bc930deac01c03, 0x5461c75a23ede3b5, 0x6e20829e5c230c45, 0x828e0f1e772a53cd, 0x116aefa749127bff}, - &fe{0x101c10bf2744c10a, 0xbbf18d053a6a3154, 0xa0ecf39ef026f602, 0xfc009d4996dc5153, 0xb9000209d5bd08d3, 0x189e5fe4470cd73c}, - &fe{0x7ebd546ca1575ed2, 0xe47d5a981d081b55, 0x57b2b625b6d4ca21, 0xb0a1ba04228520cc, 0x98738983c2107ff3, 0x13dddbc4799d81d6}, - &fe{0x09319f2e39834935, 0x039e952cbdb05c21, 0x55ba77a9a2f76493, 0xfd04e3dfc6086467, 0xfb95832e7d78742e, 0x0ef9c24eccaf5e0e}, - &fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + { + {0xeb6c359d47e52b1c, 0x18ef5f8a10634d60, 0xddfa71a0889d5b7e, 0x723e71dcc5fc1323, 0x52f45700b70d5c69, 0x0a8b981ee47691f1}, + {0x616a3c4f5535b9fb, 0x6f5f037395dbd911, 0xf25f4cc5e35c65da, 0x3e50dffea3c62658, 0x6a33dca523560776, 0x0fadeff77b6bfe3e}, + {0x2be9b66df470059c, 0x24a2c159a3d36742, 0x115dbe7ad10c2a37, 0xb6634a652ee5884d, 0x04fe8bb2b8d81af4, 0x01c2a7a256fe9c41}, + {0xf27bf8ef3b75a386, 0x898b367476c9073f, 0x24482e6b8c2f4e5f, 0xc8e0bbd6fe110806, 0x59b0c17f7631448a, 0x11037cd58b3dbfbd}, + {0x31c7912ea267eec6, 0x1dbf6f1c5fcdb700, 0xd30d4fe3ba86fdb1, 0x3cae528fbee9a2a4, 0xb1cce69b6aa9ad9a, 0x044393bb632d94fb}, + {0xc66ef6efeeb5c7e8, 0x9824c289dd72bb55, 0x71b1a4d2f119981d, 0x104fc1aafb0919cc, 0x0e49df01d942a628, 0x096c3a09773272d4}, + {0x9abc11eb5fadeff4, 0x32dca50a885728f0, 0xfb1fa3721569734c, 0xc4b76271ea6506b3, 0xd466a75599ce728e, 0x0c81d4645f4cb6ed}, + {0x4199f10e5b8be45b, 0xda64e495b1e87930, 0xcb353efe9b33e4ff, 0x9e9efb24aa6424c6, 0xf08d33680a237465, 0x0d3378023e4c7406}, + {0x7eb4ae92ec74d3a5, 0xc341b4aa9fac3497, 0x5be603899e907687, 0x03bfd9cca75cbdeb, 0x564c2935a96bfa93, 0x0ef3c33371e2fdb5}, + {0x7ee91fd449f6ac2e, 0xe5d5bd5cb9357a30, 0x773a8ca5196b1380, 0xd0fda172174ed023, 0x6cb95e0fa776aead, 0x0d22d5a40cec7cff}, + {0xf727e09285fd8519, 0xdc9d55a83017897b, 0x7549d8bd057894ae, 0x178419613d90d8f8, 0xfce95ebdeb5b490a, 0x0467ffaef23fc49e}, + {0xc1769e6a7c385f1b, 0x79bc930deac01c03, 0x5461c75a23ede3b5, 0x6e20829e5c230c45, 0x828e0f1e772a53cd, 0x116aefa749127bff}, + {0x101c10bf2744c10a, 0xbbf18d053a6a3154, 0xa0ecf39ef026f602, 0xfc009d4996dc5153, 0xb9000209d5bd08d3, 0x189e5fe4470cd73c}, + {0x7ebd546ca1575ed2, 0xe47d5a981d081b55, 0x57b2b625b6d4ca21, 0xb0a1ba04228520cc, 0x98738983c2107ff3, 0x13dddbc4799d81d6}, + {0x09319f2e39834935, 0x039e952cbdb05c21, 0x55ba77a9a2f76493, 0xfd04e3dfc6086467, 0xfb95832e7d78742e, 0x0ef9c24eccaf5e0e}, + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, }, } var isogenyConstantsG2 = [4][4]*fe2{ - [4]*fe2{ - &fe2{ + { + { fe{0x47f671c71ce05e62, 0x06dd57071206393e, 0x7c80cd2af3fd71a2, 0x048103ea9e6cd062, 0xc54516acc8d037f6, 0x13808f550920ea41}, fe{0x47f671c71ce05e62, 0x06dd57071206393e, 0x7c80cd2af3fd71a2, 0x048103ea9e6cd062, 0xc54516acc8d037f6, 0x13808f550920ea41}, }, - &fe2{ + { fe{0, 0, 0, 0, 0, 0}, fe{0x5fe55555554c71d0, 0x873fffdd236aaaa3, 0x6a6b4619b26ef918, 0x21c2888408874945, 0x2836cda7028cabc5, 0x0ac73310a7fd5abd}, }, - &fe2{ + { fe{0x0a0c5555555971c3, 0xdb0c00101f9eaaae, 0xb1fb2f941d797997, 0xd3960742ef416e1c, 0xb70040e2c20556f4, 0x149d7861e581393b}, fe{0xaff2aaaaaaa638e8, 0x439fffee91b55551, 0xb535a30cd9377c8c, 0x90e144420443a4a2, 0x941b66d3814655e2, 0x0563998853fead5e}, }, - &fe2{ + { fe{0x40aac71c71c725ed, 0x190955557a84e38e, 0xd817050a8f41abc3, 0xd86485d4c87f6fb1, 0x696eb479f885d059, 0x198e1a74328002d2}, fe{0, 0, 0, 0, 0, 0}, }, }, - [4]*fe2{ - &fe2{ + { + { fe{0, 0, 0, 0, 0, 0}, fe{0x1f3affffff13ab97, 0xf25bfc611da3ff3e, 0xca3757cb3819b208, 0x3e6427366f8cec18, 0x03977bc86095b089, 0x04f69db13f39a952}, }, - &fe2{ + { fe{0x447600000027552e, 0xdcb8009a43480020, 0x6f7ee9ce4a6e8b59, 0xb10330b7c0a95bc6, 0x6140b1fcfb1e54b7, 0x0381be097f0bb4e1}, fe{0x7588ffffffd8557d, 0x41f3ff646e0bffdf, 0xf7b1e8d2ac426aca, 0xb3741acd32dbb6f8, 0xe9daf5b9482d581f, 0x167f53e0ba7431b8}, }, - &fe2{ + { fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, fe{0, 0, 0, 0, 0, 0}, }, - &fe2{ + { fe{0, 0, 0, 0, 0, 0}, fe{0, 0, 0, 0, 0, 0}, }, }, - [4]*fe2{ - &fe2{ + { + { fe{0x96d8f684bdfc77be, 0xb530e4f43b66d0e2, 0x184a88ff379652fd, 0x57cb23ecfae804e1, 0x0fd2e39eada3eba9, 0x08c8055e31c5d5c3}, fe{0x96d8f684bdfc77be, 0xb530e4f43b66d0e2, 0x184a88ff379652fd, 0x57cb23ecfae804e1, 0x0fd2e39eada3eba9, 0x08c8055e31c5d5c3}, }, - &fe2{ + { fe{0, 0, 0, 0, 0, 0}, fe{0xbf0a71c71c91b406, 0x4d6d55d28b7638fd, 0x9d82f98e5f205aee, 0xa27aa27b1d1a18d5, 0x02c3b2b2d2938e86, 0x0c7d13420b09807f}, }, - &fe2{ + { fe{0xd7f9555555531c74, 0x21cffff748daaaa8, 0x5a9ad1866c9bbe46, 0x4870a2210221d251, 0x4a0db369c0a32af1, 0x02b1ccc429ff56af}, fe{0xe205aaaaaaac8e37, 0xfcdc000768795556, 0x0c96011a8a1537dd, 0x1c06a963f163406e, 0x010df44c82a881e6, 0x174f45260f808feb}, }, - &fe2{ + { fe{0xa470bda12f67f35c, 0xc0fe38e23327b425, 0xc9d3d0f2c6f0678d, 0x1c55c9935b5a982e, 0x27f6c0e2f0746764, 0x117c5e6e28aa9054}, fe{0, 0, 0, 0, 0, 0}, }, }, - [4]*fe2{ - &fe2{ + { + { fe{0x0162fffffa765adf, 0x8f7bea480083fb75, 0x561b3c2259e93611, 0x11e19fc1a9c875d5, 0xca713efc00367660, 0x03c6a03d41da1151}, fe{0x0162fffffa765adf, 0x8f7bea480083fb75, 0x561b3c2259e93611, 0x11e19fc1a9c875d5, 0xca713efc00367660, 0x03c6a03d41da1151}, }, - &fe2{ + { fe{0, 0, 0, 0, 0, 0}, fe{0x5db0fffffd3b02c5, 0xd713f52358ebfdba, 0x5ea60761a84d161a, 0xbb2c75a34ea6c44a, 0x0ac6735921c1119b, 0x0ee3d913bdacfbf6}, }, - &fe2{ + { fe{0x66b10000003affc5, 0xcb1400e764ec0030, 0xa73e5eb56fa5d106, 0x8984c913a0fe09a9, 0x11e10afb78ad7f13, 0x05429d0e3e918f52}, fe{0x534dffffffc4aae6, 0x5397ff174c67ffcf, 0xbff273eb870b251d, 0xdaf2827152870915, 0x393a9cbaca9e2dc3, 0x14be74dbfaee5748}, }, - &fe2{ + { fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, fe{0, 0, 0, 0, 0, 0}, }, diff --git a/crypto/bls12381/swu.go b/crypto/bls12381/swu.go index 40d8c9154db5..e78753b2403a 100644 --- a/crypto/bls12381/swu.go +++ b/crypto/bls12381/swu.go @@ -17,7 +17,7 @@ package bls12381 // swuMapG1 is implementation of Simplified Shallue-van de Woestijne-Ulas Method -// follows the implmentation at draft-irtf-cfrg-hash-to-curve-06. +// follows the implementation at draft-irtf-cfrg-hash-to-curve-06. func swuMapG1(u *fe) (*fe, *fe) { var params = swuParamsForG1 var tv [4]*fe diff --git a/crypto/bn256/cloudflare/gfp_amd64.s b/crypto/bn256/cloudflare/gfp_amd64.s index bdb4ffb78707..64c97eaed951 100644 --- a/crypto/bn256/cloudflare/gfp_amd64.s +++ b/crypto/bn256/cloudflare/gfp_amd64.s @@ -49,7 +49,7 @@ TEXT ·gfpNeg(SB),0,$0-16 SBBQ 24(DI), R11 MOVQ $0, AX - gfpCarry(R8,R9,R10,R11,AX, R12,R13,R14,R15,BX) + gfpCarry(R8,R9,R10,R11,AX, R12,R13,R14,CX,BX) MOVQ c+0(FP), DI storeBlock(R8,R9,R10,R11, 0(DI)) @@ -68,7 +68,7 @@ TEXT ·gfpAdd(SB),0,$0-24 ADCQ 24(SI), R11 ADCQ $0, R12 - gfpCarry(R8,R9,R10,R11,R12, R13,R14,R15,AX,BX) + gfpCarry(R8,R9,R10,R11,R12, R13,R14,CX,AX,BX) MOVQ c+0(FP), DI storeBlock(R8,R9,R10,R11, 0(DI)) @@ -83,7 +83,7 @@ TEXT ·gfpSub(SB),0,$0-24 MOVQ ·p2+0(SB), R12 MOVQ ·p2+8(SB), R13 MOVQ ·p2+16(SB), R14 - MOVQ ·p2+24(SB), R15 + MOVQ ·p2+24(SB), CX MOVQ $0, AX SUBQ 0(SI), R8 @@ -94,12 +94,12 @@ TEXT ·gfpSub(SB),0,$0-24 CMOVQCC AX, R12 CMOVQCC AX, R13 CMOVQCC AX, R14 - CMOVQCC AX, R15 + CMOVQCC AX, CX ADDQ R12, R8 ADCQ R13, R9 ADCQ R14, R10 - ADCQ R15, R11 + ADCQ CX, R11 MOVQ c+0(FP), DI storeBlock(R8,R9,R10,R11, 0(DI)) @@ -115,7 +115,7 @@ TEXT ·gfpMul(SB),0,$160-24 mulBMI2(0(DI),8(DI),16(DI),24(DI), 0(SI)) storeBlock( R8, R9,R10,R11, 0(SP)) - storeBlock(R12,R13,R14,R15, 32(SP)) + storeBlock(R12,R13,R14,CX, 32(SP)) gfpReduceBMI2() JMP end @@ -125,5 +125,5 @@ nobmi2Mul: end: MOVQ c+0(FP), DI - storeBlock(R12,R13,R14,R15, 0(DI)) + storeBlock(R12,R13,R14,CX, 0(DI)) RET diff --git a/crypto/bn256/cloudflare/mul_amd64.h b/crypto/bn256/cloudflare/mul_amd64.h index bab5da8313b6..9d8e4b37dbe7 100644 --- a/crypto/bn256/cloudflare/mul_amd64.h +++ b/crypto/bn256/cloudflare/mul_amd64.h @@ -165,7 +165,7 @@ \ \ // Add the 512-bit intermediate to m*N loadBlock(96+stack, R8,R9,R10,R11) \ - loadBlock(128+stack, R12,R13,R14,R15) \ + loadBlock(128+stack, R12,R13,R14,CX) \ \ MOVQ $0, AX \ ADDQ 0+stack, R8 \ @@ -175,7 +175,7 @@ ADCQ 32+stack, R12 \ ADCQ 40+stack, R13 \ ADCQ 48+stack, R14 \ - ADCQ 56+stack, R15 \ + ADCQ 56+stack, CX \ ADCQ $0, AX \ \ - gfpCarry(R12,R13,R14,R15,AX, R8,R9,R10,R11,BX) + gfpCarry(R12,R13,R14,CX,AX, R8,R9,R10,R11,BX) diff --git a/crypto/bn256/cloudflare/mul_bmi2_amd64.h b/crypto/bn256/cloudflare/mul_bmi2_amd64.h index 71ad0499afd5..403566c6fad8 100644 --- a/crypto/bn256/cloudflare/mul_bmi2_amd64.h +++ b/crypto/bn256/cloudflare/mul_bmi2_amd64.h @@ -29,7 +29,7 @@ ADCQ $0, R14 \ \ MOVQ a2, DX \ - MOVQ $0, R15 \ + MOVQ $0, CX \ MULXQ 0+rb, AX, BX \ ADDQ AX, R10 \ ADCQ BX, R11 \ @@ -43,7 +43,7 @@ MULXQ 24+rb, AX, BX \ ADCQ AX, R13 \ ADCQ BX, R14 \ - ADCQ $0, R15 \ + ADCQ $0, CX \ \ MOVQ a3, DX \ MULXQ 0+rb, AX, BX \ @@ -52,13 +52,13 @@ MULXQ 16+rb, AX, BX \ ADCQ AX, R13 \ ADCQ BX, R14 \ - ADCQ $0, R15 \ + ADCQ $0, CX \ MULXQ 8+rb, AX, BX \ ADDQ AX, R12 \ ADCQ BX, R13 \ MULXQ 24+rb, AX, BX \ ADCQ AX, R14 \ - ADCQ BX, R15 + ADCQ BX, CX #define gfpReduceBMI2() \ \ // m = (T * N') mod R, store m in R8:R9:R10:R11 @@ -106,7 +106,7 @@ ADCQ 32(SP), R12 \ ADCQ 40(SP), R13 \ ADCQ 48(SP), R14 \ - ADCQ 56(SP), R15 \ + ADCQ 56(SP), CX \ ADCQ $0, AX \ \ - gfpCarry(R12,R13,R14,R15,AX, R8,R9,R10,R11,BX) + gfpCarry(R12,R13,R14,CX,AX, R8,R9,R10,R11,BX) diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index fd1e66c7e6fa..3e48e51e84eb 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -24,37 +24,48 @@ import ( "crypto/elliptic" "errors" "fmt" - "math/big" - "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/btcec/v2" + btc_ecdsa "github.com/btcsuite/btcd/btcec/v2/ecdsa" ) // Ecrecover returns the uncompressed public key that created the given signature. func Ecrecover(hash, sig []byte) ([]byte, error) { - pub, err := SigToPub(hash, sig) + pub, err := sigToPub(hash, sig) if err != nil { return nil, err } - bytes := (*btcec.PublicKey)(pub).SerializeUncompressed() + bytes := pub.SerializeUncompressed() return bytes, err } -// SigToPub returns the public key that created the given signature. -func SigToPub(hash, sig []byte) (*ecdsa.PublicKey, error) { +func sigToPub(hash, sig []byte) (*btcec.PublicKey, error) { + if len(sig) != SignatureLength { + return nil, errors.New("invalid signature") + } // Convert to btcec input format with 'recovery id' v at the beginning. btcsig := make([]byte, SignatureLength) - btcsig[0] = sig[64] + 27 + btcsig[0] = sig[RecoveryIDOffset] + 27 copy(btcsig[1:], sig) - pub, _, err := btcec.RecoverCompact(btcec.S256(), btcsig, hash) - return (*ecdsa.PublicKey)(pub), err + pub, _, err := btc_ecdsa.RecoverCompact(btcsig, hash) + return pub, err +} + +// SigToPub returns the public key that created the given signature. +func SigToPub(hash, sig []byte) (*ecdsa.PublicKey, error) { + pub, err := sigToPub(hash, sig) + if err != nil { + return nil, err + } + return pub.ToECDSA(), nil } // Sign calculates an ECDSA signature. // // This function is susceptible to chosen plaintext attacks that can leak // information about the private key that is used for signing. Callers must -// be aware that the given hash cannot be chosen by an adversery. Common +// be aware that the given hash cannot be chosen by an adversary. Common // solution is to hash any input before calculating the signature. // // The produced signature is in the [R || S || V] format where V is 0 or 1. @@ -65,14 +76,20 @@ func Sign(hash []byte, prv *ecdsa.PrivateKey) ([]byte, error) { if prv.Curve != btcec.S256() { return nil, fmt.Errorf("private key curve is not secp256k1") } - sig, err := btcec.SignCompact(btcec.S256(), (*btcec.PrivateKey)(prv), hash, false) + // ecdsa.PrivateKey -> btcec.PrivateKey + var priv btcec.PrivateKey + if overflow := priv.Key.SetByteSlice(prv.D.Bytes()); overflow || priv.Key.IsZero() { + return nil, fmt.Errorf("invalid private key") + } + defer priv.Zero() + sig, err := btc_ecdsa.SignCompact(&priv, hash, false) // ref uncompressed pubkey if err != nil { return nil, err } // Convert to Ethereum signature format with 'recovery id' v at the end. v := sig[0] - 27 copy(sig, sig[1:]) - sig[64] = v + sig[RecoveryIDOffset] = v return sig, nil } @@ -83,13 +100,20 @@ func VerifySignature(pubkey, hash, signature []byte) bool { if len(signature) != 64 { return false } - sig := &btcec.Signature{R: new(big.Int).SetBytes(signature[:32]), S: new(big.Int).SetBytes(signature[32:])} - key, err := btcec.ParsePubKey(pubkey, btcec.S256()) + var r, s btcec.ModNScalar + if r.SetByteSlice(signature[:32]) { + return false // overflow + } + if s.SetByteSlice(signature[32:]) { + return false + } + sig := btc_ecdsa.NewSignature(&r, &s) + key, err := btcec.ParsePubKey(pubkey) if err != nil { return false } // Reject malleable signatures. libsecp256k1 does this check but btcec doesn't. - if sig.S.Cmp(secp256k1halfN) > 0 { + if s.IsOverHalfOrder() { return false } return sig.Verify(hash, key) @@ -100,16 +124,26 @@ func DecompressPubkey(pubkey []byte) (*ecdsa.PublicKey, error) { if len(pubkey) != 33 { return nil, errors.New("invalid compressed public key length") } - key, err := btcec.ParsePubKey(pubkey, btcec.S256()) + key, err := btcec.ParsePubKey(pubkey) if err != nil { return nil, err } return key.ToECDSA(), nil } -// CompressPubkey encodes a public key to the 33-byte compressed format. +// CompressPubkey encodes a public key to the 33-byte compressed format. The +// provided PublicKey must be valid. Namely, the coordinates must not be larger +// than 32 bytes each, they must be less than the field prime, and it must be a +// point on the secp256k1 curve. This is the case for a PublicKey constructed by +// elliptic.Unmarshal (see UnmarshalPubkey), or by ToECDSA and ecdsa.GenerateKey +// when constructing a PrivateKey. func CompressPubkey(pubkey *ecdsa.PublicKey) []byte { - return (*btcec.PublicKey)(pubkey).SerializeCompressed() + // NOTE: the coordinates may be validated with + // btcec.ParsePubKey(FromECDSAPub(pubkey)) + var x, y btcec.FieldVal + x.SetByteSlice(pubkey.X.Bytes()) + y.SetByteSlice(pubkey.Y.Bytes()) + return btcec.NewPublicKey(&x, &y).SerializeCompressed() } // S256 returns an instance of the secp256k1 curve. diff --git a/docker-compose.yml b/docker-compose.yml index 32dc46147281..b464df9676ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,9 @@ -version: '3.2' +version: "3.2" services: ipld-eth-db: restart: always - image: vulcanize/ipld-eth-db:v0.3.1 + image: vulcanize/ipld-eth-db:v3.2.0 environment: POSTGRES_USER: "vdbm" POSTGRES_DB: "vulcanize_testing_v3" diff --git a/docs/postmortems/2021-08-22-split-postmortem.md b/docs/postmortems/2021-08-22-split-postmortem.md index 429f22d70afb..2004f0f2870d 100644 --- a/docs/postmortems/2021-08-22-split-postmortem.md +++ b/docs/postmortems/2021-08-22-split-postmortem.md @@ -5,7 +5,7 @@ This is a post-mortem concerning the minority split that occurred on Ethereum ma ## Timeline -- 2021-08-17: Guido Vranken submitted bounty report. Investigation started, root cause identified, patch variations discussed. +- 2021-08-17: Guido Vranken submitted a bounty report. Investigation started, root cause identified, patch variations discussed. - 2021-08-18: Made public announcement over twitter about upcoming security release upcoming Tuesday. Downstream projects were also notified about the upcoming patch-release. - 2021-08-24: Released [v1.10.8](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8) containing the fix on Tuesday morning (CET). Erigon released [v2021.08.04](https://github.com/ledgerwatch/erigon/releases/tag/v2021.08.04). - 2021-08-27: At 12:50:07 UTC, issue exploited. Analysis started roughly 30m later, @@ -51,7 +51,7 @@ A memory-corruption bug within the EVM can cause a consensus error, where vulner #### Handling -On the evening of 17th, we discussed options how to handle it. We made a state test to reproduce the issue, and verified that neither `openethereum`, `nethermind` nor `besu` were affected by the same vulnerability, and started a full-sync with a patched version of `geth`. +On the evening of 17th, we discussed options on how to handle it. We made a state test to reproduce the issue, and verified that neither `openethereum`, `nethermind` nor `besu` were affected by the same vulnerability, and started a full-sync with a patched version of `geth`. It was decided that in this specific instance, it would be possible to make a public announcement and a patch release: diff --git a/eth/backend.go b/eth/backend.go index 53274fa3cee3..d60e55dc9a95 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -221,16 +221,16 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { checkpoint = params.TrustedCheckpoints[genesisHash] } if eth.handler, err = newHandler(&handlerConfig{ - Database: chainDb, - Chain: eth.blockchain, - TxPool: eth.txPool, - Merger: merger, - Network: config.NetworkId, - Sync: config.SyncMode, - BloomCache: uint64(cacheLimit), - EventMux: eth.eventMux, - Checkpoint: checkpoint, - Whitelist: config.Whitelist, + Database: chainDb, + Chain: eth.blockchain, + TxPool: eth.txPool, + Merger: merger, + Network: config.NetworkId, + Sync: config.SyncMode, + BloomCache: uint64(cacheLimit), + EventMux: eth.eventMux, + Checkpoint: checkpoint, + PeerRequiredBlocks: config.PeerRequiredBlocks, }); err != nil { return nil, err } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index a8b20d75896a..45f233df6dfa 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -20,10 +20,14 @@ package catalyst import ( "crypto/sha256" "encoding/binary" + "errors" "fmt" + "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/beacon" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/log" @@ -36,18 +40,27 @@ func Register(stack *node.Node, backend *eth.Ethereum) error { log.Warn("Catalyst mode enabled", "protocol", "eth") stack.RegisterAPIs([]rpc.API{ { - Namespace: "engine", - Version: "1.0", - Service: NewConsensusAPI(backend), - Public: true, + Namespace: "engine", + Version: "1.0", + Service: NewConsensusAPI(backend), + Public: true, + Authenticated: true, + }, + { + Namespace: "engine", + Version: "1.0", + Service: NewConsensusAPI(backend), + Public: true, + Authenticated: false, }, }) return nil } type ConsensusAPI struct { - eth *eth.Ethereum - preparedBlocks *payloadQueue // preparedBlocks caches payloads (*ExecutableDataV1) by payload ID (PayloadID) + eth *eth.Ethereum + remoteBlocks *headerQueue // Cache of remote payloads received + localBlocks *payloadQueue // Cache of local payloads generated } // NewConsensusAPI creates a new consensus api for the given backend. @@ -57,8 +70,9 @@ func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { panic("Catalyst started without valid total difficulty") } return &ConsensusAPI{ - eth: eth, - preparedBlocks: newPayloadQueue(), + eth: eth, + remoteBlocks: newHeaderQueue(), + localBlocks: newPayloadQueue(), } } @@ -72,85 +86,237 @@ func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { // We try to set our blockchain to the headBlock // If there are payloadAttributes: // we try to assemble a block with the payloadAttributes and return its payloadID -func (api *ConsensusAPI) ForkchoiceUpdatedV1(heads beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) { - log.Trace("Engine API request received", "method", "ForkChoiceUpdated", "head", heads.HeadBlockHash, "finalized", heads.FinalizedBlockHash, "safe", heads.SafeBlockHash) - if heads.HeadBlockHash == (common.Hash{}) { - return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: nil}, nil - } - if err := api.checkTerminalTotalDifficulty(heads.HeadBlockHash); err != nil { - if block := api.eth.BlockChain().GetBlockByHash(heads.HeadBlockHash); block == nil { - // TODO (MariusVanDerWijden) trigger sync - return beacon.SYNCING, nil +func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) { + log.Trace("Engine API request received", "method", "ForkchoiceUpdated", "head", update.HeadBlockHash, "finalized", update.FinalizedBlockHash, "safe", update.SafeBlockHash) + if update.HeadBlockHash == (common.Hash{}) { + log.Warn("Forkchoice requested update to zero hash") + return beacon.STATUS_INVALID, nil // TODO(karalabe): Why does someone send us this? + } + // Check whether we have the block yet in our database or not. If not, we'll + // need to either trigger a sync, or to reject this forkchoice update for a + // reason. + block := api.eth.BlockChain().GetBlockByHash(update.HeadBlockHash) + if block == nil { + // If the head hash is unknown (was not given to us in a newPayload request), + // we cannot resolve the header, so not much to do. This could be extended in + // the future to resolve from the `eth` network, but it's an unexpected case + // that should be fixed, not papered over. + header := api.remoteBlocks.get(update.HeadBlockHash) + if header == nil { + log.Warn("Forkchoice requested unknown head", "hash", update.HeadBlockHash) + return beacon.STATUS_SYNCING, nil + } + // Header advertised via a past newPayload request. Start syncing to it. + // Before we do however, make sure any legacy sync in switched off so we + // don't accidentally have 2 cycles running. + if merger := api.eth.Merger(); !merger.TDDReached() { + merger.ReachTTD() + api.eth.Downloader().Cancel() } - return beacon.INVALID, err + log.Info("Forkchoice requested sync to new head", "number", header.Number, "hash", header.Hash()) + if err := api.eth.Downloader().BeaconSync(api.eth.SyncMode(), header); err != nil { + return beacon.STATUS_SYNCING, err + } + return beacon.STATUS_SYNCING, nil } - // If the finalized block is set, check if it is in our blockchain - if heads.FinalizedBlockHash != (common.Hash{}) { - if block := api.eth.BlockChain().GetBlockByHash(heads.FinalizedBlockHash); block == nil { - // TODO (MariusVanDerWijden) trigger sync - return beacon.SYNCING, nil + // Block is known locally, just sanity check that the beacon client does not + // attempt to push us back to before the merge. + if block.Difficulty().BitLen() > 0 || block.NumberU64() == 0 { + var ( + td = api.eth.BlockChain().GetTd(update.HeadBlockHash, block.NumberU64()) + ptd = api.eth.BlockChain().GetTd(block.ParentHash(), block.NumberU64()-1) + ttd = api.eth.BlockChain().Config().TerminalTotalDifficulty + ) + if td == nil || (block.NumberU64() > 0 && ptd == nil) { + log.Error("TDs unavailable for TTD check", "number", block.NumberU64(), "hash", update.HeadBlockHash, "td", td, "parent", block.ParentHash(), "ptd", ptd) + return beacon.STATUS_INVALID, errors.New("TDs unavailable for TDD check") + } + if td.Cmp(ttd) < 0 || (block.NumberU64() > 0 && ptd.Cmp(ttd) > 0) { + log.Error("Refusing beacon update to pre-merge", "number", block.NumberU64(), "hash", update.HeadBlockHash, "diff", block.Difficulty(), "age", common.PrettyAge(time.Unix(int64(block.Time()), 0))) + return beacon.ForkChoiceResponse{PayloadStatus: beacon.PayloadStatusV1{Status: beacon.INVALIDTERMINALBLOCK}, PayloadID: nil}, nil + } + } + + if rawdb.ReadCanonicalHash(api.eth.ChainDb(), block.NumberU64()) != update.HeadBlockHash { + // Block is not canonical, set head. + if err := api.eth.BlockChain().SetChainHead(block); err != nil { + return beacon.STATUS_INVALID, err + } + } else { + // If the head block is already in our canonical chain, the beacon client is + // probably resyncing. Ignore the update. + log.Info("Ignoring beacon update to old head", "number", block.NumberU64(), "hash", update.HeadBlockHash, "age", common.PrettyAge(time.Unix(int64(block.Time()), 0)), "have", api.eth.BlockChain().CurrentBlock().NumberU64()) + } + api.eth.SetSynced() + + // If the beacon client also advertised a finalized block, mark the local + // chain final and completely in PoS mode. + if update.FinalizedBlockHash != (common.Hash{}) { + if merger := api.eth.Merger(); !merger.PoSFinalized() { + merger.FinalizePoS() + } + // TODO (MariusVanDerWijden): If the finalized block is not in our canonical tree, somethings wrong + finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash) + if finalBlock == nil { + log.Warn("Final block not available in database", "hash", update.FinalizedBlockHash) + return beacon.STATUS_INVALID, errors.New("final block not available") + } else if rawdb.ReadCanonicalHash(api.eth.ChainDb(), finalBlock.NumberU64()) != update.FinalizedBlockHash { + log.Warn("Final block not in canonical chain", "number", block.NumberU64(), "hash", update.HeadBlockHash) + return beacon.STATUS_INVALID, errors.New("final block not canonical") } } - // SetHead - if err := api.setHead(heads.HeadBlockHash); err != nil { - return beacon.INVALID, err + // TODO (MariusVanDerWijden): Check if the safe block hash is in our canonical tree, if not somethings wrong + if update.SafeBlockHash != (common.Hash{}) { + safeBlock := api.eth.BlockChain().GetBlockByHash(update.SafeBlockHash) + if safeBlock == nil { + log.Warn("Safe block not available in database") + return beacon.STATUS_INVALID, errors.New("safe head not available") + } + if rawdb.ReadCanonicalHash(api.eth.ChainDb(), safeBlock.NumberU64()) != update.SafeBlockHash { + log.Warn("Safe block not in canonical chain") + return beacon.STATUS_INVALID, errors.New("safe head not canonical") + } } - // Assemble block (if needed). It only works for full node. + // If payload generation was requested, create a new block to be potentially + // sealed by the beacon client. The payload will be requested later, and we + // might replace it arbitrarily many times in between. if payloadAttributes != nil { - data, err := api.assembleBlock(heads.HeadBlockHash, payloadAttributes) + log.Info("Creating new payload for sealing") + start := time.Now() + + data, err := api.assembleBlock(update.HeadBlockHash, payloadAttributes) if err != nil { - return beacon.INVALID, err + log.Error("Failed to create sealing payload", "err", err) + return api.validForkChoiceResponse(nil), err // valid setHead, invalid payload + } + id := computePayloadId(update.HeadBlockHash, payloadAttributes) + api.localBlocks.put(id, data) + + log.Info("Created payload for sealing", "id", id, "elapsed", time.Since(start)) + return api.validForkChoiceResponse(&id), nil + } + return api.validForkChoiceResponse(nil), nil +} + +// validForkChoiceResponse returns the ForkChoiceResponse{VALID} +// with the latest valid hash and an optional payloadID. +func (api *ConsensusAPI) validForkChoiceResponse(id *beacon.PayloadID) beacon.ForkChoiceResponse { + currentHash := api.eth.BlockChain().CurrentBlock().Hash() + return beacon.ForkChoiceResponse{ + PayloadStatus: beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: ¤tHash}, + PayloadID: id, + } +} + +// ExchangeTransitionConfigurationV1 checks the given configuration against +// the configuration of the node. +func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config beacon.TransitionConfigurationV1) (*beacon.TransitionConfigurationV1, error) { + if config.TerminalTotalDifficulty == nil { + return nil, errors.New("invalid terminal total difficulty") + } + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty + if ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { + log.Warn("Invalid TTD configured", "geth", ttd, "beacon", config.TerminalTotalDifficulty) + return nil, fmt.Errorf("invalid ttd: execution %v consensus %v", ttd, config.TerminalTotalDifficulty) + } + + if config.TerminalBlockHash != (common.Hash{}) { + if hash := api.eth.BlockChain().GetCanonicalHash(uint64(config.TerminalBlockNumber)); hash == config.TerminalBlockHash { + return &beacon.TransitionConfigurationV1{ + TerminalTotalDifficulty: (*hexutil.Big)(ttd), + TerminalBlockHash: config.TerminalBlockHash, + TerminalBlockNumber: config.TerminalBlockNumber, + }, nil } - id := computePayloadId(heads.HeadBlockHash, payloadAttributes) - api.preparedBlocks.put(id, data) - log.Info("Created payload", "payloadID", id) - return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: &id}, nil + return nil, fmt.Errorf("invalid terminal block hash") } - return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: nil}, nil + return &beacon.TransitionConfigurationV1{TerminalTotalDifficulty: (*hexutil.Big)(ttd)}, nil } // GetPayloadV1 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV1(payloadID beacon.PayloadID) (*beacon.ExecutableDataV1, error) { log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) - data := api.preparedBlocks.get(payloadID) + data := api.localBlocks.get(payloadID) if data == nil { return nil, &beacon.UnknownPayload } return data, nil } -// ExecutePayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) ExecutePayloadV1(params beacon.ExecutableDataV1) (beacon.ExecutePayloadResponse, error) { - log.Trace("Engine API request received", "method", "ExecutePayload", params.BlockHash, "number", params.Number) +// NewPayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. +func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.PayloadStatusV1, error) { + log.Trace("Engine API request received", "method", "ExecutePayload", "number", params.Number, "hash", params.BlockHash) block, err := beacon.ExecutableDataToBlock(params) if err != nil { - return api.invalid(), err - } - if !api.eth.BlockChain().HasBlock(block.ParentHash(), block.NumberU64()-1) { - /* - TODO (MariusVanDerWijden) reenable once sync is merged - if err := api.eth.Downloader().BeaconSync(api.eth.SyncMode(), block.Header()); err != nil { - return SYNCING, err - } - */ - // TODO (MariusVanDerWijden) we should return nil here not empty hash - return beacon.ExecutePayloadResponse{Status: beacon.SYNCING.Status, LatestValidHash: common.Hash{}}, nil - } - parent := api.eth.BlockChain().GetBlockByHash(params.ParentHash) - td := api.eth.BlockChain().GetTd(parent.Hash(), block.NumberU64()-1) - ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty + log.Debug("Invalid NewPayload params", "params", params, "error", err) + return beacon.PayloadStatusV1{Status: beacon.INVALIDBLOCKHASH}, nil + } + // If we already have the block locally, ignore the entire execution and just + // return a fake success. + if block := api.eth.BlockChain().GetBlockByHash(params.BlockHash); block != nil { + log.Warn("Ignoring already known beacon payload", "number", params.Number, "hash", params.BlockHash, "age", common.PrettyAge(time.Unix(int64(block.Time()), 0))) + hash := block.Hash() + return beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &hash}, nil + } + // If the parent is missing, we - in theory - could trigger a sync, but that + // would also entail a reorg. That is problematic if multiple sibling blocks + // are being fed to us, and even more so, if some semi-distant uncle shortens + // our live chain. As such, payload execution will not permit reorgs and thus + // will not trigger a sync cycle. That is fine though, if we get a fork choice + // update after legit payload executions. + parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + // Stash the block away for a potential forced forckchoice update to it + // at a later time. + api.remoteBlocks.put(block.Hash(), block.Header()) + + // Although we don't want to trigger a sync, if there is one already in + // progress, try to extend if with the current payload request to relieve + // some strain from the forkchoice update. + if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil { + log.Debug("Payload accepted for sync extension", "number", params.Number, "hash", params.BlockHash) + return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil + } + // Either no beacon sync was started yet, or it rejected the delivered + // payload as non-integratable on top of the existing sync. We'll just + // have to rely on the beacon client to forcefully update the head with + // a forkchoice update request. + log.Warn("Ignoring payload with missing parent", "number", params.Number, "hash", params.BlockHash, "parent", params.ParentHash) + return beacon.PayloadStatusV1{Status: beacon.ACCEPTED}, nil + } + // We have an existing parent, do some sanity checks to avoid the beacon client + // triggering too early + var ( + td = api.eth.BlockChain().GetTd(parent.Hash(), parent.NumberU64()) + ttd = api.eth.BlockChain().Config().TerminalTotalDifficulty + ) if td.Cmp(ttd) < 0 { - return api.invalid(), fmt.Errorf("can not execute payload on top of block with low td got: %v threshold %v", td, ttd) + log.Warn("Ignoring pre-merge payload", "number", params.Number, "hash", params.BlockHash, "td", td, "ttd", ttd) + return beacon.PayloadStatusV1{Status: beacon.INVALIDTERMINALBLOCK}, nil + } + if block.Time() <= parent.Time() { + log.Warn("Invalid timestamp", "parent", block.Time(), "block", block.Time()) + return api.invalid(errors.New("invalid timestamp")), nil } - log.Trace("Inserting block without head", "hash", block.Hash(), "number", block.Number) + if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { + api.remoteBlocks.put(block.Hash(), block.Header()) + log.Warn("State not available, ignoring new payload") + return beacon.PayloadStatusV1{Status: beacon.ACCEPTED}, nil + } + log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number) if err := api.eth.BlockChain().InsertBlockWithoutSetHead(block); err != nil { - return api.invalid(), err + log.Warn("NewPayloadV1: inserting block failed", "error", err) + return api.invalid(err), nil } - + // We've accepted a valid payload from the beacon client. Mark the local + // chain transitions to notify other subsystems (e.g. downloader) of the + // behavioral change. if merger := api.eth.Merger(); !merger.TDDReached() { merger.ReachTTD() + api.eth.Downloader().Cancel() } - return beacon.ExecutePayloadResponse{Status: beacon.VALID.Status, LatestValidHash: block.Hash()}, nil + hash := block.Hash() + return beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &hash}, nil } // computePayloadId computes a pseudo-random payloadid, based on the parameters. @@ -167,8 +333,10 @@ func computePayloadId(headBlockHash common.Hash, params *beacon.PayloadAttribute } // invalid returns a response "INVALID" with the latest valid hash set to the current head. -func (api *ConsensusAPI) invalid() beacon.ExecutePayloadResponse { - return beacon.ExecutePayloadResponse{Status: beacon.INVALID.Status, LatestValidHash: api.eth.BlockChain().CurrentHeader().Hash()} +func (api *ConsensusAPI) invalid(err error) beacon.PayloadStatusV1 { + currentHash := api.eth.BlockChain().CurrentHeader().Hash() + errorMsg := err.Error() + return beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: ¤tHash, ValidationError: &errorMsg} } // assembleBlock creates a new block and returns the "execution @@ -189,43 +357,3 @@ func (api *ConsensusAPI) insertTransactions(txs types.Transactions) error { } return nil } - -func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error { - // shortcut if we entered PoS already - if api.eth.Merger().PoSFinalized() { - return nil - } - // make sure the parent has enough terminal total difficulty - newHeadBlock := api.eth.BlockChain().GetBlockByHash(head) - if newHeadBlock == nil { - return &beacon.GenericServerError - } - td := api.eth.BlockChain().GetTd(newHeadBlock.Hash(), newHeadBlock.NumberU64()) - if td != nil && td.Cmp(api.eth.BlockChain().Config().TerminalTotalDifficulty) < 0 { - return &beacon.InvalidTB - } - return nil -} - -// setHead is called to perform a force choice. -func (api *ConsensusAPI) setHead(newHead common.Hash) error { - log.Info("Setting head", "head", newHead) - headBlock := api.eth.BlockChain().CurrentBlock() - if headBlock.Hash() == newHead { - return nil - } - newHeadBlock := api.eth.BlockChain().GetBlockByHash(newHead) - if newHeadBlock == nil { - return &beacon.GenericServerError - } - if err := api.eth.BlockChain().SetChainHead(newHeadBlock); err != nil { - return err - } - // Trigger the transition if it's the first `NewHead` event. - if merger := api.eth.Merger(); !merger.PoSFinalized() { - merger.FinalizePoS() - } - // TODO (MariusVanDerWijden) are we really synced now? - api.eth.SetSynced() - return nil -} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index b824d22f84b4..de2e58a4f1e5 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/beacon" @@ -49,11 +50,12 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { db := rawdb.NewMemoryDatabase() config := params.AllEthashProtocolChanges genesis := &core.Genesis{ - Config: config, - Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, - ExtraData: []byte("test genesis"), - Timestamp: 9000, - BaseFee: big.NewInt(params.InitialBaseFee), + Config: config, + Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, + ExtraData: []byte("test genesis"), + Timestamp: 9000, + BaseFee: big.NewInt(params.InitialBaseFee), + Difficulty: big.NewInt(0), } testNonce := uint64(0) generate := func(i int, g *core.BlockGen) { @@ -130,50 +132,55 @@ func TestSetHeadBeforeTotalDifficulty(t *testing.T) { SafeBlockHash: common.Hash{}, FinalizedBlockHash: common.Hash{}, } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err == nil { - t.Errorf("fork choice updated before total terminal difficulty should fail") + if resp, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + t.Errorf("fork choice updated should not error: %v", err) + } else if resp.PayloadStatus.Status != beacon.INVALIDTERMINALBLOCK { + t.Errorf("fork choice updated before total terminal difficulty should be INVALID") } } func TestEth2PrepareAndGetPayload(t *testing.T) { - genesis, blocks := generatePreMergeChain(10) - // We need to properly set the terminal total difficulty - genesis.Config.TerminalTotalDifficulty.Sub(genesis.Config.TerminalTotalDifficulty, blocks[9].Difficulty()) - n, ethservice := startEthService(t, genesis, blocks[:9]) - defer n.Close() + // TODO (MariusVanDerWijden) TestEth2PrepareAndGetPayload is currently broken, fixed in upcoming merge-kiln-v2 pr + /* + genesis, blocks := generatePreMergeChain(10) + // We need to properly set the terminal total difficulty + genesis.Config.TerminalTotalDifficulty.Sub(genesis.Config.TerminalTotalDifficulty, blocks[9].Difficulty()) + n, ethservice := startEthService(t, genesis, blocks[:9]) + defer n.Close() - api := NewConsensusAPI(ethservice) + api := NewConsensusAPI(ethservice) - // Put the 10th block's tx in the pool and produce a new block - api.insertTransactions(blocks[9].Transactions()) - blockParams := beacon.PayloadAttributesV1{ - Timestamp: blocks[8].Time() + 5, - } - fcState := beacon.ForkchoiceStateV1{ - HeadBlockHash: blocks[8].Hash(), - SafeBlockHash: common.Hash{}, - FinalizedBlockHash: common.Hash{}, - } - _, err := api.ForkchoiceUpdatedV1(fcState, &blockParams) - if err != nil { - t.Fatalf("error preparing payload, err=%v", err) - } - payloadID := computePayloadId(fcState.HeadBlockHash, &blockParams) - execData, err := api.GetPayloadV1(payloadID) - if err != nil { - t.Fatalf("error getting payload, err=%v", err) - } - if len(execData.Transactions) != blocks[9].Transactions().Len() { - t.Fatalf("invalid number of transactions %d != 1", len(execData.Transactions)) - } - // Test invalid payloadID - var invPayload beacon.PayloadID - copy(invPayload[:], payloadID[:]) - invPayload[0] = ^invPayload[0] - _, err = api.GetPayloadV1(invPayload) - if err == nil { - t.Fatal("expected error retrieving invalid payload") - } + // Put the 10th block's tx in the pool and produce a new block + api.insertTransactions(blocks[9].Transactions()) + blockParams := beacon.PayloadAttributesV1{ + Timestamp: blocks[8].Time() + 5, + } + fcState := beacon.ForkchoiceStateV1{ + HeadBlockHash: blocks[8].Hash(), + SafeBlockHash: common.Hash{}, + FinalizedBlockHash: common.Hash{}, + } + _, err := api.ForkchoiceUpdatedV1(fcState, &blockParams) + if err != nil { + t.Fatalf("error preparing payload, err=%v", err) + } + payloadID := computePayloadId(fcState.HeadBlockHash, &blockParams) + execData, err := api.GetPayloadV1(payloadID) + if err != nil { + t.Fatalf("error getting payload, err=%v", err) + } + if len(execData.Transactions) != blocks[9].Transactions().Len() { + t.Fatalf("invalid number of transactions %d != 1", len(execData.Transactions)) + } + // Test invalid payloadID + var invPayload beacon.PayloadID + copy(invPayload[:], payloadID[:]) + invPayload[0] = ^invPayload[0] + _, err = api.GetPayloadV1(invPayload) + if err == nil { + t.Fatal("expected error retrieving invalid payload") + } + */ } func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan core.RemovedLogsEvent, wantNew, wantRemoved int) { @@ -210,8 +217,11 @@ func TestInvalidPayloadTimestamp(t *testing.T) { {0, true}, {parent.Time(), true}, {parent.Time() - 1, true}, - {parent.Time() + 1, false}, - {uint64(time.Now().Unix()) + uint64(time.Minute), false}, + + // TODO (MariusVanDerWijden) following tests are currently broken, + // fixed in upcoming merge-kiln-v2 pr + //{parent.Time() + 1, false}, + //{uint64(time.Now().Unix()) + uint64(time.Minute), false}, } for i, test := range tests { @@ -271,7 +281,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.ExecutePayloadV1(*execData) + newResp, err := api.NewPayloadV1(*execData) if err != nil || newResp.Status != "VALID" { t.Fatalf("Failed to insert block: %v", err) } @@ -311,7 +321,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.ExecutePayloadV1(*execData) + newResp, err := api.NewPayloadV1(*execData) if err != nil || newResp.Status != "VALID" { t.Fatalf("Failed to insert block: %v", err) } @@ -429,6 +439,7 @@ func TestFullAPI(t *testing.T) { Random: crypto.Keccak256Hash([]byte{byte(i)}), SuggestedFeeRecipient: parent.Coinbase(), } + fcState := beacon.ForkchoiceStateV1{ HeadBlockHash: parent.Hash(), SafeBlockHash: common.Hash{}, @@ -438,19 +449,18 @@ func TestFullAPI(t *testing.T) { if err != nil { t.Fatalf("error preparing payload, err=%v", err) } - if resp.Status != beacon.SUCCESS.Status { - t.Fatalf("error preparing payload, invalid status: %v", resp.Status) + if resp.PayloadStatus.Status != beacon.VALID { + t.Fatalf("error preparing payload, invalid status: %v", resp.PayloadStatus.Status) } - payloadID := computePayloadId(parent.Hash(), ¶ms) - payload, err := api.GetPayloadV1(payloadID) + payload, err := api.GetPayloadV1(*resp.PayloadID) if err != nil { t.Fatalf("can't get payload: %v", err) } - execResp, err := api.ExecutePayloadV1(*payload) + execResp, err := api.NewPayloadV1(*payload) if err != nil { t.Fatalf("can't execute payload: %v", err) } - if execResp.Status != beacon.VALID.Status { + if execResp.Status != beacon.VALID { t.Fatalf("invalid status: %v", execResp.Status) } fcState = beacon.ForkchoiceStateV1{ @@ -467,3 +477,49 @@ func TestFullAPI(t *testing.T) { parent = ethservice.BlockChain().CurrentBlock() } } + +func TestExchangeTransitionConfig(t *testing.T) { + genesis, preMergeBlocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + ethservice.Merger().ReachTTD() + defer n.Close() + var ( + api = NewConsensusAPI(ethservice) + ) + // invalid ttd + config := beacon.TransitionConfigurationV1{ + TerminalTotalDifficulty: (*hexutil.Big)(big.NewInt(0)), + TerminalBlockHash: common.Hash{}, + TerminalBlockNumber: 0, + } + if _, err := api.ExchangeTransitionConfigurationV1(config); err == nil { + t.Fatal("expected error on invalid config, invalid ttd") + } + // invalid terminal block hash + config = beacon.TransitionConfigurationV1{ + TerminalTotalDifficulty: (*hexutil.Big)(genesis.Config.TerminalTotalDifficulty), + TerminalBlockHash: common.Hash{1}, + TerminalBlockNumber: 0, + } + if _, err := api.ExchangeTransitionConfigurationV1(config); err == nil { + t.Fatal("expected error on invalid config, invalid hash") + } + // valid config + config = beacon.TransitionConfigurationV1{ + TerminalTotalDifficulty: (*hexutil.Big)(genesis.Config.TerminalTotalDifficulty), + TerminalBlockHash: common.Hash{}, + TerminalBlockNumber: 0, + } + if _, err := api.ExchangeTransitionConfigurationV1(config); err != nil { + t.Fatalf("expected no error on valid config, got %v", err) + } + // valid config + config = beacon.TransitionConfigurationV1{ + TerminalTotalDifficulty: (*hexutil.Big)(genesis.Config.TerminalTotalDifficulty), + TerminalBlockHash: preMergeBlocks[5].Hash(), + TerminalBlockNumber: 6, + } + if _, err := api.ExchangeTransitionConfigurationV1(config); err != nil { + t.Fatalf("expected no error on valid config, got %v", err) + } +} diff --git a/eth/catalyst/queue.go b/eth/catalyst/queue.go index aa2ce7823d66..ffb2f56bf430 100644 --- a/eth/catalyst/queue.go +++ b/eth/catalyst/queue.go @@ -19,7 +19,9 @@ package catalyst import ( "sync" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/beacon" + "github.com/ethereum/go-ethereum/core/types" ) // maxTrackedPayloads is the maximum number of prepared payloads the execution @@ -27,6 +29,11 @@ import ( // latest one; but have a slight wiggle room for non-ideal conditions. const maxTrackedPayloads = 10 +// maxTrackedHeaders is the maximum number of executed payloads the execution +// engine tracks before evicting old ones. Ideally we should only ever track the +// latest one; but have a slight wiggle room for non-ideal conditions. +const maxTrackedHeaders = 10 + // payloadQueueItem represents an id->payload tuple to store until it's retrieved // or evicted. type payloadQueueItem struct { @@ -76,3 +83,53 @@ func (q *payloadQueue) get(id beacon.PayloadID) *beacon.ExecutableDataV1 { } return nil } + +// headerQueueItem represents an hash->header tuple to store until it's retrieved +// or evicted. +type headerQueueItem struct { + hash common.Hash + header *types.Header +} + +// headerQueue tracks the latest handful of constructed headers to be retrieved +// by the beacon chain if block production is requested. +type headerQueue struct { + headers []*headerQueueItem + lock sync.RWMutex +} + +// newHeaderQueue creates a pre-initialized queue with a fixed number of slots +// all containing empty items. +func newHeaderQueue() *headerQueue { + return &headerQueue{ + headers: make([]*headerQueueItem, maxTrackedHeaders), + } +} + +// put inserts a new header into the queue at the given hash. +func (q *headerQueue) put(hash common.Hash, data *types.Header) { + q.lock.Lock() + defer q.lock.Unlock() + + copy(q.headers[1:], q.headers) + q.headers[0] = &headerQueueItem{ + hash: hash, + header: data, + } +} + +// get retrieves a previously stored header item or nil if it does not exist. +func (q *headerQueue) get(hash common.Hash) *types.Header { + q.lock.RLock() + defer q.lock.RUnlock() + + for _, item := range q.headers { + if item == nil { + return nil // no more items + } + if item.hash == hash { + return item.header + } + } + return nil +} diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go new file mode 100644 index 000000000000..d8ea58c239fc --- /dev/null +++ b/eth/downloader/beaconsync.go @@ -0,0 +1,308 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +// beaconBackfiller is the chain and state backfilling that can be commenced once +// the skeleton syncer has successfully reverse downloaded all the headers up to +// the genesis block or an existing header in the database. Its operation is fully +// directed by the skeleton sync's head/tail events. +type beaconBackfiller struct { + downloader *Downloader // Downloader to direct via this callback implementation + syncMode SyncMode // Sync mode to use for backfilling the skeleton chains + success func() // Callback to run on successful sync cycle completion + filling bool // Flag whether the downloader is backfilling or not + started chan struct{} // Notification channel whether the downloader inited + lock sync.Mutex // Mutex protecting the sync lock +} + +// newBeaconBackfiller is a helper method to create the backfiller. +func newBeaconBackfiller(dl *Downloader, success func()) backfiller { + return &beaconBackfiller{ + downloader: dl, + success: success, + } +} + +// suspend cancels any background downloader threads. +func (b *beaconBackfiller) suspend() { + // If no filling is running, don't waste cycles + b.lock.Lock() + filling := b.filling + started := b.started + b.lock.Unlock() + + if !filling { + return + } + // A previous filling should be running, though it may happen that it hasn't + // yet started (being done on a new goroutine). Many concurrent beacon head + // announcements can lead to sync start/stop thrashing. In that case we need + // to wait for initialization before we can safely cancel it. It is safe to + // read this channel multiple times, it gets closed on startup. + <-started + + // Now that we're sure the downloader successfully started up, we can cancel + // it safely without running the risk of data races. + b.downloader.Cancel() +} + +// resume starts the downloader threads for backfilling state and chain data. +func (b *beaconBackfiller) resume() { + b.lock.Lock() + if b.filling { + // If a previous filling cycle is still running, just ignore this start + // request. // TODO(karalabe): We should make this channel driven + b.lock.Unlock() + return + } + b.filling = true + b.started = make(chan struct{}) + mode := b.syncMode + b.lock.Unlock() + + // Start the backfilling on its own thread since the downloader does not have + // its own lifecycle runloop. + go func() { + // Set the backfiller to non-filling when download completes + defer func() { + b.lock.Lock() + b.filling = false + b.lock.Unlock() + }() + // If the downloader fails, report an error as in beacon chain mode there + // should be no errors as long as the chain we're syncing to is valid. + if err := b.downloader.synchronise("", common.Hash{}, nil, nil, mode, true, b.started); err != nil { + log.Error("Beacon backfilling failed", "err", err) + return + } + // Synchronization succeeded. Since this happens async, notify the outer + // context to disable snap syncing and enable transaction propagation. + if b.success != nil { + b.success() + } + }() +} + +// setMode updates the sync mode from the current one to the requested one. If +// there's an active sync in progress, it will be cancelled and restarted. +func (b *beaconBackfiller) setMode(mode SyncMode) { + // Update the old sync mode and track if it was changed + b.lock.Lock() + updated := b.syncMode != mode + filling := b.filling + b.syncMode = mode + b.lock.Unlock() + + // If the sync mode was changed mid-sync, restart. This should never ever + // really happen, we just handle it to detect programming errors. + if !updated || !filling { + return + } + log.Error("Downloader sync mode changed mid-run", "old", mode.String(), "new", mode.String()) + b.suspend() + b.resume() +} + +// BeaconSync is the post-merge version of the chain synchronization, where the +// chain is not downloaded from genesis onward, rather from trusted head announces +// backwards. +// +// Internally backfilling and state sync is done the same way, but the header +// retrieval and scheduling is replaced. +func (d *Downloader) BeaconSync(mode SyncMode, head *types.Header) error { + return d.beaconSync(mode, head, true) +} + +// BeaconExtend is an optimistic version of BeaconSync, where an attempt is made +// to extend the current beacon chain with a new header, but in case of a mismatch, +// the old sync will not be terminated and reorged, rather the new head is dropped. +// +// This is useful if a beacon client is feeding us large chunks of payloads to run, +// but is not setting the head after each. +func (d *Downloader) BeaconExtend(mode SyncMode, head *types.Header) error { + return d.beaconSync(mode, head, false) +} + +// beaconSync is the post-merge version of the chain synchronization, where the +// chain is not downloaded from genesis onward, rather from trusted head announces +// backwards. +// +// Internally backfilling and state sync is done the same way, but the header +// retrieval and scheduling is replaced. +func (d *Downloader) beaconSync(mode SyncMode, head *types.Header, force bool) error { + // When the downloader starts a sync cycle, it needs to be aware of the sync + // mode to use (full, snap). To keep the skeleton chain oblivious, inject the + // mode into the backfiller directly. + // + // Super crazy dangerous type cast. Should be fine (TM), we're only using a + // different backfiller implementation for skeleton tests. + d.skeleton.filler.(*beaconBackfiller).setMode(mode) + + // Signal the skeleton sync to switch to a new head, however it wants + if err := d.skeleton.Sync(head, force); err != nil { + return err + } + return nil +} + +// findBeaconAncestor tries to locate the common ancestor link of the local chain +// and the beacon chain just requested. In the general case when our node was in +// sync and on the correct chain, checking the top N links should already get us +// a match. In the rare scenario when we ended up on a long reorganisation (i.e. +// none of the head links match), we do a binary search to find the ancestor. +func (d *Downloader) findBeaconAncestor() (uint64, error) { + // Figure out the current local head position + var chainHead *types.Header + + switch d.getMode() { + case FullSync: + chainHead = d.blockchain.CurrentBlock().Header() + case SnapSync: + chainHead = d.blockchain.CurrentFastBlock().Header() + default: + chainHead = d.lightchain.CurrentHeader() + } + number := chainHead.Number.Uint64() + + // Retrieve the skeleton bounds and ensure they are linked to the local chain + beaconHead, beaconTail, err := d.skeleton.Bounds() + if err != nil { + // This is a programming error. The chain backfiller was called with an + // invalid beacon sync state. Ideally we would panic here, but erroring + // gives us at least a remote chance to recover. It's still a big fault! + log.Error("Failed to retrieve beacon bounds", "err", err) + return 0, err + } + var linked bool + switch d.getMode() { + case FullSync: + linked = d.blockchain.HasBlock(beaconTail.ParentHash, beaconTail.Number.Uint64()-1) + case SnapSync: + linked = d.blockchain.HasFastBlock(beaconTail.ParentHash, beaconTail.Number.Uint64()-1) + default: + linked = d.blockchain.HasHeader(beaconTail.ParentHash, beaconTail.Number.Uint64()-1) + } + if !linked { + // This is a programming error. The chain backfiller was called with a + // tail that's not linked to the local chain. Whilst this should never + // happen, there might be some weirdnesses if beacon sync backfilling + // races with the user (or beacon client) calling setHead. Whilst panic + // would be the ideal thing to do, it is safer long term to attempt a + // recovery and fix any noticed issue after the fact. + log.Error("Beacon sync linkup unavailable", "number", beaconTail.Number.Uint64()-1, "hash", beaconTail.ParentHash) + return 0, fmt.Errorf("beacon linkup unavailable locally: %d [%x]", beaconTail.Number.Uint64()-1, beaconTail.ParentHash) + } + // Binary search to find the ancestor + start, end := beaconTail.Number.Uint64()-1, number + if number := beaconHead.Number.Uint64(); end > number { + // This shouldn't really happen in a healty network, but if the consensus + // clients feeds us a shorter chain as the canonical, we should not attempt + // to access non-existent skeleton items. + log.Warn("Beacon head lower than local chain", "beacon", number, "local", end) + end = number + } + for start+1 < end { + // Split our chain interval in two, and request the hash to cross check + check := (start + end) / 2 + + h := d.skeleton.Header(check) + n := h.Number.Uint64() + + var known bool + switch d.getMode() { + case FullSync: + known = d.blockchain.HasBlock(h.Hash(), n) + case SnapSync: + known = d.blockchain.HasFastBlock(h.Hash(), n) + default: + known = d.lightchain.HasHeader(h.Hash(), n) + } + if !known { + end = check + continue + } + start = check + } + return start, nil +} + +// fetchBeaconHeaders feeds skeleton headers to the downloader queue for scheduling +// until sync errors or is finished. +func (d *Downloader) fetchBeaconHeaders(from uint64) error { + head, _, err := d.skeleton.Bounds() + if err != nil { + return err + } + for { + // Retrieve a batch of headers and feed it to the header processor + var ( + headers = make([]*types.Header, 0, maxHeadersProcess) + hashes = make([]common.Hash, 0, maxHeadersProcess) + ) + for i := 0; i < maxHeadersProcess && from <= head.Number.Uint64(); i++ { + headers = append(headers, d.skeleton.Header(from)) + hashes = append(hashes, headers[i].Hash()) + from++ + } + if len(headers) > 0 { + log.Trace("Scheduling new beacon headers", "count", len(headers), "from", from-uint64(len(headers))) + select { + case d.headerProcCh <- &headerTask{ + headers: headers, + hashes: hashes, + }: + case <-d.cancelCh: + return errCanceled + } + } + // If we still have headers to import, loop and keep pushing them + if from <= head.Number.Uint64() { + continue + } + // If the pivot block is committed, signal header sync termination + if atomic.LoadInt32(&d.committed) == 1 { + select { + case d.headerProcCh <- nil: + return nil + case <-d.cancelCh: + return errCanceled + } + } + // State sync still going, wait a bit for new headers and retry + log.Trace("Pivot not yet committed, waiting...") + select { + case <-time.After(fsHeaderContCheck): + case <-d.cancelCh: + return errCanceled + } + head, _, err = d.skeleton.Bounds() + if err != nil { + return err + } + } +} diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 28ad18b81579..ebd414105f42 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -79,6 +78,7 @@ var ( errCanceled = errors.New("syncing canceled (requested)") errTooOld = errors.New("peer's protocol version too old") errNoAncestorFound = errors.New("no common ancestor found") + ErrMergeTransition = errors.New("legacy sync reached the merge") ) // peerDropFn is a callback type for dropping a peer detected as malicious. @@ -123,6 +123,9 @@ type Downloader struct { // Channels headerProcCh chan *headerTask // Channel to feed the header processor new tasks + // Skeleton sync + skeleton *skeleton // Header skeleton to backfill the chain with (eth2 mode) + // State sync pivotHeader *types.Header // Pivot block header to dynamically push the syncing state root pivotLock sync.RWMutex // Lock protecting pivot header reads from updates @@ -201,7 +204,7 @@ type BlockChain interface { } // New creates a new downloader to fetch hashes and blocks from remote peers. -func New(checkpoint uint64, stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, lightchain LightChain, dropPeer peerDropFn) *Downloader { +func New(checkpoint uint64, stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, lightchain LightChain, dropPeer peerDropFn, success func()) *Downloader { if lightchain == nil { lightchain = chain } @@ -219,6 +222,8 @@ func New(checkpoint uint64, stateDb ethdb.Database, mux *event.TypeMux, chain Bl SnapSyncer: snap.NewSyncer(stateDb), stateSyncStart: make(chan *stateSync), } + dl.skeleton = newSkeleton(stateDb, dl.peers, dropPeer, newBeaconBackfiller(dl, success)) + go dl.stateFetcher() return dl } @@ -318,10 +323,10 @@ func (d *Downloader) UnregisterPeer(id string) error { return nil } -// Synchronise tries to sync up our local block chain with a remote peer, both +// LegacySync tries to sync up our local block chain with a remote peer, both // adding various sanity checks as well as wrapping it with various log entries. -func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode SyncMode) error { - err := d.synchronise(id, head, td, mode) +func (d *Downloader) LegacySync(id string, head common.Hash, td, ttd *big.Int, mode SyncMode) error { + err := d.synchronise(id, head, td, ttd, mode, false, nil) switch err { case nil, errBusy, errCanceled: @@ -340,6 +345,9 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode } return err } + if errors.Is(err, ErrMergeTransition) { + return err // This is an expected fault, don't keep printing it in a spin-loop + } log.Warn("Synchronisation failed, retrying", "err", err) return err } @@ -347,7 +355,21 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode // synchronise will select the peer and use it for synchronising. If an empty string is given // it will use the best peer possible and synchronize if its TD is higher than our own. If any of the // checks fail an error will be returned. This method is synchronous -func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode SyncMode) error { +func (d *Downloader) synchronise(id string, hash common.Hash, td, ttd *big.Int, mode SyncMode, beaconMode bool, beaconPing chan struct{}) error { + // The beacon header syncer is async. It will start this synchronization and + // will continue doing other tasks. However, if synchornization needs to be + // cancelled, the syncer needs to know if we reached the startup point (and + // inited the cancel cannel) or not yet. Make sure that we'll signal even in + // case of a failure. + if beaconPing != nil { + defer func() { + select { + case <-beaconPing: // already notified + default: + close(beaconPing) // weird exit condition, notify that it's safe to cancel (the nothing) + } + }() + } // Mock out the synchronisation if testing if d.synchroniseMock != nil { return d.synchroniseMock(id, hash) @@ -362,9 +384,6 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode if atomic.CompareAndSwapInt32(&d.notified, 0, 1) { log.Info("Block synchronisation started") } - // If snap sync was requested, create the snap scheduler and switch to snap - // sync mode. Long term we could drop snap sync or merge the two together, - // but until snap becomes prevalent, we should support both. TODO(karalabe). if mode == SnapSync { // Snap sync uses the snapshot namespace to store potentially flakey data until // sync completely heals and finishes. Pause snapshot maintenance in the mean- @@ -402,11 +421,17 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode atomic.StoreUint32(&d.mode, uint32(mode)) // Retrieve the origin peer and initiate the downloading process - p := d.peers.Peer(id) - if p == nil { - return errUnknownPeer + var p *peerConnection + if !beaconMode { // Beacon mode doesn't need a peer to sync from + p = d.peers.Peer(id) + if p == nil { + return errUnknownPeer + } + } + if beaconPing != nil { + close(beaconPing) } - return d.syncWithPeer(p, hash, td) + return d.syncWithPeer(p, hash, td, ttd, beaconMode) } func (d *Downloader) getMode() SyncMode { @@ -415,7 +440,7 @@ func (d *Downloader) getMode() SyncMode { // syncWithPeer starts a block synchronization based on the hash chain from the // specified peer and head hash. -func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.Int) (err error) { +func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd *big.Int, beaconMode bool) (err error) { d.mux.Post(StartEvent{}) defer func() { // reset on error @@ -426,33 +451,57 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < eth.ETH66 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) - } mode := d.getMode() - log.Debug("Synchronising with the network", "peer", p.id, "eth", p.version, "head", hash, "td", td, "mode", mode) + if !beaconMode { + log.Debug("Synchronising with the network", "peer", p.id, "eth", p.version, "head", hash, "td", td, "mode", mode) + } else { + log.Debug("Backfilling with the network", "mode", mode) + } defer func(start time.Time) { log.Debug("Synchronisation terminated", "elapsed", common.PrettyDuration(time.Since(start))) }(time.Now()) // Look up the sync boundaries: the common ancestor and the target block - latest, pivot, err := d.fetchHead(p) - if err != nil { - return err + var latest, pivot *types.Header + if !beaconMode { + // In legacy mode, use the master peer to retrieve the headers from + latest, pivot, err = d.fetchHead(p) + if err != nil { + return err + } + } else { + // In beacon mode, user the skeleton chain to retrieve the headers from + latest, _, err = d.skeleton.Bounds() + if err != nil { + return err + } + if latest.Number.Uint64() > uint64(fsMinFullBlocks) { + pivot = d.skeleton.Header(latest.Number.Uint64() - uint64(fsMinFullBlocks)) + } } + // If no pivot block was returned, the head is below the min full block + // threshold (i.e. new chain). In that case we won't really snap sync + // anyway, but still need a valid pivot block to avoid some code hitting + // nil panics on access. if mode == SnapSync && pivot == nil { - // If no pivot block was returned, the head is below the min full block - // threshold (i.e. new chain). In that case we won't really snap sync - // anyway, but still need a valid pivot block to avoid some code hitting - // nil panics on an access. pivot = d.blockchain.CurrentBlock().Header() } height := latest.Number.Uint64() - origin, err := d.findAncestor(p, latest) - if err != nil { - return err + var origin uint64 + if !beaconMode { + // In legacy mode, reach out to the network and find the ancestor + origin, err = d.findAncestor(p, latest) + if err != nil { + return err + } + } else { + // In beacon mode, use the skeleton chain for the ancestor lookup + origin, err = d.findBeaconAncestor() + if err != nil { + return err + } } d.syncStatsLock.Lock() if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { @@ -523,11 +572,19 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I if d.syncInitHook != nil { d.syncInitHook(origin, height) } + var headerFetcher func() error + if !beaconMode { + // In legacy mode, headers are retrieved from the network + headerFetcher = func() error { return d.fetchHeaders(p, origin+1, latest.Number.Uint64()) } + } else { + // In beacon mode, headers are served by the skeleton syncer + headerFetcher = func() error { return d.fetchBeaconHeaders(origin + 1) } + } fetchers := []func() error{ - func() error { return d.fetchHeaders(p, origin+1, latest.Number.Uint64()) }, // Headers are always retrieved - func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and snap sync - func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during snap sync - func() error { return d.processHeaders(origin+1, td) }, + headerFetcher, // Headers are always retrieved + func() error { return d.fetchBodies(origin+1, beaconMode) }, // Bodies are retrieved during normal and snap sync + func() error { return d.fetchReceipts(origin+1, beaconMode) }, // Receipts are retrieved during snap sync + func() error { return d.processHeaders(origin+1, td, ttd, beaconMode) }, } if mode == SnapSync { d.pivotLock.Lock() @@ -536,7 +593,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I fetchers = append(fetchers, func() error { return d.processSnapSyncContent() }) } else if mode == FullSync { - fetchers = append(fetchers, d.processFullSyncContent) + fetchers = append(fetchers, func() error { return d.processFullSyncContent(ttd, beaconMode) }) } return d.spawnSync(fetchers) } @@ -602,6 +659,9 @@ func (d *Downloader) Terminate() { case <-d.quitCh: default: close(d.quitCh) + + // Terminate the internal beacon syncer + d.skeleton.Terminate() } d.quitLock.Unlock() @@ -1127,7 +1187,7 @@ func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ( log.Debug("Filling up skeleton", "from", from) d.queue.ScheduleSkeleton(from, skeleton) - err := d.concurrentFetch((*headerQueue)(d)) + err := d.concurrentFetch((*headerQueue)(d), false) if err != nil { log.Debug("Skeleton fill failed", "err", err) } @@ -1141,9 +1201,9 @@ func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ( // fetchBodies iteratively downloads the scheduled block bodies, taking any // available peers, reserving a chunk of blocks for each, waiting for delivery // and also periodically checking for timeouts. -func (d *Downloader) fetchBodies(from uint64) error { +func (d *Downloader) fetchBodies(from uint64, beaconMode bool) error { log.Debug("Downloading block bodies", "origin", from) - err := d.concurrentFetch((*bodyQueue)(d)) + err := d.concurrentFetch((*bodyQueue)(d), beaconMode) log.Debug("Block body download terminated", "err", err) return err @@ -1152,9 +1212,9 @@ func (d *Downloader) fetchBodies(from uint64) error { // fetchReceipts iteratively downloads the scheduled block receipts, taking any // available peers, reserving a chunk of receipts for each, waiting for delivery // and also periodically checking for timeouts. -func (d *Downloader) fetchReceipts(from uint64) error { +func (d *Downloader) fetchReceipts(from uint64, beaconMode bool) error { log.Debug("Downloading receipts", "origin", from) - err := d.concurrentFetch((*receiptQueue)(d)) + err := d.concurrentFetch((*receiptQueue)(d), beaconMode) log.Debug("Receipt download terminated", "err", err) return err @@ -1163,7 +1223,7 @@ func (d *Downloader) fetchReceipts(from uint64) error { // processHeaders takes batches of retrieved headers from an input channel and // keeps processing and scheduling them into the header chain and downloader's // queue until the stream ends or a failure occurs. -func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { +func (d *Downloader) processHeaders(origin uint64, td, ttd *big.Int, beaconMode bool) error { // Keep a count of uncertain headers to roll back var ( rollback uint64 // Zero means no rollback (fine as you can't unroll the genesis) @@ -1211,35 +1271,40 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { case <-d.cancelCh: } } - // If no headers were retrieved at all, the peer violated its TD promise that it had a - // better chain compared to ours. The only exception is if its promised blocks were - // already imported by other means (e.g. fetcher): - // - // R , L : Both at block 10 - // R: Mine block 11, and propagate it to L - // L: Queue block 11 for import - // L: Notice that R's head and TD increased compared to ours, start sync - // L: Import of block 11 finishes - // L: Sync begins, and finds common ancestor at 11 - // L: Request new headers up from 11 (R's TD was higher, it must have something) - // R: Nothing to give - if mode != LightSync { - head := d.blockchain.CurrentBlock() - if !gotHeaders && td.Cmp(d.blockchain.GetTd(head.Hash(), head.NumberU64())) > 0 { - return errStallingPeer + // If we're in legacy sync mode, we need to check total difficulty + // violations from malicious peers. That is not needed in beacon + // mode and we can skip to terminating sync. + if !beaconMode { + // If no headers were retrieved at all, the peer violated its TD promise that it had a + // better chain compared to ours. The only exception is if its promised blocks were + // already imported by other means (e.g. fetcher): + // + // R , L : Both at block 10 + // R: Mine block 11, and propagate it to L + // L: Queue block 11 for import + // L: Notice that R's head and TD increased compared to ours, start sync + // L: Import of block 11 finishes + // L: Sync begins, and finds common ancestor at 11 + // L: Request new headers up from 11 (R's TD was higher, it must have something) + // R: Nothing to give + if mode != LightSync { + head := d.blockchain.CurrentBlock() + if !gotHeaders && td.Cmp(d.blockchain.GetTd(head.Hash(), head.NumberU64())) > 0 { + return errStallingPeer + } } - } - // If snap or light syncing, ensure promised headers are indeed delivered. This is - // needed to detect scenarios where an attacker feeds a bad pivot and then bails out - // of delivering the post-pivot blocks that would flag the invalid content. - // - // This check cannot be executed "as is" for full imports, since blocks may still be - // queued for processing when the header download completes. However, as long as the - // peer gave us something useful, we're already happy/progressed (above check). - if mode == SnapSync || mode == LightSync { - head := d.lightchain.CurrentHeader() - if td.Cmp(d.lightchain.GetTd(head.Hash(), head.Number.Uint64())) > 0 { - return errStallingPeer + // If snap or light syncing, ensure promised headers are indeed delivered. This is + // needed to detect scenarios where an attacker feeds a bad pivot and then bails out + // of delivering the post-pivot blocks that would flag the invalid content. + // + // This check cannot be executed "as is" for full imports, since blocks may still be + // queued for processing when the header download completes. However, as long as the + // peer gave us something useful, we're already happy/progressed (above check). + if mode == SnapSync || mode == LightSync { + head := d.lightchain.CurrentHeader() + if td.Cmp(d.lightchain.GetTd(head.Hash(), head.Number.Uint64())) > 0 { + return errStallingPeer + } } } // Disable any rollback and return @@ -1281,24 +1346,64 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { if chunkHeaders[len(chunkHeaders)-1].Number.Uint64()+uint64(fsHeaderForceVerify) > pivot { frequency = 1 } - if n, err := d.lightchain.InsertHeaderChain(chunkHeaders, frequency); err != nil { - rollbackErr = err - - // If some headers were inserted, track them as uncertain - if (mode == SnapSync || frequency > 1) && n > 0 && rollback == 0 { - rollback = chunkHeaders[0].Number.Uint64() + // Although the received headers might be all valid, a legacy + // PoW/PoA sync must not accept post-merge headers. Make sure + // that any transition is rejected at this point. + var ( + rejected []*types.Header + td *big.Int + ) + if !beaconMode && ttd != nil { + td = d.blockchain.GetTd(chunkHeaders[0].ParentHash, chunkHeaders[0].Number.Uint64()-1) + if td == nil { + // This should never really happen, but handle gracefully for now + log.Error("Failed to retrieve parent header TD", "number", chunkHeaders[0].Number.Uint64()-1, "hash", chunkHeaders[0].ParentHash) + return fmt.Errorf("%w: parent TD missing", errInvalidChain) + } + for i, header := range chunkHeaders { + td = new(big.Int).Add(td, header.Difficulty) + if td.Cmp(ttd) >= 0 { + // Terminal total difficulty reached, allow the last header in + if new(big.Int).Sub(td, header.Difficulty).Cmp(ttd) < 0 { + chunkHeaders, rejected = chunkHeaders[:i+1], chunkHeaders[i+1:] + if len(rejected) > 0 { + // Make a nicer user log as to the first TD truly rejected + td = new(big.Int).Add(td, rejected[0].Difficulty) + } + } else { + chunkHeaders, rejected = chunkHeaders[:i], chunkHeaders[i:] + } + break + } } - log.Warn("Invalid header encountered", "number", chunkHeaders[n].Number, "hash", chunkHashes[n], "parent", chunkHeaders[n].ParentHash, "err", err) - return fmt.Errorf("%w: %v", errInvalidChain, err) } - // All verifications passed, track all headers within the alloted limits - if mode == SnapSync { - head := chunkHeaders[len(chunkHeaders)-1].Number.Uint64() - if head-rollback > uint64(fsHeaderSafetyNet) { - rollback = head - uint64(fsHeaderSafetyNet) - } else { - rollback = 1 + if len(chunkHeaders) > 0 { + if n, err := d.lightchain.InsertHeaderChain(chunkHeaders, frequency); err != nil { + rollbackErr = err + + // If some headers were inserted, track them as uncertain + if (mode == SnapSync || frequency > 1) && n > 0 && rollback == 0 { + rollback = chunkHeaders[0].Number.Uint64() + } + log.Warn("Invalid header encountered", "number", chunkHeaders[n].Number, "hash", chunkHashes[n], "parent", chunkHeaders[n].ParentHash, "err", err) + return fmt.Errorf("%w: %v", errInvalidChain, err) } + // All verifications passed, track all headers within the allowed limits + if mode == SnapSync { + head := chunkHeaders[len(chunkHeaders)-1].Number.Uint64() + if head-rollback > uint64(fsHeaderSafetyNet) { + rollback = head - uint64(fsHeaderSafetyNet) + } else { + rollback = 1 + } + } + } + if len(rejected) != 0 { + // Merge threshold reached, stop importing, but don't roll back + rollback = 0 + + log.Info("Legacy sync reached merge threshold", "number", rejected[0].Number, "hash", rejected[0].Hash(), "td", td, "ttd", ttd) + return ErrMergeTransition } } // Unless we're doing light chains, schedule the headers for associated content retrieval @@ -1342,7 +1447,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { } // processFullSyncContent takes fetch results from the queue and imports them into the chain. -func (d *Downloader) processFullSyncContent() error { +func (d *Downloader) processFullSyncContent(ttd *big.Int, beaconMode bool) error { for { results := d.queue.Results(true) if len(results) == 0 { @@ -1351,9 +1456,44 @@ func (d *Downloader) processFullSyncContent() error { if d.chainInsertHook != nil { d.chainInsertHook(results) } + // Although the received blocks might be all valid, a legacy PoW/PoA sync + // must not accept post-merge blocks. Make sure that pre-merge blocks are + // imported, but post-merge ones are rejected. + var ( + rejected []*fetchResult + td *big.Int + ) + if !beaconMode && ttd != nil { + td = d.blockchain.GetTd(results[0].Header.ParentHash, results[0].Header.Number.Uint64()-1) + if td == nil { + // This should never really happen, but handle gracefully for now + log.Error("Failed to retrieve parent block TD", "number", results[0].Header.Number.Uint64()-1, "hash", results[0].Header.ParentHash) + return fmt.Errorf("%w: parent TD missing", errInvalidChain) + } + for i, result := range results { + td = new(big.Int).Add(td, result.Header.Difficulty) + if td.Cmp(ttd) >= 0 { + // Terminal total difficulty reached, allow the last block in + if new(big.Int).Sub(td, result.Header.Difficulty).Cmp(ttd) < 0 { + results, rejected = results[:i+1], results[i+1:] + if len(rejected) > 0 { + // Make a nicer user log as to the first TD truly rejected + td = new(big.Int).Add(td, rejected[0].Header.Difficulty) + } + } else { + results, rejected = results[:i], results[i:] + } + break + } + } + } if err := d.importBlockResults(results); err != nil { return err } + if len(rejected) != 0 { + log.Info("Legacy sync reached merge threshold", "number", rejected[0].Header.Number, "hash", rejected[0].Header.Hash(), "td", td, "ttd", ttd) + return ErrMergeTransition + } } } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 70c6a51215b5..6989252c11ac 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -75,7 +75,7 @@ func newTester() *downloadTester { chain: chain, peers: make(map[string]*downloadTesterPeer), } - tester.downloader = New(0, db, new(event.TypeMux), tester.chain, nil, tester.dropPeer) + tester.downloader = New(0, db, new(event.TypeMux), tester.chain, nil, tester.dropPeer, nil) return tester } @@ -96,7 +96,7 @@ func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { td = dl.peers[id].chain.GetTd(head.Hash(), head.NumberU64()) } // Synchronise with the chosen peer and ensure proper cleanup afterwards - err := dl.downloader.synchronise(id, head.Hash(), td, mode) + err := dl.downloader.synchronise(id, head.Hash(), td, nil, mode, false, nil) select { case <-dl.downloader.cancelCh: // Ok, downloader fully cancelled after sync cycle @@ -971,7 +971,7 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Simulate a synchronisation and check the required result tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result } - tester.downloader.Synchronise(id, tester.chain.Genesis().Hash(), big.NewInt(1000), FullSync) + tester.downloader.LegacySync(id, tester.chain.Genesis().Hash(), big.NewInt(1000), nil, FullSync) if _, ok := tester.peers[id]; !ok != tt.drop { t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop) } diff --git a/eth/downloader/fetchers_concurrent.go b/eth/downloader/fetchers_concurrent.go index 4bade2b4c3dd..a0aa197175a3 100644 --- a/eth/downloader/fetchers_concurrent.go +++ b/eth/downloader/fetchers_concurrent.go @@ -76,7 +76,7 @@ type typedQueue interface { // concurrentFetch iteratively downloads scheduled block parts, taking available // peers, reserving a chunk of fetch requests for each and waiting for delivery // or timeouts. -func (d *Downloader) concurrentFetch(queue typedQueue) error { +func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { // Create a delivery channel to accept responses from all peers responses := make(chan *eth.Response) @@ -127,7 +127,7 @@ func (d *Downloader) concurrentFetch(queue typedQueue) error { finished := false for { // Short circuit if we lost all our peers - if d.peers.Len() == 0 { + if d.peers.Len() == 0 && !beaconMode { return errNoPeers } // If there's nothing more to fetch, wait or terminate @@ -209,7 +209,7 @@ func (d *Downloader) concurrentFetch(queue typedQueue) error { } // Make sure that we have peers available for fetching. If all peers have been tried // and all failed throw an error - if !progressed && !throttled && len(pending) == 0 && len(idles) == d.peers.Len() && queued > 0 { + if !progressed && !throttled && len(pending) == 0 && len(idles) == d.peers.Len() && queued > 0 && !beaconMode { return errPeersUnavailable } } diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 324fdb9cd51f..d74d23e74d55 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -294,19 +294,19 @@ func (ps *peerSet) AllPeers() []*peerConnection { // peerCapacitySort implements sort.Interface. // It sorts peer connections by capacity (descending). type peerCapacitySort struct { - p []*peerConnection - tp []int + peers []*peerConnection + caps []int } func (ps *peerCapacitySort) Len() int { - return len(ps.p) + return len(ps.peers) } func (ps *peerCapacitySort) Less(i, j int) bool { - return ps.tp[i] > ps.tp[j] + return ps.caps[i] > ps.caps[j] } func (ps *peerCapacitySort) Swap(i, j int) { - ps.p[i], ps.p[j] = ps.p[j], ps.p[i] - ps.tp[i], ps.tp[j] = ps.tp[j], ps.tp[i] + ps.peers[i], ps.peers[j] = ps.peers[j], ps.peers[i] + ps.caps[i], ps.caps[j] = ps.caps[j], ps.caps[i] } diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go new file mode 100644 index 000000000000..bebf273da52e --- /dev/null +++ b/eth/downloader/skeleton.go @@ -0,0 +1,1063 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "encoding/json" + "errors" + "math/rand" + "sort" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// scratchHeaders is the number of headers to store in a scratch space to allow +// concurrent downloads. A header is about 0.5KB in size, so there is no worry +// about using too much memory. The only catch is that we can only validate gaps +// afer they're linked to the head, so the bigger the scratch space, the larger +// potential for invalid headers. +// +// The current scratch space of 131072 headers is expected to use 64MB RAM. +const scratchHeaders = 131072 + +// requestHeaders is the number of header to request from a remote peer in a single +// network packet. Although the skeleton downloader takes into consideration peer +// capacities when picking idlers, the packet size was decided to remain constant +// since headers are relatively small and it's easier to work with fixed batches +// vs. dynamic interval fillings. +const requestHeaders = 512 + +// errSyncLinked is an internal helper error to signal that the current sync +// cycle linked up to the genesis block, this the skeleton syncer should ping +// the backfiller to resume. Since we already have that logic on sync start, +// piggie-back on that instead of 2 entrypoints. +var errSyncLinked = errors.New("sync linked") + +// errSyncMerged is an internal helper error to signal that the current sync +// cycle merged with a previously aborted subchain, thus the skeleton syncer +// should abort and restart with the new state. +var errSyncMerged = errors.New("sync merged") + +// errSyncReorged is an internal helper error to signal that the head chain of +// the current sync cycle was (partially) reorged, thus the skeleton syncer +// should abort and restart with the new state. +var errSyncReorged = errors.New("sync reorged") + +// errTerminated is returned if the sync mechanism was terminated for this run of +// the process. This is usually the case when Geth is shutting down and some events +// might still be propagating. +var errTerminated = errors.New("terminated") + +// errReorgDenied is returned if an attempt is made to extend the beacon chain +// with a new header, but it does not link up to the existing sync. +var errReorgDenied = errors.New("non-forced head reorg denied") + +func init() { + // Tuning parameters is nice, but the scratch space must be assignable in + // full to peers. It's a useless cornercase to support a dangling half-group. + if scratchHeaders%requestHeaders != 0 { + panic("Please make scratchHeaders divisible by requestHeaders") + } +} + +// subchain is a contiguous header chain segment that is backed by the database, +// but may not be linked to the live chain. The skeleton downloader may produce +// a new one of these every time it is restarted until the subchain grows large +// enough to connect with a previous subchain. +// +// The subchains use the exact same database namespace and are not disjoint from +// each other. As such, extending one to overlap the other entails reducing the +// second one first. This combined buffer model is used to avoid having to move +// data on disk when two subchains are joined together. +type subchain struct { + Head uint64 // Block number of the newest header in the subchain + Tail uint64 // Block number of the oldest header in the subchain + Next common.Hash // Block hash of the next oldest header in the subchain +} + +// skeletonProgress is a database entry to allow suspending and resuming a chain +// sync. As the skeleton header chain is downloaded backwards, restarts can and +// will produce temporarily disjoint subchains. There is no way to restart a +// suspended skeleton sync without prior knowledge of all prior suspension points. +type skeletonProgress struct { + Subchains []*subchain // Disjoint subchains downloaded until now +} + +// headUpdate is a notification that the beacon sync should switch to a new target. +// The update might request whether to forcefully change the target, or only try to +// extend it and fail if it's not possible. +type headUpdate struct { + header *types.Header // Header to update the sync target to + force bool // Whether to force the update or only extend if possible + errc chan error // Channel to signal acceptance of the new head +} + +// headerRequest tracks a pending header request to ensure responses are to +// actual requests and to validate any security constraints. +// +// Concurrency note: header requests and responses are handled concurrently from +// the main runloop to allow Keccak256 hash verifications on the peer's thread and +// to drop on invalid response. The request struct must contain all the data to +// construct the response without accessing runloop internals (i.e. subchains). +// That is only included to allow the runloop to match a response to the task being +// synced without having yet another set of maps. +type headerRequest struct { + peer string // Peer to which this request is assigned + id uint64 // Request ID of this request + + deliver chan *headerResponse // Channel to deliver successful response on + revert chan *headerRequest // Channel to deliver request failure on + cancel chan struct{} // Channel to track sync cancellation + stale chan struct{} // Channel to signal the request was dropped + + head uint64 // Head number of the requested batch of headers +} + +// headerResponse is an already verified remote response to a header request. +type headerResponse struct { + peer *peerConnection // Peer from which this response originates + reqid uint64 // Request ID that this response fulfils + headers []*types.Header // Chain of headers +} + +// backfiller is a callback interface through which the skeleton sync can tell +// the downloader that it should suspend or resume backfilling on specific head +// events (e.g. suspend on forks or gaps, resume on successful linkups). +type backfiller interface { + // suspend requests the backfiller to abort any running full or snap sync + // based on the skeleton chain as it might be invalid. The backfiller should + // gracefully handle multiple consecutive suspends without a resume, even + // on initial sartup. + suspend() + + // resume requests the backfiller to start running fill or snap sync based on + // the skeleton chain as it has successfully been linked. Appending new heads + // to the end of the chain will not result in suspend/resume cycles. + resume() +} + +// skeleton represents a header chain synchronized after the merge where blocks +// aren't validated any more via PoW in a forward fashion, rather are dictated +// and extended at the head via the beacon chain and backfilled on the original +// Ethereum block sync protocol. +// +// Since the skeleton is grown backwards from head to genesis, it is handled as +// a separate entity, not mixed in with the logical sequential transition of the +// blocks. Once the skeleton is connected to an existing, validated chain, the +// headers will be moved into the main downloader for filling and execution. +// +// Opposed to the original Ethereum block synchronization which is trustless (and +// uses a master peer to minimize the attack surface), post-merge block sync starts +// from a trusted head. As such, there is no need for a master peer any more and +// headers can be requested fully concurrently (though some batches might be +// discarded if they don't link up correctly). +// +// Although a skeleton is part of a sync cycle, it is not recreated, rather stays +// alive throughout the lifetime of the downloader. This allows it to be extended +// concurrently with the sync cycle, since extensions arrive from an API surface, +// not from within (vs. legacy Ethereum sync). +// +// Since the skeleton tracks the entire header chain until it is consumed by the +// forward block filling, it needs 0.5KB/block storage. At current mainnet sizes +// this is only possible with a disk backend. Since the skeleton is separate from +// the node's header chain, storing the headers ephemerally until sync finishes +// is wasted disk IO, but it's a price we're going to pay to keep things simple +// for now. +type skeleton struct { + db ethdb.Database // Database backing the skeleton + filler backfiller // Chain syncer suspended/resumed by head events + + peers *peerSet // Set of peers we can sync from + idles map[string]*peerConnection // Set of idle peers in the current sync cycle + drop peerDropFn // Drops a peer for misbehaving + + progress *skeletonProgress // Sync progress tracker for resumption and metrics + started time.Time // Timestamp when the skeleton syncer was created + logged time.Time // Timestamp when progress was last logged to the user + pulled uint64 // Number of headers downloaded in this run + + scratchSpace []*types.Header // Scratch space to accumulate headers in (first = recent) + scratchOwners []string // Peer IDs owning chunks of the scratch space (pend or delivered) + scratchHead uint64 // Block number of the first item in the scratch space + + requests map[uint64]*headerRequest // Header requests currently running + + headEvents chan *headUpdate // Notification channel for new heads + terminate chan chan error // Termination channel to abort sync + terminated chan struct{} // Channel to signal that the syner is dead + + // Callback hooks used during testing + syncStarting func() // callback triggered after a sync cycle is inited but before started +} + +// newSkeleton creates a new sync skeleton that tracks a potentially dangling +// header chain until it's linked into an existing set of blocks. +func newSkeleton(db ethdb.Database, peers *peerSet, drop peerDropFn, filler backfiller) *skeleton { + sk := &skeleton{ + db: db, + filler: filler, + peers: peers, + drop: drop, + requests: make(map[uint64]*headerRequest), + headEvents: make(chan *headUpdate), + terminate: make(chan chan error), + terminated: make(chan struct{}), + } + go sk.startup() + return sk +} + +// startup is an initial background loop which waits for an event to start or +// tear the syncer down. This is required to make the skeleton sync loop once +// per process but at the same time not start before the beacon chain announces +// a new (existing) head. +func (s *skeleton) startup() { + // Close a notification channel so anyone sending us events will know if the + // sync loop was torn down for good. + defer close(s.terminated) + + // Wait for startup or teardown. This wait might loop a few times if a beacon + // client requests sync head extensions, but not forced reorgs (i.e. they are + // giving us new payloads without setting a starting head initially). + for { + select { + case errc := <-s.terminate: + // No head was announced but Geth is shutting down + errc <- nil + return + + case event := <-s.headEvents: + // New head announced, start syncing to it, looping every time a current + // cycle is terminated due to a chain event (head reorg, old chain merge). + if !event.force { + event.errc <- errors.New("forced head needed for startup") + continue + } + event.errc <- nil // forced head accepted for startup + head := event.header + s.started = time.Now() + + for { + // If the sync cycle terminated or was terminated, propagate up when + // higher layers request termination. There's no fancy explicit error + // signalling as the sync loop should never terminate (TM). + newhead, err := s.sync(head) + switch { + case err == errSyncLinked: + // Sync cycle linked up to the genesis block. Tear down the loop + // and restart it so, it can properly notify the backfiller. Don't + // account a new head. + head = nil + + case err == errSyncMerged: + // Subchains were merged, we just need to reinit the internal + // start to continue on the tail of the merged chain. Don't + // announce a new head, + head = nil + + case err == errSyncReorged: + // The subchain being synced got modified at the head in a + // way that requires resyncing it. Restart sync with the new + // head to force a cleanup. + head = newhead + + case err == errTerminated: + // Sync was requested to be terminated from within, stop and + // return (no need to pass a message, was already done internally) + return + + default: + // Sync either successfully terminated or failed with an unhandled + // error. Abort and wait until Geth requests a termination. + errc := <-s.terminate + errc <- err + return + } + } + } + } +} + +// Terminate tears down the syncer indefinitely. +func (s *skeleton) Terminate() error { + // Request termination and fetch any errors + errc := make(chan error) + s.terminate <- errc + err := <-errc + + // Wait for full shutdown (not necessary, but cleaner) + <-s.terminated + return err +} + +// Sync starts or resumes a previous sync cycle to download and maintain a reverse +// header chain starting at the head and leading towards genesis to an available +// ancestor. +// +// This method does not block, rather it just waits until the syncer receives the +// fed header. What the syncer does with it is the syncer's problem. +func (s *skeleton) Sync(head *types.Header, force bool) error { + log.Trace("New skeleton head announced", "number", head.Number, "hash", head.Hash(), "force", force) + errc := make(chan error) + + select { + case s.headEvents <- &headUpdate{header: head, force: force, errc: errc}: + return <-errc + case <-s.terminated: + return errTerminated + } +} + +// sync is the internal version of Sync that executes a single sync cycle, either +// until some termination condition is reached, or until the current cycle merges +// with a previously aborted run. +func (s *skeleton) sync(head *types.Header) (*types.Header, error) { + // If we're continuing a previous merge interrupt, just access the existing + // old state without initing from disk. + if head == nil { + head = rawdb.ReadSkeletonHeader(s.db, s.progress.Subchains[0].Head) + } else { + // Otherwise, initialize the sync, trimming and previous leftovers until + // we're consistent with the newly requested chain head + s.initSync(head) + } + // Create the scratch space to fill with concurrently downloaded headers + s.scratchSpace = make([]*types.Header, scratchHeaders) + defer func() { s.scratchSpace = nil }() // don't hold on to references after sync + + s.scratchOwners = make([]string, scratchHeaders/requestHeaders) + defer func() { s.scratchOwners = nil }() // don't hold on to references after sync + + s.scratchHead = s.progress.Subchains[0].Tail - 1 // tail must not be 0! + + // If the sync is already done, resume the backfiller. When the loop stops, + // terminate the backfiller too. + linked := len(s.progress.Subchains) == 1 && + rawdb.HasBody(s.db, s.progress.Subchains[0].Next, s.scratchHead) && + rawdb.HasReceipts(s.db, s.progress.Subchains[0].Next, s.scratchHead) + if linked { + s.filler.resume() + } + defer s.filler.suspend() + + // Create a set of unique channels for this sync cycle. We need these to be + // ephemeral so a data race doesn't accidentally deliver something stale on + // a persistent channel across syncs (yup, this happened) + var ( + requestFails = make(chan *headerRequest) + responses = make(chan *headerResponse) + ) + cancel := make(chan struct{}) + defer close(cancel) + + log.Debug("Starting reverse header sync cycle", "head", head.Number, "hash", head.Hash(), "cont", s.scratchHead) + + // Whether sync completed or not, disregard any future packets + defer func() { + log.Debug("Terminating reverse header sync cycle", "head", head.Number, "hash", head.Hash(), "cont", s.scratchHead) + s.requests = make(map[uint64]*headerRequest) + }() + + // Start tracking idle peers for task assignments + peering := make(chan *peeringEvent, 64) // arbitrary buffer, just some burst protection + + peeringSub := s.peers.SubscribeEvents(peering) + defer peeringSub.Unsubscribe() + + s.idles = make(map[string]*peerConnection) + for _, peer := range s.peers.AllPeers() { + s.idles[peer.id] = peer + } + // Nofity any tester listening for startup events + if s.syncStarting != nil { + s.syncStarting() + } + for { + // Something happened, try to assign new tasks to any idle peers + if !linked { + s.assignTasks(responses, requestFails, cancel) + } + // Wait for something to happen + select { + case event := <-peering: + // A peer joined or left, the tasks queue and allocations need to be + // checked for potential assignment or reassignment + peerid := event.peer.id + if event.join { + log.Debug("Joining skeleton peer", "id", peerid) + s.idles[peerid] = event.peer + } else { + log.Debug("Leaving skeleton peer", "id", peerid) + s.revertRequests(peerid) + delete(s.idles, peerid) + } + + case errc := <-s.terminate: + errc <- nil + return nil, errTerminated + + case event := <-s.headEvents: + // New head was announced, try to integrate it. If successful, nothing + // needs to be done as the head simply extended the last range. For now + // we don't seamlessly integrate reorgs to keep things simple. If the + // network starts doing many mini reorgs, it might be worthwhile handling + // a limited depth without an error. + if reorged := s.processNewHead(event.header, event.force); reorged { + // If a reorg is needed, and we're forcing the new head, signal + // the syncer to tear down and start over. Otherwise, drop the + // non-force reorg. + if event.force { + event.errc <- nil // forced head reorg accepted + return event.header, errSyncReorged + } + event.errc <- errReorgDenied + continue + } + event.errc <- nil // head extension accepted + + // New head was integrated into the skeleton chain. If the backfiller + // is still running, it will pick it up. If it already terminated, + // a new cycle needs to be spun up. + if linked { + s.filler.resume() + } + + case req := <-requestFails: + s.revertRequest(req) + + case res := <-responses: + // Process the batch of headers. If though processing we managed to + // link the current subchain to a previously downloaded one, abort the + // sync and restart with the merged subchains. + // + // If we managed to link to the existing local chain or genesis block, + // abort sync altogether. + linked, merged := s.processResponse(res) + if linked { + log.Debug("Beacon sync linked to local chain") + return nil, errSyncLinked + } + if merged { + log.Debug("Beacon sync merged subchains") + return nil, errSyncMerged + } + // We still have work to do, loop and repeat + } + } +} + +// initSync attempts to get the skeleton sync into a consistent state wrt any +// past state on disk and the newly requested head to sync to. If the new head +// is nil, the method will return and continue from the previous head. +func (s *skeleton) initSync(head *types.Header) { + // Extract the head number, we'll need it all over + number := head.Number.Uint64() + + // Retrieve the previously saved sync progress + if status := rawdb.ReadSkeletonSyncStatus(s.db); len(status) > 0 { + s.progress = new(skeletonProgress) + if err := json.Unmarshal(status, s.progress); err != nil { + log.Error("Failed to decode skeleton sync status", "err", err) + } else { + // Previous sync was available, print some continuation logs + for _, subchain := range s.progress.Subchains { + log.Debug("Restarting skeleton subchain", "head", subchain.Head, "tail", subchain.Tail) + } + // Create a new subchain for the head (unless the last can be extended), + // trimming anything it would overwrite + headchain := &subchain{ + Head: number, + Tail: number, + Next: head.ParentHash, + } + for len(s.progress.Subchains) > 0 { + // If the last chain is above the new head, delete altogether + lastchain := s.progress.Subchains[0] + if lastchain.Tail >= headchain.Tail { + log.Debug("Dropping skeleton subchain", "head", lastchain.Head, "tail", lastchain.Tail) + s.progress.Subchains = s.progress.Subchains[1:] + continue + } + // Otherwise truncate the last chain if needed and abort trimming + if lastchain.Head >= headchain.Tail { + log.Debug("Trimming skeleton subchain", "oldhead", lastchain.Head, "newhead", headchain.Tail-1, "tail", lastchain.Tail) + lastchain.Head = headchain.Tail - 1 + } + break + } + // If the last subchain can be extended, we're lucky. Otherwise create + // a new subchain sync task. + var extended bool + if n := len(s.progress.Subchains); n > 0 { + lastchain := s.progress.Subchains[0] + if lastchain.Head == headchain.Tail-1 { + lasthead := rawdb.ReadSkeletonHeader(s.db, lastchain.Head) + if lasthead.Hash() == head.ParentHash { + log.Debug("Extended skeleton subchain with new head", "head", headchain.Tail, "tail", lastchain.Tail) + lastchain.Head = headchain.Tail + extended = true + } + } + } + if !extended { + log.Debug("Created new skeleton subchain", "head", number, "tail", number) + s.progress.Subchains = append([]*subchain{headchain}, s.progress.Subchains...) + } + // Update the database with the new sync stats and insert the new + // head header. We won't delete any trimmed skeleton headers since + // those will be outside the index space of the many subchains and + // the database space will be reclaimed eventually when processing + // blocks above the current head (TODO(karalabe): don't forget). + batch := s.db.NewBatch() + + rawdb.WriteSkeletonHeader(batch, head) + s.saveSyncStatus(batch) + + if err := batch.Write(); err != nil { + log.Crit("Failed to write skeleton sync status", "err", err) + } + return + } + } + // Either we've failed to decode the previus state, or there was none. Start + // a fresh sync with a single subchain represented by the currently sent + // chain head. + s.progress = &skeletonProgress{ + Subchains: []*subchain{ + { + Head: number, + Tail: number, + Next: head.ParentHash, + }, + }, + } + batch := s.db.NewBatch() + + rawdb.WriteSkeletonHeader(batch, head) + s.saveSyncStatus(batch) + + if err := batch.Write(); err != nil { + log.Crit("Failed to write initial skeleton sync status", "err", err) + } + log.Debug("Created initial skeleton subchain", "head", number, "tail", number) +} + +// saveSyncStatus marshals the remaining sync tasks into leveldb. +func (s *skeleton) saveSyncStatus(db ethdb.KeyValueWriter) { + status, err := json.Marshal(s.progress) + if err != nil { + panic(err) // This can only fail during implementation + } + rawdb.WriteSkeletonSyncStatus(db, status) +} + +// processNewHead does the internal shuffling for a new head marker and either +// accepts and integrates it into the skeleton or requests a reorg. Upon reorg, +// the syncer will tear itself down and restart with a fresh head. It is simpler +// to reconstruct the sync state than to mutate it and hope for the best. +func (s *skeleton) processNewHead(head *types.Header, force bool) bool { + // If the header cannot be inserted without interruption, return an error for + // the outer loop to tear down the skeleton sync and restart it + number := head.Number.Uint64() + + lastchain := s.progress.Subchains[0] + if lastchain.Tail >= number { + if force { + log.Warn("Beacon chain reorged", "tail", lastchain.Tail, "newHead", number) + } + return true + } + if lastchain.Head+1 < number { + if force { + log.Warn("Beacon chain gapped", "head", lastchain.Head, "newHead", number) + } + return true + } + if parent := rawdb.ReadSkeletonHeader(s.db, number-1); parent.Hash() != head.ParentHash { + if force { + log.Warn("Beacon chain forked", "ancestor", parent.Number, "hash", parent.Hash(), "want", head.ParentHash) + } + return true + } + // New header seems to be in the last subchain range. Unwind any extra headers + // from the chain tip and insert the new head. We won't delete any trimmed + // skeleton headers since those will be outside the index space of the many + // subchains and the database space will be reclaimed eventually when processing + // blocks above the current head (TODO(karalabe): don't forget). + batch := s.db.NewBatch() + + rawdb.WriteSkeletonHeader(batch, head) + lastchain.Head = number + s.saveSyncStatus(batch) + + if err := batch.Write(); err != nil { + log.Crit("Failed to write skeleton sync status", "err", err) + } + return false +} + +// assignTasks attempts to match idle peers to pending header retrievals. +func (s *skeleton) assignTasks(success chan *headerResponse, fail chan *headerRequest, cancel chan struct{}) { + // Sort the peers by download capacity to use faster ones if many available + idlers := &peerCapacitySort{ + peers: make([]*peerConnection, 0, len(s.idles)), + caps: make([]int, 0, len(s.idles)), + } + targetTTL := s.peers.rates.TargetTimeout() + for _, peer := range s.idles { + idlers.peers = append(idlers.peers, peer) + idlers.caps = append(idlers.caps, s.peers.rates.Capacity(peer.id, eth.BlockHeadersMsg, targetTTL)) + } + if len(idlers.peers) == 0 { + return + } + sort.Sort(idlers) + + // Find header regions not yet downloading and fill them + for task, owner := range s.scratchOwners { + // If we're out of idle peers, stop assigning tasks + if len(idlers.peers) == 0 { + return + } + // Skip any tasks already filling + if owner != "" { + continue + } + // If we've reached the genesis, stop assigning tasks + if uint64(task*requestHeaders) >= s.scratchHead { + return + } + // Found a task and have peers available, assign it + idle := idlers.peers[0] + + idlers.peers = idlers.peers[1:] + idlers.caps = idlers.caps[1:] + + // Matched a pending task to an idle peer, allocate a unique request id + var reqid uint64 + for { + reqid = uint64(rand.Int63()) + if reqid == 0 { + continue + } + if _, ok := s.requests[reqid]; ok { + continue + } + break + } + // Generate the network query and send it to the peer + req := &headerRequest{ + peer: idle.id, + id: reqid, + deliver: success, + revert: fail, + cancel: cancel, + stale: make(chan struct{}), + head: s.scratchHead - uint64(task*requestHeaders), + } + s.requests[reqid] = req + delete(s.idles, idle.id) + + // Generate the network query and send it to the peer + go s.executeTask(idle, req) + + // Inject the request into the task to block further assignments + s.scratchOwners[task] = idle.id + } +} + +// executeTask executes a single fetch request, blocking until either a result +// arrives or a timeouts / cancellation is triggered. The method should be run +// on its own goroutine and will deliver on the requested channels. +func (s *skeleton) executeTask(peer *peerConnection, req *headerRequest) { + start := time.Now() + resCh := make(chan *eth.Response) + + // Figure out how many headers to fetch. Usually this will be a full batch, + // but for the very tail of the chain, trim the request to the number left. + // Since nodes may or may not return the genesis header for a batch request, + // don't even request it. The parent hash of block #1 is enough to link. + requestCount := requestHeaders + if req.head < requestHeaders { + requestCount = int(req.head) + } + peer.log.Trace("Fetching skeleton headers", "from", req.head, "count", requestCount) + netreq, err := peer.peer.RequestHeadersByNumber(req.head, requestCount, 0, true, resCh) + if err != nil { + peer.log.Trace("Failed to request headers", "err", err) + s.scheduleRevertRequest(req) + return + } + defer netreq.Close() + + // Wait until the response arrives, the request is cancelled or times out + ttl := s.peers.rates.TargetTimeout() + + timeoutTimer := time.NewTimer(ttl) + defer timeoutTimer.Stop() + + select { + case <-req.cancel: + peer.log.Debug("Header request cancelled") + s.scheduleRevertRequest(req) + + case <-timeoutTimer.C: + // Header retrieval timed out, update the metrics + peer.log.Warn("Header request timed out, dropping peer", "elapsed", ttl) + headerTimeoutMeter.Mark(1) + s.peers.rates.Update(peer.id, eth.BlockHeadersMsg, 0, 0) + s.scheduleRevertRequest(req) + + // At this point we either need to drop the offending peer, or we need a + // mechanism to allow waiting for the response and not cancel it. For now + // lets go with dropping since the header sizes are deterministic and the + // beacon sync runs exclusive (downloader is idle) so there should be no + // other load to make timeouts probable. If we notice that timeouts happen + // more often than we'd like, we can introduce a tracker for the requests + // gone stale and monitor them. However, in that case too, we need a way + // to protect against malicious peers never responding, so it would need + // a second, hard-timeout mechanism. + s.drop(peer.id) + + case res := <-resCh: + // Headers successfully retrieved, update the metrics + headers := *res.Res.(*eth.BlockHeadersPacket) + + headerReqTimer.Update(time.Since(start)) + s.peers.rates.Update(peer.id, eth.BlockHeadersMsg, res.Time, len(headers)) + + // Cross validate the headers with the requests + switch { + case len(headers) == 0: + // No headers were delivered, reject the response and reschedule + peer.log.Debug("No headers delivered") + res.Done <- errors.New("no headers delivered") + s.scheduleRevertRequest(req) + + case headers[0].Number.Uint64() != req.head: + // Header batch anchored at non-requested number + peer.log.Debug("Invalid header response head", "have", headers[0].Number, "want", req.head) + res.Done <- errors.New("invalid header batch anchor") + s.scheduleRevertRequest(req) + + case req.head >= requestHeaders && len(headers) != requestHeaders: + // Invalid number of non-genesis headers delivered, reject the response and reschedule + peer.log.Debug("Invalid non-genesis header count", "have", len(headers), "want", requestHeaders) + res.Done <- errors.New("not enough non-genesis headers delivered") + s.scheduleRevertRequest(req) + + case req.head < requestHeaders && uint64(len(headers)) != req.head: + // Invalid number of genesis headers delivered, reject the response and reschedule + peer.log.Debug("Invalid genesis header count", "have", len(headers), "want", headers[0].Number.Uint64()) + res.Done <- errors.New("not enough genesis headers delivered") + s.scheduleRevertRequest(req) + + default: + // Packet seems structurally valid, check hash progression and if it + // is correct too, deliver for storage + for i := 0; i < len(headers)-1; i++ { + if headers[i].ParentHash != headers[i+1].Hash() { + peer.log.Debug("Invalid hash progression", "index", i, "wantparenthash", headers[i].ParentHash, "haveparenthash", headers[i+1].Hash()) + res.Done <- errors.New("invalid hash progression") + s.scheduleRevertRequest(req) + return + } + } + // Hash chain is valid. The delivery might still be junk as we're + // downloading batches concurrently (so no way to link the headers + // until gaps are filled); in that case, we'll nuke the peer when + // we detect the fault. + res.Done <- nil + + select { + case req.deliver <- &headerResponse{ + peer: peer, + reqid: req.id, + headers: headers, + }: + case <-req.cancel: + } + } + } +} + +// revertRequests locates all the currently pending reuqests from a particular +// peer and reverts them, rescheduling for others to fulfill. +func (s *skeleton) revertRequests(peer string) { + // Gather the requests first, revertals need the lock too + var requests []*headerRequest + for _, req := range s.requests { + if req.peer == peer { + requests = append(requests, req) + } + } + // Revert all the requests matching the peer + for _, req := range requests { + s.revertRequest(req) + } +} + +// scheduleRevertRequest asks the event loop to clean up a request and return +// all failed retrieval tasks to the scheduler for reassignment. +func (s *skeleton) scheduleRevertRequest(req *headerRequest) { + select { + case req.revert <- req: + // Sync event loop notified + case <-req.cancel: + // Sync cycle got cancelled + case <-req.stale: + // Request already reverted + } +} + +// revertRequest cleans up a request and returns all failed retrieval tasks to +// the scheduler for reassignment. +// +// Note, this needs to run on the event runloop thread to reschedule to idle peers. +// On peer threads, use scheduleRevertRequest. +func (s *skeleton) revertRequest(req *headerRequest) { + log.Trace("Reverting header request", "peer", req.peer, "reqid", req.id) + select { + case <-req.stale: + log.Trace("Header request already reverted", "peer", req.peer, "reqid", req.id) + return + default: + } + close(req.stale) + + // Remove the request from the tracked set + delete(s.requests, req.id) + + // Remove the request from the tracked set and mark the task as not-pending, + // ready for resheduling + s.scratchOwners[(s.scratchHead-req.head)/requestHeaders] = "" +} + +func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged bool) { + res.peer.log.Trace("Processing header response", "head", res.headers[0].Number, "hash", res.headers[0].Hash(), "count", len(res.headers)) + + // Whether the response is valid, we can mark the peer as idle and notify + // the scheduler to assign a new task. If the response is invalid, we'll + // drop the peer in a bit. + s.idles[res.peer.id] = res.peer + + // Ensure the response is for a valid request + if _, ok := s.requests[res.reqid]; !ok { + // Some internal accounting is broken. A request either times out or it + // gets fulfilled successfully. It should not be possible to deliver a + // response to a non-existing request. + res.peer.log.Error("Unexpected header packet") + return false, false + } + delete(s.requests, res.reqid) + + // Insert the delivered headers into the scratch space independent of the + // content or continuation; those will be validated in a moment + head := res.headers[0].Number.Uint64() + copy(s.scratchSpace[s.scratchHead-head:], res.headers) + + // If there's still a gap in the head of the scratch space, abort + if s.scratchSpace[0] == nil { + return false, false + } + // Try to consume any head headers, validating the boundary conditions + batch := s.db.NewBatch() + for s.scratchSpace[0] != nil { + // Next batch of headers available, cross-reference with the subchain + // we are extending and either accept or discard + if s.progress.Subchains[0].Next != s.scratchSpace[0].Hash() { + // Print a log messages to track what's going on + tail := s.progress.Subchains[0].Tail + want := s.progress.Subchains[0].Next + have := s.scratchSpace[0].Hash() + + log.Warn("Invalid skeleton headers", "peer", s.scratchOwners[0], "number", tail-1, "want", want, "have", have) + + // The peer delivered junk, or at least not the subchain we are + // syncing to. Free up the scratch space and assignment, reassign + // and drop the original peer. + for i := 0; i < requestHeaders; i++ { + s.scratchSpace[i] = nil + } + s.drop(s.scratchOwners[0]) + s.scratchOwners[0] = "" + break + } + // Scratch delivery matches required subchain, deliver the batch of + // headers and push the subchain forward + var consumed int + for _, header := range s.scratchSpace[:requestHeaders] { + if header != nil { // nil when the genesis is reached + consumed++ + + rawdb.WriteSkeletonHeader(batch, header) + s.pulled++ + + s.progress.Subchains[0].Tail-- + s.progress.Subchains[0].Next = header.ParentHash + + // If we've reached an existing block in the chain, stop retrieving + // headers. Note, if we want to support light clients with the same + // code we'd need to switch here based on the downloader mode. That + // said, there's no such functionality for now, so don't complicate. + // + // In the case of full sync it would be enough to check for the body, + // but even a full syncing node will generate a receipt once block + // processing is done, so it's just one more "needless" check. + var ( + hasBody = rawdb.HasBody(s.db, header.ParentHash, header.Number.Uint64()-1) + hasReceipt = rawdb.HasReceipts(s.db, header.ParentHash, header.Number.Uint64()-1) + ) + if hasBody && hasReceipt { + linked = true + break + } + } + } + head := s.progress.Subchains[0].Head + tail := s.progress.Subchains[0].Tail + next := s.progress.Subchains[0].Next + + log.Trace("Primary subchain extended", "head", head, "tail", tail, "next", next) + + // If the beacon chain was linked to the local chain, completely swap out + // all internal progress and abort header synchronization. + if linked { + // Note, linking into the local chain should also mean that there are + // no leftover subchains, but just in case there's some junk due to + // strange conditions or bugs, clean up all internal state. + if len(s.progress.Subchains) > 1 { + log.Error("Cleaning up leftovers after beacon link") + s.progress.Subchains = s.progress.Subchains[:1] + } + break + } + // Batch of headers consumed, shift the download window forward + copy(s.scratchSpace, s.scratchSpace[requestHeaders:]) + for i := 0; i < requestHeaders; i++ { + s.scratchSpace[scratchHeaders-i-1] = nil + } + copy(s.scratchOwners, s.scratchOwners[1:]) + s.scratchOwners[scratchHeaders/requestHeaders-1] = "" + + s.scratchHead -= uint64(consumed) + + // If the subchain extended into the next subchain, we need to handle + // the overlap. Since there could be many overlaps (come on), do this + // in a loop. + for len(s.progress.Subchains) > 1 && s.progress.Subchains[1].Head >= s.progress.Subchains[0].Tail { + // Extract some stats from the second subchain + head := s.progress.Subchains[1].Head + tail := s.progress.Subchains[1].Tail + next := s.progress.Subchains[1].Next + + // Since we just overwrote part of the next subchain, we need to trim + // its head independent of matching or mismatching content + if s.progress.Subchains[1].Tail >= s.progress.Subchains[0].Tail { + // Fully overwritten, get rid of the subchain as a whole + log.Debug("Previous subchain fully overwritten", "head", head, "tail", tail, "next", next) + s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) + continue + } else { + // Partially overwritten, trim the head to the overwritten size + log.Debug("Previous subchain partially overwritten", "head", head, "tail", tail, "next", next) + s.progress.Subchains[1].Head = s.progress.Subchains[0].Tail - 1 + } + // If the old subchain is an extension of the new one, merge the two + // and let the skeleton syncer restart (to clean internal state) + if rawdb.ReadSkeletonHeader(s.db, s.progress.Subchains[1].Head).Hash() == s.progress.Subchains[0].Next { + log.Debug("Previous subchain merged", "head", head, "tail", tail, "next", next) + s.progress.Subchains[0].Tail = s.progress.Subchains[1].Tail + s.progress.Subchains[0].Next = s.progress.Subchains[1].Next + + s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) + merged = true + } + } + // If subchains were merged, all further available headers in the scratch + // space are invalid since we skipped ahead. Stop processing the scratch + // space to avoid dropping peers thinking they delivered invalid data. + if merged { + break + } + } + s.saveSyncStatus(batch) + if err := batch.Write(); err != nil { + log.Crit("Failed to write skeleton headers and progress", "err", err) + } + // Print a progress report making the UX a bit nicer + left := s.progress.Subchains[0].Tail - 1 + if linked { + left = 0 + } + if time.Since(s.logged) > 8*time.Second || left == 0 { + s.logged = time.Now() + + if s.pulled == 0 { + log.Info("Beacon sync starting", "left", left) + } else { + eta := float64(time.Since(s.started)) / float64(s.pulled) * float64(left) + log.Info("Syncing beacon headers", "downloaded", s.pulled, "left", left, "eta", common.PrettyDuration(eta)) + } + } + return linked, merged +} + +// Bounds retrieves the current head and tail tracked by the skeleton syncer. +// This method is used by the backfiller, whose life cycle is controlled by the +// skeleton syncer. +// +// Note, the method will not use the internal state of the skeleton, but will +// rather blindly pull stuff from the database. This is fine, because the back- +// filler will only run when the skeleton chain is fully downloaded and stable. +// There might be new heads appended, but those are atomic from the perspective +// of this method. Any head reorg will first tear down the backfiller and only +// then make the modification. +func (s *skeleton) Bounds() (head *types.Header, tail *types.Header, err error) { + // Read the current sync progress from disk and figure out the current head. + // Although there's a lot of error handling here, these are mostly as sanity + // checks to avoid crashing if a programming error happens. These should not + // happen in live code. + status := rawdb.ReadSkeletonSyncStatus(s.db) + if len(status) == 0 { + return nil, nil, errors.New("beacon sync not yet started") + } + progress := new(skeletonProgress) + if err := json.Unmarshal(status, progress); err != nil { + return nil, nil, err + } + head = rawdb.ReadSkeletonHeader(s.db, progress.Subchains[0].Head) + tail = rawdb.ReadSkeletonHeader(s.db, progress.Subchains[0].Tail) + + return head, tail, nil +} + +// Header retrieves a specific header tracked by the skeleton syncer. This method +// is meant to be used by the backfiller, whose life cycle is controlled by the +// skeleton syncer. +// +// Note, outside the permitted runtimes, this method might return nil results and +// subsequent calls might return headers from different chains. +func (s *skeleton) Header(number uint64) *types.Header { + return rawdb.ReadSkeletonHeader(s.db, number) +} diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go new file mode 100644 index 000000000000..cbe0d51d3716 --- /dev/null +++ b/eth/downloader/skeleton_test.go @@ -0,0 +1,896 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "encoding/json" + "errors" + "fmt" + "math/big" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/log" +) + +// hookedBackfiller is a tester backfiller with all interface methods mocked and +// hooked so tests can implement only the things they need. +type hookedBackfiller struct { + // suspendHook is an optional hook to be called when the filler is requested + // to be suspended. + suspendHook func() + + // resumeHook is an optional hook to be called when the filler is requested + // to be resumed. + resumeHook func() +} + +// newHookedBackfiller creates a hooked backfiller with all callbacks disabled, +// essentially acting as a noop. +func newHookedBackfiller() backfiller { + return new(hookedBackfiller) +} + +// suspend requests the backfiller to abort any running full or snap sync +// based on the skeleton chain as it might be invalid. The backfiller should +// gracefully handle multiple consecutive suspends without a resume, even +// on initial sartup. +func (hf *hookedBackfiller) suspend() { + if hf.suspendHook != nil { + hf.suspendHook() + } +} + +// resume requests the backfiller to start running fill or snap sync based on +// the skeleton chain as it has successfully been linked. Appending new heads +// to the end of the chain will not result in suspend/resume cycles. +func (hf *hookedBackfiller) resume() { + if hf.resumeHook != nil { + hf.resumeHook() + } +} + +// skeletonTestPeer is a mock peer that can only serve header requests from a +// pre-perated header chain (which may be arbitrarily wrong for testing). +// +// Requesting anything else from these peers will hard panic. Note, do *not* +// implement any other methods. We actually want to make sure that the skeleton +// syncer only depends on - and will only ever do so - on header requests. +type skeletonTestPeer struct { + id string // Unique identifier of the mock peer + headers []*types.Header // Headers to serve when requested + + serve func(origin uint64) []*types.Header // Hook to allow custom responses + + served uint64 // Number of headers served by this peer + dropped uint64 // Flag whether the peer was dropped (stop responding) +} + +// newSkeletonTestPeer creates a new mock peer to test the skeleton sync with. +func newSkeletonTestPeer(id string, headers []*types.Header) *skeletonTestPeer { + return &skeletonTestPeer{ + id: id, + headers: headers, + } +} + +// newSkeletonTestPeer creates a new mock peer to test the skeleton sync with, +// and sets an optional serve hook that can return headers for delivery instead +// of the predefined chain. Useful for emulating malicious behavior that would +// otherwise require dedicated peer types. +func newSkeletonTestPeerWithHook(id string, headers []*types.Header, serve func(origin uint64) []*types.Header) *skeletonTestPeer { + return &skeletonTestPeer{ + id: id, + headers: headers, + serve: serve, + } +} + +// RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered +// origin; associated with a particular peer in the download tester. The returned +// function can be used to retrieve batches of headers from the particular peer. +func (p *skeletonTestPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool, sink chan *eth.Response) (*eth.Request, error) { + // Since skeleton test peer are in-memory mocks, dropping the does not make + // them inaccepssible. As such, check a local `dropped` field to see if the + // peer has been dropped and should not respond any more. + if atomic.LoadUint64(&p.dropped) != 0 { + return nil, errors.New("peer already dropped") + } + // Skeleton sync retrieves batches of headers going backward without gaps. + // This ensures we can follow a clean parent progression without any reorg + // hiccups. There is no need for any other type of header retrieval, so do + // panic if there's such a request. + if !reverse || skip != 0 { + // Note, if other clients want to do these kinds of requests, it's their + // problem, it will still work. We just don't want *us* making complicated + // requests without a very strong reason to. + panic(fmt.Sprintf("invalid header retrieval: reverse %v, want true; skip %d, want 0", reverse, skip)) + } + // If the skeleton syncer requests the genesis block, panic. Whilst it could + // be considered a valid request, our code specifically should not request it + // ever since we want to link up headers to an existing local chain, which at + // worse will be the genesis. + if int64(origin)-int64(amount) < 0 { + panic(fmt.Sprintf("headers requested before (or at) genesis: origin %d, amount %d", origin, amount)) + } + // To make concurrency easier, the skeleton syncer always requests fixed size + // batches of headers. Panic if the peer is requested an amount other than the + // configured batch size (apart from the request leading to the genesis). + if amount > requestHeaders || (amount < requestHeaders && origin > uint64(amount)) { + panic(fmt.Sprintf("non-chunk size header batch requested: requested %d, want %d, origin %d", amount, requestHeaders, origin)) + } + // Simple reverse header retrieval. Fill from the peer's chain and return. + // If the tester has a serve hook set, try to use that before falling back + // to the default behavior. + var headers []*types.Header + if p.serve != nil { + headers = p.serve(origin) + } + if headers == nil { + headers = make([]*types.Header, 0, amount) + if len(p.headers) > int(origin) { // Don't serve headers if we're missing the origin + for i := 0; i < amount; i++ { + // Consider nil headers as a form of attack and withhold them. Nil + // cannot be decoded from RLP, so it's not possible to produce an + // attack by sending/receiving those over eth. + header := p.headers[int(origin)-i] + if header == nil { + continue + } + headers = append(headers, header) + } + } + } + atomic.AddUint64(&p.served, uint64(len(headers))) + + hashes := make([]common.Hash, len(headers)) + for i, header := range headers { + hashes[i] = header.Hash() + } + // Deliver the headers to the downloader + req := ð.Request{ + Peer: p.id, + } + res := ð.Response{ + Req: req, + Res: (*eth.BlockHeadersPacket)(&headers), + Meta: hashes, + Time: 1, + Done: make(chan error), + } + go func() { + sink <- res + if err := <-res.Done; err != nil { + log.Warn("Skeleton test peer response rejected", "err", err) + atomic.AddUint64(&p.dropped, 1) + } + }() + return req, nil +} + +func (p *skeletonTestPeer) Head() (common.Hash, *big.Int) { + panic("skeleton sync must not request the remote head") +} + +func (p *skeletonTestPeer) RequestHeadersByHash(common.Hash, int, int, bool, chan *eth.Response) (*eth.Request, error) { + panic("skeleton sync must not request headers by hash") +} + +func (p *skeletonTestPeer) RequestBodies([]common.Hash, chan *eth.Response) (*eth.Request, error) { + panic("skeleton sync must not request block bodies") +} + +func (p *skeletonTestPeer) RequestReceipts([]common.Hash, chan *eth.Response) (*eth.Request, error) { + panic("skeleton sync must not request receipts") +} + +// Tests various sync initialzations based on previous leftovers in the database +// and announced heads. +func TestSkeletonSyncInit(t *testing.T) { + // Create a few key headers + var ( + genesis = &types.Header{Number: big.NewInt(0)} + block49 = &types.Header{Number: big.NewInt(49)} + block49B = &types.Header{Number: big.NewInt(49), Extra: []byte("B")} + block50 = &types.Header{Number: big.NewInt(50), ParentHash: block49.Hash()} + ) + tests := []struct { + headers []*types.Header // Database content (beside the genesis) + oldstate []*subchain // Old sync state with various interrupted subchains + head *types.Header // New head header to announce to reorg to + newstate []*subchain // Expected sync state after the reorg + }{ + // Completely empty database with only the genesis set. The sync is expected + // to create a single subchain with the requested head. + { + head: block50, + newstate: []*subchain{{Head: 50, Tail: 50}}, + }, + // Empty database with only the genesis set with a leftover empty sync + // progess. This is a synthetic case, just for the sake of covering things. + { + oldstate: []*subchain{}, + head: block50, + newstate: []*subchain{{Head: 50, Tail: 50}}, + }, + // A single leftover subchain is present, older than the new head. The + // old subchain should be left as is and a new one appended to the sync + // status. + { + oldstate: []*subchain{{Head: 10, Tail: 5}}, + head: block50, + newstate: []*subchain{ + {Head: 50, Tail: 50}, + {Head: 10, Tail: 5}, + }, + }, + // Multiple leftover subchains are present, older than the new head. The + // old subchains should be left as is and a new one appended to the sync + // status. + { + oldstate: []*subchain{ + {Head: 20, Tail: 15}, + {Head: 10, Tail: 5}, + }, + head: block50, + newstate: []*subchain{ + {Head: 50, Tail: 50}, + {Head: 20, Tail: 15}, + {Head: 10, Tail: 5}, + }, + }, + // A single leftover subchain is present, newer than the new head. The + // newer subchain should be deleted and a fresh one created for the head. + { + oldstate: []*subchain{{Head: 65, Tail: 60}}, + head: block50, + newstate: []*subchain{{Head: 50, Tail: 50}}, + }, + // Multiple leftover subchain is present, newer than the new head. The + // newer subchains should be deleted and a fresh one created for the head. + { + oldstate: []*subchain{ + {Head: 75, Tail: 70}, + {Head: 65, Tail: 60}, + }, + head: block50, + newstate: []*subchain{{Head: 50, Tail: 50}}, + }, + + // Two leftover subchains are present, one fully older and one fully + // newer than the announced head. The head should delete the newer one, + // keeping the older one. + { + oldstate: []*subchain{ + {Head: 65, Tail: 60}, + {Head: 10, Tail: 5}, + }, + head: block50, + newstate: []*subchain{ + {Head: 50, Tail: 50}, + {Head: 10, Tail: 5}, + }, + }, + // Multiple leftover subchains are present, some fully older and some + // fully newer than the announced head. The head should delete the newer + // ones, keeping the older ones. + { + oldstate: []*subchain{ + {Head: 75, Tail: 70}, + {Head: 65, Tail: 60}, + {Head: 20, Tail: 15}, + {Head: 10, Tail: 5}, + }, + head: block50, + newstate: []*subchain{ + {Head: 50, Tail: 50}, + {Head: 20, Tail: 15}, + {Head: 10, Tail: 5}, + }, + }, + // A single leftover subchain is present and the new head is extending + // it with one more header. We expect the subchain head to be pushed + // forward. + { + headers: []*types.Header{block49}, + oldstate: []*subchain{{Head: 49, Tail: 5}}, + head: block50, + newstate: []*subchain{{Head: 50, Tail: 5}}, + }, + // A single leftover subchain is present and although the new head does + // extend it number wise, the hash chain does not link up. We expect a + // new subchain to be created for the dangling head. + { + headers: []*types.Header{block49B}, + oldstate: []*subchain{{Head: 49, Tail: 5}}, + head: block50, + newstate: []*subchain{ + {Head: 50, Tail: 50}, + {Head: 49, Tail: 5}, + }, + }, + // A single leftover subchain is present. A new head is announced that + // links into the middle of it, correctly anchoring into an existing + // header. We expect the old subchain to be truncated and extended with + // the new head. + { + headers: []*types.Header{block49}, + oldstate: []*subchain{{Head: 100, Tail: 5}}, + head: block50, + newstate: []*subchain{{Head: 50, Tail: 5}}, + }, + // A single leftover subchain is present. A new head is announced that + // links into the middle of it, but does not anchor into an existing + // header. We expect the old subchain to be truncated and a new chain + // be created for the dangling head. + { + headers: []*types.Header{block49B}, + oldstate: []*subchain{{Head: 100, Tail: 5}}, + head: block50, + newstate: []*subchain{ + {Head: 50, Tail: 50}, + {Head: 49, Tail: 5}, + }, + }, + } + for i, tt := range tests { + // Create a fresh database and initialize it with the starting state + db := rawdb.NewMemoryDatabase() + + rawdb.WriteHeader(db, genesis) + for _, header := range tt.headers { + rawdb.WriteSkeletonHeader(db, header) + } + if tt.oldstate != nil { + blob, _ := json.Marshal(&skeletonProgress{Subchains: tt.oldstate}) + rawdb.WriteSkeletonSyncStatus(db, blob) + } + // Create a skeleton sync and run a cycle + wait := make(chan struct{}) + + skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller()) + skeleton.syncStarting = func() { close(wait) } + skeleton.Sync(tt.head, true) + + <-wait + skeleton.Terminate() + + // Ensure the correct resulting sync status + var progress skeletonProgress + json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress) + + if len(progress.Subchains) != len(tt.newstate) { + t.Errorf("test %d: subchain count mismatch: have %d, want %d", i, len(progress.Subchains), len(tt.newstate)) + continue + } + for j := 0; j < len(progress.Subchains); j++ { + if progress.Subchains[j].Head != tt.newstate[j].Head { + t.Errorf("test %d: subchain %d head mismatch: have %d, want %d", i, j, progress.Subchains[j].Head, tt.newstate[j].Head) + } + if progress.Subchains[j].Tail != tt.newstate[j].Tail { + t.Errorf("test %d: subchain %d tail mismatch: have %d, want %d", i, j, progress.Subchains[j].Tail, tt.newstate[j].Tail) + } + } + } +} + +// Tests that a running skeleton sync can be extended with properly linked up +// headers but not with side chains. +func TestSkeletonSyncExtend(t *testing.T) { + // Create a few key headers + var ( + genesis = &types.Header{Number: big.NewInt(0)} + block49 = &types.Header{Number: big.NewInt(49)} + block49B = &types.Header{Number: big.NewInt(49), Extra: []byte("B")} + block50 = &types.Header{Number: big.NewInt(50), ParentHash: block49.Hash()} + block51 = &types.Header{Number: big.NewInt(51), ParentHash: block50.Hash()} + ) + tests := []struct { + head *types.Header // New head header to announce to reorg to + extend *types.Header // New head header to announce to extend with + newstate []*subchain // Expected sync state after the reorg + err error // Whether extension succeeds or not + }{ + // Initialize a sync and try to extend it with a subsequent block. + { + head: block49, + extend: block50, + newstate: []*subchain{ + {Head: 50, Tail: 49}, + }, + }, + // Initialize a sync and try to extend it with the existing head block. + { + head: block49, + extend: block49, + newstate: []*subchain{ + {Head: 49, Tail: 49}, + }, + err: errReorgDenied, + }, + // Initialize a sync and try to extend it with a sibling block. + { + head: block49, + extend: block49B, + newstate: []*subchain{ + {Head: 49, Tail: 49}, + }, + err: errReorgDenied, + }, + // Initialize a sync and try to extend it with a number-wise sequential + // header, but a hash wise non-linking one. + { + head: block49B, + extend: block50, + newstate: []*subchain{ + {Head: 49, Tail: 49}, + }, + err: errReorgDenied, + }, + // Initialize a sync and try to extend it with a non-linking future block. + { + head: block49, + extend: block51, + newstate: []*subchain{ + {Head: 49, Tail: 49}, + }, + err: errReorgDenied, + }, + // Initialize a sync and try to extend it with a past canonical block. + { + head: block50, + extend: block49, + newstate: []*subchain{ + {Head: 50, Tail: 50}, + }, + err: errReorgDenied, + }, + // Initialize a sync and try to extend it with a past sidechain block. + { + head: block50, + extend: block49B, + newstate: []*subchain{ + {Head: 50, Tail: 50}, + }, + err: errReorgDenied, + }, + } + for i, tt := range tests { + // Create a fresh database and initialize it with the starting state + db := rawdb.NewMemoryDatabase() + rawdb.WriteHeader(db, genesis) + + // Create a skeleton sync and run a cycle + wait := make(chan struct{}) + + skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller()) + skeleton.syncStarting = func() { close(wait) } + skeleton.Sync(tt.head, true) + + <-wait + if err := skeleton.Sync(tt.extend, false); err != tt.err { + t.Errorf("extension failure mismatch: have %v, want %v", err, tt.err) + } + skeleton.Terminate() + + // Ensure the correct resulting sync status + var progress skeletonProgress + json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress) + + if len(progress.Subchains) != len(tt.newstate) { + t.Errorf("test %d: subchain count mismatch: have %d, want %d", i, len(progress.Subchains), len(tt.newstate)) + continue + } + for j := 0; j < len(progress.Subchains); j++ { + if progress.Subchains[j].Head != tt.newstate[j].Head { + t.Errorf("test %d: subchain %d head mismatch: have %d, want %d", i, j, progress.Subchains[j].Head, tt.newstate[j].Head) + } + if progress.Subchains[j].Tail != tt.newstate[j].Tail { + t.Errorf("test %d: subchain %d tail mismatch: have %d, want %d", i, j, progress.Subchains[j].Tail, tt.newstate[j].Tail) + } + } + } +} + +// Tests that the skeleton sync correctly retrieves headers from one or more +// peers without duplicates or other strange side effects. +func TestSkeletonSyncRetrievals(t *testing.T) { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + + // Since skeleton headers don't need to be meaningful, beyond a parent hash + // progression, create a long fake chain to test with. + chain := []*types.Header{{Number: big.NewInt(0)}} + for i := 1; i < 10000; i++ { + chain = append(chain, &types.Header{ + ParentHash: chain[i-1].Hash(), + Number: big.NewInt(int64(i)), + }) + } + tests := []struct { + headers []*types.Header // Database content (beside the genesis) + oldstate []*subchain // Old sync state with various interrupted subchains + + head *types.Header // New head header to announce to reorg to + peers []*skeletonTestPeer // Initial peer set to start the sync with + midstate []*subchain // Expected sync state after initial cycle + midserve uint64 // Expected number of header retrievals after initial cycle + middrop uint64 // Expectd number of peers dropped after initial cycle + + newHead *types.Header // New header to annount on top of the old one + newPeer *skeletonTestPeer // New peer to join the skeleton syncer + endstate []*subchain // Expected sync state after the post-init event + endserve uint64 // Expected number of header retrievals after the post-init event + enddrop uint64 // Expectd number of peers dropped after the post-init event + }{ + // Completely empty database with only the genesis set. The sync is expected + // to create a single subchain with the requested head. No peers however, so + // the sync should be stuck without any progression. + // + // When a new peer is added, it should detect the join and fill the headers + // to the genesis block. + { + head: chain[len(chain)-1], + midstate: []*subchain{{Head: uint64(len(chain) - 1), Tail: uint64(len(chain) - 1)}}, + + newPeer: newSkeletonTestPeer("test-peer", chain), + endstate: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}}, + endserve: uint64(len(chain) - 2), // len - head - genesis + }, + // Completely empty database with only the genesis set. The sync is expected + // to create a single subchain with the requested head. With one valid peer, + // the sync is expected to complete already in the initial round. + // + // Adding a second peer should not have any effect. + { + head: chain[len(chain)-1], + peers: []*skeletonTestPeer{newSkeletonTestPeer("test-peer-1", chain)}, + midstate: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}}, + midserve: uint64(len(chain) - 2), // len - head - genesis + + newPeer: newSkeletonTestPeer("test-peer-2", chain), + endstate: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}}, + endserve: uint64(len(chain) - 2), // len - head - genesis + }, + // Completely empty database with only the genesis set. The sync is expected + // to create a single subchain with the requested head. With many valid peers, + // the sync is expected to complete already in the initial round. + // + // Adding a new peer should not have any effect. + { + head: chain[len(chain)-1], + peers: []*skeletonTestPeer{ + newSkeletonTestPeer("test-peer-1", chain), + newSkeletonTestPeer("test-peer-2", chain), + newSkeletonTestPeer("test-peer-3", chain), + }, + midstate: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}}, + midserve: uint64(len(chain) - 2), // len - head - genesis + + newPeer: newSkeletonTestPeer("test-peer-4", chain), + endstate: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}}, + endserve: uint64(len(chain) - 2), // len - head - genesis + }, + // This test checks if a peer tries to withhold a header - *on* the sync + // boundary - instead of sending the requested amount. The malicious short + // package should not be accepted. + // + // Joining with a new peer should however unblock the sync. + { + head: chain[requestHeaders+100], + peers: []*skeletonTestPeer{ + newSkeletonTestPeer("header-skipper", append(append(append([]*types.Header{}, chain[:99]...), nil), chain[100:]...)), + }, + midstate: []*subchain{{Head: requestHeaders + 100, Tail: 100}}, + midserve: requestHeaders + 101 - 3, // len - head - genesis - missing + middrop: 1, // penalize shortened header deliveries + + newPeer: newSkeletonTestPeer("good-peer", chain), + endstate: []*subchain{{Head: requestHeaders + 100, Tail: 1}}, + endserve: (requestHeaders + 101 - 3) + (100 - 1), // midserve + lenrest - genesis + enddrop: 1, // no new drops + }, + // This test checks if a peer tries to withhold a header - *off* the sync + // boundary - instead of sending the requested amount. The malicious short + // package should not be accepted. + // + // Joining with a new peer should however unblock the sync. + { + head: chain[requestHeaders+100], + peers: []*skeletonTestPeer{ + newSkeletonTestPeer("header-skipper", append(append(append([]*types.Header{}, chain[:50]...), nil), chain[51:]...)), + }, + midstate: []*subchain{{Head: requestHeaders + 100, Tail: 100}}, + midserve: requestHeaders + 101 - 3, // len - head - genesis - missing + middrop: 1, // penalize shortened header deliveries + + newPeer: newSkeletonTestPeer("good-peer", chain), + endstate: []*subchain{{Head: requestHeaders + 100, Tail: 1}}, + endserve: (requestHeaders + 101 - 3) + (100 - 1), // midserve + lenrest - genesis + enddrop: 1, // no new drops + }, + // This test checks if a peer tries to duplicate a header - *on* the sync + // boundary - instead of sending the correct sequence. The malicious duped + // package should not be accepted. + // + // Joining with a new peer should however unblock the sync. + { + head: chain[requestHeaders+100], // We want to force the 100th header to be a request boundary + peers: []*skeletonTestPeer{ + newSkeletonTestPeer("header-duper", append(append(append([]*types.Header{}, chain[:99]...), chain[98]), chain[100:]...)), + }, + midstate: []*subchain{{Head: requestHeaders + 100, Tail: 100}}, + midserve: requestHeaders + 101 - 2, // len - head - genesis + middrop: 1, // penalize invalid header sequences + + newPeer: newSkeletonTestPeer("good-peer", chain), + endstate: []*subchain{{Head: requestHeaders + 100, Tail: 1}}, + endserve: (requestHeaders + 101 - 2) + (100 - 1), // midserve + lenrest - genesis + enddrop: 1, // no new drops + }, + // This test checks if a peer tries to duplicate a header - *off* the sync + // boundary - instead of sending the correct sequence. The malicious duped + // package should not be accepted. + // + // Joining with a new peer should however unblock the sync. + { + head: chain[requestHeaders+100], // We want to force the 100th header to be a request boundary + peers: []*skeletonTestPeer{ + newSkeletonTestPeer("header-duper", append(append(append([]*types.Header{}, chain[:50]...), chain[49]), chain[51:]...)), + }, + midstate: []*subchain{{Head: requestHeaders + 100, Tail: 100}}, + midserve: requestHeaders + 101 - 2, // len - head - genesis + middrop: 1, // penalize invalid header sequences + + newPeer: newSkeletonTestPeer("good-peer", chain), + endstate: []*subchain{{Head: requestHeaders + 100, Tail: 1}}, + endserve: (requestHeaders + 101 - 2) + (100 - 1), // midserve + lenrest - genesis + enddrop: 1, // no new drops + }, + // This test checks if a peer tries to inject a different header - *on* + // the sync boundary - instead of sending the correct sequence. The bad + // package should not be accepted. + // + // Joining with a new peer should however unblock the sync. + { + head: chain[requestHeaders+100], // We want to force the 100th header to be a request boundary + peers: []*skeletonTestPeer{ + newSkeletonTestPeer("header-changer", + append( + append( + append([]*types.Header{}, chain[:99]...), + &types.Header{ + ParentHash: chain[98].Hash(), + Number: big.NewInt(int64(99)), + GasLimit: 1, + }, + ), chain[100:]..., + ), + ), + }, + midstate: []*subchain{{Head: requestHeaders + 100, Tail: 100}}, + midserve: requestHeaders + 101 - 2, // len - head - genesis + middrop: 1, // different set of headers, drop // TODO(karalabe): maybe just diff sync? + + newPeer: newSkeletonTestPeer("good-peer", chain), + endstate: []*subchain{{Head: requestHeaders + 100, Tail: 1}}, + endserve: (requestHeaders + 101 - 2) + (100 - 1), // midserve + lenrest - genesis + enddrop: 1, // no new drops + }, + // This test checks if a peer tries to inject a different header - *off* + // the sync boundary - instead of sending the correct sequence. The bad + // package should not be accepted. + // + // Joining with a new peer should however unblock the sync. + { + head: chain[requestHeaders+100], // We want to force the 100th header to be a request boundary + peers: []*skeletonTestPeer{ + newSkeletonTestPeer("header-changer", + append( + append( + append([]*types.Header{}, chain[:50]...), + &types.Header{ + ParentHash: chain[49].Hash(), + Number: big.NewInt(int64(50)), + GasLimit: 1, + }, + ), chain[51:]..., + ), + ), + }, + midstate: []*subchain{{Head: requestHeaders + 100, Tail: 100}}, + midserve: requestHeaders + 101 - 2, // len - head - genesis + middrop: 1, // different set of headers, drop + + newPeer: newSkeletonTestPeer("good-peer", chain), + endstate: []*subchain{{Head: requestHeaders + 100, Tail: 1}}, + endserve: (requestHeaders + 101 - 2) + (100 - 1), // midserve + lenrest - genesis + enddrop: 1, // no new drops + }, + // This test reproduces a bug caught during review (kudos to @holiman) + // where a subchain is merged with a previously interrupted one, causing + // pending data in the scratch space to become "invalid" (since we jump + // ahead during subchain merge). In that case it is expected to ignore + // the queued up data instead of trying to process on top of a shifted + // task set. + // + // The test is a bit convoluted since it needs to trigger a concurrency + // issue. First we sync up an initial chain of 2x512 items. Then announce + // 2x512+2 as head and delay delivering the head batch to fill the scratch + // space first. The delivery head should merge with the previous download + // and the scratch space must not be consumed further. + { + head: chain[2*requestHeaders], + peers: []*skeletonTestPeer{ + newSkeletonTestPeerWithHook("peer-1", chain, func(origin uint64) []*types.Header { + if origin == chain[2*requestHeaders+1].Number.Uint64() { + time.Sleep(100 * time.Millisecond) + } + return nil // Fallback to default behavior, just delayed + }), + newSkeletonTestPeerWithHook("peer-2", chain, func(origin uint64) []*types.Header { + if origin == chain[2*requestHeaders+1].Number.Uint64() { + time.Sleep(100 * time.Millisecond) + } + return nil // Fallback to default behavior, just delayed + }), + }, + midstate: []*subchain{{Head: 2 * requestHeaders, Tail: 1}}, + midserve: 2*requestHeaders - 1, // len - head - genesis + + newHead: chain[2*requestHeaders+2], + endstate: []*subchain{{Head: 2*requestHeaders + 2, Tail: 1}}, + endserve: 4 * requestHeaders, + }, + } + for i, tt := range tests { + // Create a fresh database and initialize it with the starting state + db := rawdb.NewMemoryDatabase() + rawdb.WriteHeader(db, chain[0]) + + // Create a peer set to feed headers through + peerset := newPeerSet() + for _, peer := range tt.peers { + peerset.Register(newPeerConnection(peer.id, eth.ETH66, peer, log.New("id", peer.id))) + } + // Create a peer dropper to track malicious peers + dropped := make(map[string]int) + drop := func(peer string) { + if p := peerset.Peer(peer); p != nil { + atomic.AddUint64(&p.peer.(*skeletonTestPeer).dropped, 1) + } + peerset.Unregister(peer) + dropped[peer]++ + } + // Create a skeleton sync and run a cycle + skeleton := newSkeleton(db, peerset, drop, newHookedBackfiller()) + skeleton.Sync(tt.head, true) + + var progress skeletonProgress + // Wait a bit (bleah) for the initial sync loop to go to idle. This might + // be either a finish or a never-start hence why there's no event to hook. + check := func() error { + if len(progress.Subchains) != len(tt.midstate) { + return fmt.Errorf("test %d, mid state: subchain count mismatch: have %d, want %d", i, len(progress.Subchains), len(tt.midstate)) + + } + for j := 0; j < len(progress.Subchains); j++ { + if progress.Subchains[j].Head != tt.midstate[j].Head { + return fmt.Errorf("test %d, mid state: subchain %d head mismatch: have %d, want %d", i, j, progress.Subchains[j].Head, tt.midstate[j].Head) + } + if progress.Subchains[j].Tail != tt.midstate[j].Tail { + return fmt.Errorf("test %d, mid state: subchain %d tail mismatch: have %d, want %d", i, j, progress.Subchains[j].Tail, tt.midstate[j].Tail) + } + } + return nil + } + + waitStart := time.Now() + for waitTime := 20 * time.Millisecond; time.Since(waitStart) < time.Second; waitTime = waitTime * 2 { + time.Sleep(waitTime) + // Check the post-init end state if it matches the required results + json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress) + if err := check(); err == nil { + break + } + } + if err := check(); err != nil { + t.Error(err) + continue + } + var served uint64 + for _, peer := range tt.peers { + served += atomic.LoadUint64(&peer.served) + } + if served != tt.midserve { + t.Errorf("test %d, mid state: served headers mismatch: have %d, want %d", i, served, tt.midserve) + } + var drops uint64 + for _, peer := range tt.peers { + drops += atomic.LoadUint64(&peer.dropped) + } + if drops != tt.middrop { + t.Errorf("test %d, mid state: dropped peers mismatch: have %d, want %d", i, drops, tt.middrop) + } + // Apply the post-init events if there's any + if tt.newHead != nil { + skeleton.Sync(tt.newHead, true) + } + if tt.newPeer != nil { + if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH66, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { + t.Errorf("test %d: failed to register new peer: %v", i, err) + } + } + // Wait a bit (bleah) for the second sync loop to go to idle. This might + // be either a finish or a never-start hence why there's no event to hook. + check = func() error { + if len(progress.Subchains) != len(tt.endstate) { + return fmt.Errorf("test %d, end state: subchain count mismatch: have %d, want %d", i, len(progress.Subchains), len(tt.endstate)) + } + for j := 0; j < len(progress.Subchains); j++ { + if progress.Subchains[j].Head != tt.endstate[j].Head { + return fmt.Errorf("test %d, end state: subchain %d head mismatch: have %d, want %d", i, j, progress.Subchains[j].Head, tt.endstate[j].Head) + } + if progress.Subchains[j].Tail != tt.endstate[j].Tail { + return fmt.Errorf("test %d, end state: subchain %d tail mismatch: have %d, want %d", i, j, progress.Subchains[j].Tail, tt.endstate[j].Tail) + } + } + return nil + } + waitStart = time.Now() + for waitTime := 20 * time.Millisecond; time.Since(waitStart) < time.Second; waitTime = waitTime * 2 { + time.Sleep(waitTime) + // Check the post-init end state if it matches the required results + json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress) + if err := check(); err == nil { + break + } + } + if err := check(); err != nil { + t.Error(err) + continue + } + // Check that the peers served no more headers than we actually needed + served = 0 + for _, peer := range tt.peers { + served += atomic.LoadUint64(&peer.served) + } + if tt.newPeer != nil { + served += atomic.LoadUint64(&tt.newPeer.served) + } + if served != tt.endserve { + t.Errorf("test %d, end state: served headers mismatch: have %d, want %d", i, served, tt.endserve) + } + drops = 0 + for _, peer := range tt.peers { + drops += atomic.LoadUint64(&peer.dropped) + } + if tt.newPeer != nil { + drops += atomic.LoadUint64(&tt.newPeer.dropped) + } + if drops != tt.middrop { + t.Errorf("test %d, end state: dropped peers mismatch: have %d, want %d", i, drops, tt.middrop) + } + // Clean up any leftover skeleton sync resources + skeleton.Terminate() + } +} diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 1327c6814945..789b087804ae 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -138,8 +138,10 @@ type Config struct { TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. - // Whitelist of required block number -> hash values to accept - Whitelist map[uint64]common.Hash `toml:"-"` + // PeerRequiredBlocks is a set of block number -> hash mappings which must be in the + // canonical chain of all remote peers. Setting the option makes geth verify the + // presence of these blocks for every new peer connection. + PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` // Light client options LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 70a9649bff83..874e30dffdac 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -26,7 +26,7 @@ func (c Config) MarshalTOML() (interface{}, error) { NoPruning bool NoPrefetch bool TxLookupLimit uint64 `toml:",omitempty"` - Whitelist map[uint64]common.Hash `toml:"-"` + PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` LightIngress int `toml:",omitempty"` LightEgress int `toml:",omitempty"` @@ -71,7 +71,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPruning = c.NoPruning enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit - enc.Whitelist = c.Whitelist + enc.PeerRequiredBlocks = c.PeerRequiredBlocks enc.LightServ = c.LightServ enc.LightIngress = c.LightIngress enc.LightEgress = c.LightEgress @@ -120,7 +120,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { NoPruning *bool NoPrefetch *bool TxLookupLimit *uint64 `toml:",omitempty"` - Whitelist map[uint64]common.Hash `toml:"-"` + PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` LightIngress *int `toml:",omitempty"` LightEgress *int `toml:",omitempty"` @@ -184,8 +184,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } - if dec.Whitelist != nil { - c.Whitelist = dec.Whitelist + if dec.PeerRequiredBlocks != nil { + c.PeerRequiredBlocks = dec.PeerRequiredBlocks } if dec.LightServ != nil { c.LightServ = *dec.LightServ diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 970dfd4467a5..4113089afb1e 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -117,7 +117,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { reward, _ := tx.EffectiveGasTip(bf.block.BaseFee()) sorter[i] = txGasAndReward{gasUsed: bf.receipts[i].GasUsed, reward: reward} } - sort.Sort(sorter) + sort.Stable(sorter) var txIndex int sumGasUsed := sorter[0].gasUsed diff --git a/eth/handler.go b/eth/handler.go index 921a62dba501..40edfa2d1758 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -86,7 +86,8 @@ type handlerConfig struct { BloomCache uint64 // Megabytes to alloc for snap sync bloom EventMux *event.TypeMux // Legacy event mux, deprecate for `feed` Checkpoint *params.TrustedCheckpoint // Hard coded checkpoint for sync challenges - Whitelist map[uint64]common.Hash // Hard coded whitelist for sync challenged + + PeerRequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges } type handler struct { @@ -115,7 +116,7 @@ type handler struct { txsSub event.Subscription minedBlockSub *event.TypeMuxSubscription - whitelist map[uint64]common.Hash + peerRequiredBlocks map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop quitSync chan struct{} @@ -132,16 +133,16 @@ func newHandler(config *handlerConfig) (*handler, error) { config.EventMux = new(event.TypeMux) // Nicety initialization for tests } h := &handler{ - networkID: config.Network, - forkFilter: forkid.NewFilter(config.Chain), - eventMux: config.EventMux, - database: config.Database, - txpool: config.TxPool, - chain: config.Chain, - peers: newPeerSet(), - merger: config.Merger, - whitelist: config.Whitelist, - quitSync: make(chan struct{}), + networkID: config.Network, + forkFilter: forkid.NewFilter(config.Chain), + eventMux: config.EventMux, + database: config.Database, + txpool: config.TxPool, + chain: config.Chain, + peers: newPeerSet(), + merger: config.Merger, + peerRequiredBlocks: config.PeerRequiredBlocks, + quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { // The database seems empty as the current block is the genesis. Yet the snap @@ -171,10 +172,30 @@ func newHandler(config *handlerConfig) (*handler, error) { h.checkpointNumber = (config.Checkpoint.SectionIndex+1)*params.CHTFrequency - 1 h.checkpointHash = config.Checkpoint.SectionHead } + // If sync succeeds, pass a callback to potentially disable snap sync mode + // and enable transaction propagation. + success := func() { + // If we were running snap sync and it finished, disable doing another + // round on next sync cycle + if atomic.LoadUint32(&h.snapSync) == 1 { + log.Info("Snap sync complete, auto disabling") + atomic.StoreUint32(&h.snapSync, 0) + } + // If we've successfully finished a sync cycle and passed any required + // checkpoint, enable accepting transactions from the network + head := h.chain.CurrentBlock() + if head.NumberU64() >= h.checkpointNumber { + // Checkpoint passed, sanity check the timestamp to have a fallback mechanism + // for non-checkpointed (number = 0) private networks. + if head.Time() >= uint64(time.Now().AddDate(0, -1, 0).Unix()) { + atomic.StoreUint32(&h.acceptTxs, 1) + } + } + } // Construct the downloader (long sync) and its backing state bloom if snap // sync is requested. The downloader is responsible for deallocating the state // bloom when it's done. - h.downloader = downloader.New(h.checkpointNumber, config.Database, h.eventMux, h.chain, nil, h.removePeer) + h.downloader = downloader.New(h.checkpointNumber, config.Database, h.eventMux, h.chain, nil, h.removePeer, success) // Construct the fetcher (short sync) validator := func(header *types.Header) error { @@ -403,8 +424,8 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { } }() } - // If we have any explicit whitelist block hashes, request them - for number, hash := range h.whitelist { + // If we have any explicit peer required block hashes, request them + for number := range h.peerRequiredBlocks { resCh := make(chan *eth.Response) if _, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh); err != nil { return err @@ -417,25 +438,25 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { case res := <-resCh: headers := ([]*types.Header)(*res.Res.(*eth.BlockHeadersPacket)) if len(headers) == 0 { - // Whitelisted blocks are allowed to be missing if the remote + // Required blocks are allowed to be missing if the remote // node is not yet synced res.Done <- nil return } // Validate the header and either drop the peer or continue if len(headers) > 1 { - res.Done <- errors.New("too many headers in whitelist response") + res.Done <- errors.New("too many headers in required block response") return } if headers[0].Number.Uint64() != number || headers[0].Hash() != hash { - peer.Log().Info("Whitelist mismatch, dropping peer", "number", number, "hash", headers[0].Hash(), "want", hash) - res.Done <- errors.New("whitelist block mismatch") + peer.Log().Info("Required block mismatch, dropping peer", "number", number, "hash", headers[0].Hash(), "want", hash) + res.Done <- errors.New("required block mismatch") return } - peer.Log().Debug("Whitelist block verified", "number", number, "hash", hash) + peer.Log().Debug("Peer required block verified", "number", number, "hash", hash) res.Done <- nil case <-timeout.C: - peer.Log().Warn("Whitelist challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) + peer.Log().Warn("Required block challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) h.removePeer(peer.ID()) } }(number, hash) diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 6e1c57cb6c45..7d5027ae77f6 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -570,7 +570,7 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo t.Fatalf("failed to answer challenge: %v", err) } } else { - responseRlp, _ := rlp.EncodeToBytes(types.Header{Number: response.Number}) + responseRlp, _ := rlp.EncodeToBytes(&types.Header{Number: response.Number}) if err := remote.ReplyBlockHeadersRLP(request.RequestId, []rlp.RawValue{responseRlp}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } diff --git a/eth/peerset.go b/eth/peerset.go index 1e864a8e46f2..3e54a481e36b 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -230,7 +230,7 @@ func (ps *peerSet) snapLen() int { } // peerWithHighestTD retrieves the known peer with the currently highest total -// difficulty. +// difficulty, but below the given PoS switchover threshold. func (ps *peerSet) peerWithHighestTD() *eth.Peer { ps.lock.RLock() defer ps.lock.RUnlock() diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 7d9b378839b3..55e612b801e0 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -264,11 +264,11 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + p2p.Send(peer.app, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ RequestId: 123, GetBlockHeadersPacket: tt.query, }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, &BlockHeadersPacket66{ RequestId: 123, BlockHeadersPacket: headers, }); err != nil { @@ -279,14 +279,12 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + p2p.Send(peer.app, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ RequestId: 456, GetBlockHeadersPacket: tt.query, }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 456, - BlockHeadersPacket: headers, - }); err != nil { + expected := &BlockHeadersPacket66{RequestId: 456, BlockHeadersPacket: headers} + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, expected); err != nil { t.Errorf("test %d by hash: headers mismatch: %v", i, err) } } @@ -364,11 +362,11 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ + p2p.Send(peer.app, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ RequestId: 123, GetBlockBodiesPacket: hashes, }) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, &BlockBodiesPacket66{ RequestId: 123, BlockBodiesPacket: bodies, }); err != nil { @@ -436,7 +434,7 @@ func testGetNodeData(t *testing.T, protocol uint) { it.Release() // Request all hashes. - p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ + p2p.Send(peer.app, GetNodeDataMsg, &GetNodeDataPacket66{ RequestId: 123, GetNodeDataPacket: hashes, }) @@ -546,11 +544,11 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ + p2p.Send(peer.app, GetReceiptsMsg, &GetReceiptsPacket66{ RequestId: 123, GetReceiptsPacket: hashes, }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, &ReceiptsPacket66{ RequestId: 123, ReceiptsPacket: receipts, }); err != nil { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 4161420f3a35..a8af9640bb89 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -241,7 +241,7 @@ func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs [ p.knownTxs.Add(hashes...) // Not packed into PooledTransactionsPacket to avoid RLP decoding - return p2p.Send(p.rw, PooledTransactionsMsg, PooledTransactionsRLPPacket66{ + return p2p.Send(p.rw, PooledTransactionsMsg, &PooledTransactionsRLPPacket66{ RequestId: id, PooledTransactionsRLPPacket: txs, }) @@ -298,7 +298,7 @@ func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { // ReplyBlockHeaders is the eth/66 version of SendBlockHeaders. func (p *Peer) ReplyBlockHeadersRLP(id uint64, headers []rlp.RawValue) error { - return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersRLPPacket66{ + return p2p.Send(p.rw, BlockHeadersMsg, &BlockHeadersRLPPacket66{ RequestId: id, BlockHeadersRLPPacket: headers, }) @@ -307,7 +307,7 @@ func (p *Peer) ReplyBlockHeadersRLP(id uint64, headers []rlp.RawValue) error { // ReplyBlockBodiesRLP is the eth/66 version of SendBlockBodiesRLP. func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { // Not packed into BlockBodiesPacket to avoid RLP decoding - return p2p.Send(p.rw, BlockBodiesMsg, BlockBodiesRLPPacket66{ + return p2p.Send(p.rw, BlockBodiesMsg, &BlockBodiesRLPPacket66{ RequestId: id, BlockBodiesRLPPacket: bodies, }) @@ -315,7 +315,7 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { // ReplyNodeData is the eth/66 response to GetNodeData. func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { - return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket66{ + return p2p.Send(p.rw, NodeDataMsg, &NodeDataPacket66{ RequestId: id, NodeDataPacket: data, }) @@ -323,7 +323,7 @@ func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { // ReplyReceiptsRLP is the eth/66 response to GetReceipts. func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { - return p2p.Send(p.rw, ReceiptsMsg, ReceiptsRLPPacket66{ + return p2p.Send(p.rw, ReceiptsMsg, &ReceiptsRLPPacket66{ RequestId: id, ReceiptsRLPPacket: receipts, }) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index d4e7f167666c..665d7601cfe2 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -168,7 +168,7 @@ type bytecodeResponse struct { // to actual requests and to validate any security constraints. // // Concurrency note: storage requests and responses are handled concurrently from -// the main runloop to allow Merkel proof verifications on the peer's thread and +// the main runloop to allow Merkle proof verifications on the peer's thread and // to drop on invalid response. The request struct must contain all the data to // construct the response without accessing runloop internals (i.e. tasks). That // is only included to allow the runloop to match a response to the task being @@ -2826,7 +2826,10 @@ func (s *Syncer) reportSyncProgress(force bool) { new(big.Int).Mul(new(big.Int).SetUint64(uint64(synced)), hashSpace), accountFills, ).Uint64()) - + // Don't report anything until we have a meaningful progress + if estBytes < 1.0 { + return + } elapsed := time.Since(s.startTime) estTime := elapsed / time.Duration(synced) * time.Duration(estBytes) diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 47ab1f026dd2..879ce8b6b2db 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1349,7 +1349,7 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { accTrie, _ := trie.New(common.Hash{}, db) var entries entrySlice for i := uint64(1); i <= uint64(n); i++ { - value, _ := rlp.EncodeToBytes(types.StateAccount{ + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), Root: emptyRoot, @@ -1394,7 +1394,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { } // Fill boundary accounts for i := 0; i < len(boundaries); i++ { - value, _ := rlp.EncodeToBytes(types.StateAccount{ + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: uint64(0), Balance: big.NewInt(int64(i)), Root: emptyRoot, @@ -1406,7 +1406,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { } // Fill other accounts if required for i := uint64(1); i <= uint64(n); i++ { - value, _ := rlp.EncodeToBytes(types.StateAccount{ + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), Root: emptyRoot, @@ -1442,7 +1442,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) stTrie, stEntries := makeStorageTrieWithSeed(uint64(slots), i, db) stRoot := stTrie.Hash() stTrie.Commit(nil) - value, _ := rlp.EncodeToBytes(types.StateAccount{ + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), Root: stRoot, @@ -1489,7 +1489,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie if code { codehash = getCodeHash(i) } - value, _ := rlp.EncodeToBytes(types.StateAccount{ + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), Root: stRoot, diff --git a/eth/sync.go b/eth/sync.go index b8ac67d3b2d1..d67d2311d0d9 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -17,6 +17,7 @@ package eth import ( + "errors" "math/big" "sync/atomic" "time" @@ -65,6 +66,7 @@ type chainSyncer struct { handler *handler force *time.Timer forced bool // true when force timer fired + warned time.Time peerEventCh chan struct{} doneCh chan error // non-nil when sync is running } @@ -119,10 +121,18 @@ func (cs *chainSyncer) loop() { select { case <-cs.peerEventCh: // Peer information changed, recheck. - case <-cs.doneCh: + case err := <-cs.doneCh: cs.doneCh = nil cs.force.Reset(forceSyncCycle) cs.forced = false + + // If we've reached the merge transition but no beacon client is available, or + // it has not yet switched us over, keep warning the user that their infra is + // potentially flaky. + if errors.Is(err, downloader.ErrMergeTransition) && time.Since(cs.warned) > 10*time.Second { + log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") + cs.warned = time.Now() + } case <-cs.force.C: cs.forced = true @@ -143,9 +153,16 @@ func (cs *chainSyncer) loop() { // nextSyncOp determines whether sync is required at this time. func (cs *chainSyncer) nextSyncOp() *chainSyncOp { if cs.doneCh != nil { - return nil // Sync already running. + return nil // Sync already running } - // Disable the td based sync trigger after the transition + // If a beacon client once took over control, disable the entire legacy sync + // path from here on end. Note, there is a slight "race" between reaching TTD + // and the beacon client taking over. The downloader will enforce that nothing + // above the first TTD will be delivered to the chain for import. + // + // An alternative would be to check the local chain for exceeding the TTD and + // avoid triggering a sync in that case, but that could also miss sibling or + // other family TTD block being accepted. if cs.handler.merger.TDDReached() { return nil } @@ -159,16 +176,24 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { if cs.handler.peers.len() < minPeers { return nil } - // We have enough peers, check TD + // We have enough peers, pick the one with the highest TD, but avoid going + // over the terminal total difficulty. Above that we expect the consensus + // clients to direct the chain head to sync to. peer := cs.handler.peers.peerWithHighestTD() if peer == nil { return nil } mode, ourTD := cs.modeAndLocalHead() - op := peerToSyncOp(mode, peer) if op.td.Cmp(ourTD) <= 0 { - return nil // We're in sync. + // We seem to be in sync according to the legacy rules. In the merge + // world, it can also mean we're stuck on the merge block, waiting for + // a beacon client. In the latter case, notify the user. + if ttd := cs.handler.chain.Config().TerminalTotalDifficulty; ttd != nil && ourTD.Cmp(ttd) >= 0 && time.Since(cs.warned) > 10*time.Second { + log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") + cs.warned = time.Now() + } + return nil // We're in sync } return op } @@ -227,7 +252,7 @@ func (h *handler) doSync(op *chainSyncOp) error { } } // Run the sync cycle, and disable snap sync if we're past the pivot block - err := h.downloader.Synchronise(op.peer.ID(), op.head, op.td, op.mode) + err := h.downloader.LegacySync(op.peer.ID(), op.head, op.td, h.chain.Config().TerminalTotalDifficulty, op.mode) if err != nil { return err } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index b16b68c1fa04..6055a556eb7c 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -452,7 +452,7 @@ func (api *API) TraceBlockByHash(ctx context.Context, hash common.Hash, config * // TraceBlock returns the structured logs created during the execution of EVM // and returns them as a JSON object. -func (api *API) TraceBlock(ctx context.Context, blob []byte, config *TraceConfig) ([]*txTraceResult, error) { +func (api *API) TraceBlock(ctx context.Context, blob hexutil.Bytes, config *TraceConfig) ([]*txTraceResult, error) { block := new(types.Block) if err := rlp.Decode(bytes.NewReader(blob), block); err != nil { return nil, fmt.Errorf("could not decode block: %v", err) diff --git a/ethdb/batch.go b/ethdb/batch.go index 1353693318a6..541f40c838d2 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -43,6 +43,9 @@ type Batcher interface { // NewBatch creates a write-only database that buffers changes to its host db // until a final write is called. NewBatch() Batch + + // NewBatchWithSize creates a write-only database batch with pre-allocated buffer. + NewBatchWithSize(size int) Batch } // HookedBatch wraps an arbitrary batch where each operation may be hooked into diff --git a/ethdb/database.go b/ethdb/database.go index 0a5729c6c1ec..b2e7c7228a0a 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -64,6 +64,7 @@ type KeyValueStore interface { Iteratee Stater Compacter + Snapshotter io.Closer } @@ -86,6 +87,10 @@ type AncientReader interface { // Ancients returns the ancient item numbers in the ancient store. Ancients() (uint64, error) + // Tail returns the number of first stored item in the freezer. + // This number can also be interpreted as the total deleted item numbers. + Tail() (uint64, error) + // AncientSize returns the ancient size of the specified category. AncientSize(kind string) (uint64, error) } @@ -106,11 +111,24 @@ type AncientWriter interface { // The integer return value is the total size of the written data. ModifyAncients(func(AncientWriteOp) error) (int64, error) - // TruncateAncients discards all but the first n ancient data from the ancient store. - TruncateAncients(n uint64) error + // TruncateHead discards all but the first n ancient data from the ancient store. + // After the truncation, the latest item can be accessed it item_n-1(start from 0). + TruncateHead(n uint64) error + + // TruncateTail discards the first n ancient data from the ancient store. The already + // deleted items are ignored. After the truncation, the earliest item can be accessed + // is item_n(start from 0). The deleted items may not be removed from the ancient store + // immediately, but only when the accumulated deleted data reach the threshold then + // will be removed all together. + TruncateTail(n uint64) error // Sync flushes all in-memory ancient store data to disk. Sync() error + + // MigrateTable processes and migrates entries of a given table to a new format. + // The second argument is a function that takes a raw entry and returns it + // in the newest format. + MigrateTable(string, func([]byte) ([]byte, error)) error } // AncientWriteOp is given to the function argument of ModifyAncients. @@ -153,5 +171,6 @@ type Database interface { Iteratee Stater Compacter + Snapshotter io.Closer } diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index 06ee2211e6f8..6b206af48d5e 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -313,6 +313,68 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { } }) + t.Run("Snapshot", func(t *testing.T) { + db := New() + defer db.Close() + + initial := map[string]string{ + "k1": "v1", "k2": "v2", "k3": "", "k4": "", + } + for k, v := range initial { + db.Put([]byte(k), []byte(v)) + } + snapshot, err := db.NewSnapshot() + if err != nil { + t.Fatal(err) + } + for k, v := range initial { + got, err := snapshot.Get([]byte(k)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, []byte(v)) { + t.Fatalf("Unexpected value want: %v, got %v", v, got) + } + } + + // Flush more modifications into the database, ensure the snapshot + // isn't affected. + var ( + update = map[string]string{"k1": "v1-b", "k3": "v3-b"} + insert = map[string]string{"k5": "v5-b"} + delete = map[string]string{"k2": ""} + ) + for k, v := range update { + db.Put([]byte(k), []byte(v)) + } + for k, v := range insert { + db.Put([]byte(k), []byte(v)) + } + for k := range delete { + db.Delete([]byte(k)) + } + for k, v := range initial { + got, err := snapshot.Get([]byte(k)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, []byte(v)) { + t.Fatalf("Unexpected value want: %v, got %v", v, got) + } + } + for k := range insert { + got, err := snapshot.Get([]byte(k)) + if err == nil || len(got) != 0 { + t.Fatal("Unexpected value") + } + } + for k := range delete { + got, err := snapshot.Get([]byte(k)) + if err != nil || len(got) == 0 { + t.Fatal("Unexpected deletion") + } + } + }) } func iterateKeys(it ethdb.Iterator) []string { diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 9a782dedbe14..15bd4e6eb3b5 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -213,6 +213,14 @@ func (db *Database) NewBatch() ethdb.Batch { } } +// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. +func (db *Database) NewBatchWithSize(size int) ethdb.Batch { + return &batch{ + db: db.db, + b: leveldb.MakeBatch(size), + } +} + // NewIterator creates a binary-alphabetical iterator over a subset // of database content with a particular key prefix, starting at a particular // initial key (or after, if it does not exist). @@ -220,6 +228,19 @@ func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { return db.db.NewIterator(bytesPrefixRange(prefix, start), nil) } +// NewSnapshot creates a database snapshot based on the current state. +// The created snapshot will not be affected by all following mutations +// happened on the database. +// Note don't forget to release the snapshot once it's used up, otherwise +// the stale data will never be cleaned up by the underlying compactor. +func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { + snap, err := db.db.GetSnapshot() + if err != nil { + return nil, err + } + return &snapshot{db: snap}, nil +} + // Stat returns a particular internal stat of the database. func (db *Database) Stat(property string) (string, error) { return db.db.GetProperty(property) @@ -519,3 +540,26 @@ func bytesPrefixRange(prefix, start []byte) *util.Range { r.Start = append(r.Start, start...) return r } + +// snapshot wraps a leveldb snapshot for implementing the Snapshot interface. +type snapshot struct { + db *leveldb.Snapshot +} + +// Has retrieves if a key is present in the snapshot backing by a key-value +// data store. +func (snap *snapshot) Has(key []byte) (bool, error) { + return snap.db.Has(key, nil) +} + +// Get retrieves the given key if it's present in the snapshot backing by +// key-value data store. +func (snap *snapshot) Get(key []byte) ([]byte, error) { + return snap.db.Get(key, nil) +} + +// Release releases associated resources. Release should always succeed and can +// be called multiple times without causing error. +func (snap *snapshot) Release() { + snap.db.Release() +} diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index 78181e860c15..95ec9bb8aa46 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -35,6 +35,10 @@ var ( // errMemorydbNotFound is returned if a key is requested that is not found in // the provided memory database. errMemorydbNotFound = errors.New("not found") + + // errSnapshotReleased is returned if callers want to retrieve data from a + // released snapshot. + errSnapshotReleased = errors.New("snapshot released") ) // Database is an ephemeral key-value store. Apart from basic data storage @@ -53,7 +57,7 @@ func New() *Database { } } -// NewWithCap returns a wrapped map pre-allocated to the provided capcity with +// NewWithCap returns a wrapped map pre-allocated to the provided capacity with // all the required database interface methods implemented. func NewWithCap(size int) *Database { return &Database{ @@ -129,6 +133,13 @@ func (db *Database) NewBatch() ethdb.Batch { } } +// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. +func (db *Database) NewBatchWithSize(size int) ethdb.Batch { + return &batch{ + db: db, + } +} + // NewIterator creates a binary-alphabetical iterator over a subset // of database content with a particular key prefix, starting at a particular // initial key (or after, if it does not exist). @@ -163,6 +174,13 @@ func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { } } +// NewSnapshot creates a database snapshot based on the current state. +// The created snapshot will not be affected by all following mutations +// happened on the database. +func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { + return newSnapshot(db), nil +} + // Stat returns a particular internal stat of the database. func (db *Database) Stat(property string) (string, error) { return "", errors.New("unknown property") @@ -313,3 +331,59 @@ func (it *iterator) Value() []byte { func (it *iterator) Release() { it.keys, it.values = nil, nil } + +// snapshot wraps a batch of key-value entries deep copied from the in-memory +// database for implementing the Snapshot interface. +type snapshot struct { + db map[string][]byte + lock sync.RWMutex +} + +// newSnapshot initializes the snapshot with the given database instance. +func newSnapshot(db *Database) *snapshot { + db.lock.RLock() + defer db.lock.RUnlock() + + copied := make(map[string][]byte) + for key, val := range db.db { + copied[key] = common.CopyBytes(val) + } + return &snapshot{db: copied} +} + +// Has retrieves if a key is present in the snapshot backing by a key-value +// data store. +func (snap *snapshot) Has(key []byte) (bool, error) { + snap.lock.RLock() + defer snap.lock.RUnlock() + + if snap.db == nil { + return false, errSnapshotReleased + } + _, ok := snap.db[string(key)] + return ok, nil +} + +// Get retrieves the given key if it's present in the snapshot backing by +// key-value data store. +func (snap *snapshot) Get(key []byte) ([]byte, error) { + snap.lock.RLock() + defer snap.lock.RUnlock() + + if snap.db == nil { + return nil, errSnapshotReleased + } + if entry, ok := snap.db[string(key)]; ok { + return common.CopyBytes(entry), nil + } + return nil, errMemorydbNotFound +} + +// Release releases associated resources. Release should always succeed and can +// be called multiple times without causing error. +func (snap *snapshot) Release() { + snap.lock.Lock() + defer snap.lock.Unlock() + + snap.db = nil +} diff --git a/ethdb/snapshot.go b/ethdb/snapshot.go new file mode 100644 index 000000000000..753e0f6b1f1e --- /dev/null +++ b/ethdb/snapshot.go @@ -0,0 +1,41 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethdb + +type Snapshot interface { + // Has retrieves if a key is present in the snapshot backing by a key-value + // data store. + Has(key []byte) (bool, error) + + // Get retrieves the given key if it's present in the snapshot backing by + // key-value data store. + Get(key []byte) ([]byte, error) + + // Release releases associated resources. Release should always succeed and can + // be called multiple times without causing error. + Release() +} + +// Snapshotter wraps the Snapshot method of a backing data store. +type Snapshotter interface { + // NewSnapshot creates a database snapshot based on the current state. + // The created snapshot will not be affected by all following mutations + // happened on the database. + // Note don't forget to release the snapshot once it's used up, otherwise + // the stale data will never be cleaned up by the underlying compactor. + NewSnapshot() (Snapshot, error) +} diff --git a/foundry/README.md b/foundry/README.md deleted file mode 100644 index 1e0dca3f88d0..000000000000 --- a/foundry/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Foundry README - -# Overview - -This document will go through the steps needed to test using Foundry. Currently, we use Foundry in the following capacity. - -1. Create a private network with our internal version of Geth. -2. Deploy a smart contract to the private network. -3. Test the smart contract on the private network. -4. Create a transaction on the private network. - -# Steps - -The steps to create a new project are as follows. - -## 1. Creating New Project - -1. `cd foundry/projects`. -2. Create a directory that captures your project: `mkdir local-private-network; cd local-private-network`. -3. Create a [new foundry project](https://onbjerg.github.io/foundry-book/forge/creating-a-new-project.html): `forge init stateful`. -4. Follow the foundry [documentation](https://onbjerg.github.io/foundry-book/forge/tests.html) for writing smart contract tests. - -## 2. Deployments - -You can choose to have custom deployments for your workflow. However, it is recommended to utilize Docker. - -# Existing Projects - -Below, you can find existing projects and their descriptions. - -## `local-private-network` - -The purpose of this project is as follows: - -1. Compile the geth from the local source. -2. Build a docker container with `ipld-eth-db` and another container for the `local-private-network`. -3. Run the compiled version of geth. -4. Deploy a smart contract to the private blockchain. -5. Trigger a transaction on the newly deployed smart contract. - -## Using This Project - -If you want to test your local geth code, do the following: - -1. cd `foundry/projects/local-private-network`. -2. `./wrapper.sh` - This script will do all the heavy lifting for you. -3. Keep an eye out for the outputs from the docker container. -4. Enter the docker container and do as you please. -5. If you want to change your geth code, you will have to run `./wrapper.sh` for subsequent runs. -6. If you do not change your geth code, you have to run: `docker-compose up --build`. - -### Key Notes: - -- The command to [deploy](https://onbjerg.github.io/foundry-book/forge/deploying.html) the smart contract is: `forge create --keystore $ETH_KEYSTORE_FILE --rpc-url [http://127.0.0.1:8545](http://127.0.0.1:8545/) --constructor-args 1 --password "" --legacy /root/stateful/src/Stateful.sol:Stateful` -- The command to interact create a [transaction](https://onbjerg.github.io/foundry-book/reference/cast.html) is: `cast send --keystore $ETH_KEYSTORE_FILE --rpc-url [http://127.0.0.1:8545](http://127.0.0.1:8545/) --password "" --legacy $DEPLOYED_ADDRESS "off()"` -- The `Dockerfile` compiles `cast` and `forge`. -- The `foundry/projects/local-private-network/deploy-local-network.sh` file does most heavy lifting. It spins up geth and triggers various events. -- The `foundry/projects/local-private-network/start-private-network.sh` file triggers `deploy-local-network.sh`. This file runs all the tests. -- The `geth` node will stay running even after the tests are terminated. diff --git a/foundry/projects/local-private-network/Dockerfile b/foundry/projects/local-private-network/Dockerfile deleted file mode 100644 index a618b609f2be..000000000000 --- a/foundry/projects/local-private-network/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM frolvlad/alpine-bash - -# copy all files - -RUN apk update ; apk add --no-cache --allow-untrusted ca-certificates curl bash git jq - -ENV GLIBC_REPO=https://github.com/sgerrand/alpine-pkg-glibc -ENV GLIBC_VERSION=2.35-r0 - -RUN set -ex && \ - apk --update add libstdc++ curl ca-certificates && \ - for pkg in glibc-${GLIBC_VERSION} glibc-bin-${GLIBC_VERSION}; \ - do curl -sSL ${GLIBC_REPO}/releases/download/${GLIBC_VERSION}/${pkg}.apk -o /tmp/${pkg}.apk; done && \ - apk add --allow-untrusted /tmp/*.apk ; \ - rm -v /tmp/*.apk ;/usr/glibc-compat/sbin/ldconfig /lib /usr/glibc-compat/lib - -RUN apk add gcompat; echo "Sorry" -WORKDIR /root - -COPY stateful ./stateful -ADD ./start-private-network.sh . -ADD ./deploy-local-network.sh . -ADD ../../geth-linux-amd64 /bin/geth - -RUN curl -L https://foundry.paradigm.xyz | bash; \ - /bin/bash -c 'source $HOME/.bashrc'; \ - /root/.foundry/bin/foundryup - -ENV PATH "$PATH:/root/.foundry/bin/" -RUN echo "export PATH=${PATH}" >> $HOME/.bashrc; - -RUN chmod +x /bin/geth - - -EXPOSE 8545 -EXPOSE 8546 -ENTRYPOINT ["./start-private-network.sh"] \ No newline at end of file diff --git a/foundry/projects/local-private-network/Stateful/lib/ds-test b/foundry/projects/local-private-network/Stateful/lib/ds-test deleted file mode 160000 index 0a5da56b0d65..000000000000 --- a/foundry/projects/local-private-network/Stateful/lib/ds-test +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0a5da56b0d65960e6a994d2ec8245e6edd38c248 diff --git a/foundry/projects/local-private-network/compile-geth.sh b/foundry/projects/local-private-network/compile-geth.sh deleted file mode 100755 index 75b2435a645f..000000000000 --- a/foundry/projects/local-private-network/compile-geth.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' - -start_path=$(pwd) -cd ../../../ -echo -e "${GREEN}Building geth!${NC}" -docker build -t vulcanize/go-ethereum -f Dockerfile . -docker run --rm --entrypoint cat vulcanize/go-ethereum /usr/local/bin/geth > foundry/projects/local-private-network/geth-linux-amd64 -chmod +x foundry/projects/local-private-network/geth-linux-amd64 - -echo -e "${GREEN}geth build complete!${NC}" -cd $start_path diff --git a/foundry/projects/local-private-network/deploy-local-network.sh b/foundry/projects/local-private-network/deploy-local-network.sh deleted file mode 100755 index 74c410c86ac9..000000000000 --- a/foundry/projects/local-private-network/deploy-local-network.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/bin/bash -set -e - -OPTS="./deploy-local-network.sh [] ... -./deploy-local-network.sh --help --- -db-user=name database user -db-password=password database password -db-name=name database name -db-host=address database host -db-port=port database port -db-write=bool turn on database write mode -db-type=name the type of database -db-driver=name the driver used for the database -db-waitforsync=bool Should the statediff service start once geth has synced to head (default: false) -rpc-port=port change RPC port (default: 8545) -rpc-addr=address change RPC address (default: 127.0.0.1) -chain-id=number change chain ID (default: 99) -period=seconds use a block time instead of instamine -accounts=number create multiple accounts (default: 1) -address=address eth address to add to genesis -save=name after finishing, save snapshot -load=name start from a previously saved snapshot -dir=directory testnet directory -" - -eval "$( - git rev-parse --parseopt -- "$@" <<<"$OPTS" || echo exit $? -)" - -DB_USER=vdbm -DB_PASSWORD=password -DB_NAME=vulcanize_public -DB_HOST=127.0.0.1 -DB_PORT=5432 -DB_TYPE=postgres -DB_DRIVER=sqlx -DB_WAIT_FOR_SYNC=false -RPC_PORT=8545 -RPC_ADDRESS=127.0.0.1 -PERIOD=0 -CHAINID=99 -ACCOUNTS=0 -ADDRESS= -gethdir=$HOME/testnet - -while [[ $1 ]]; do - case $1 in - --) shift; break;; - --db-user) shift; DB_USER=$1;; - --db-password) shift; DB_PASSWORD=$1;; - --db-name) shift; DB_NAME=$1;; - --db-host) shift; DB_HOST=$1;; - --db-port) shift; DB_PORT=$1;; - --db-write) shift; DB_WRITE=$1;; - --db-type) shift; DB_TYPE=$1;; - --db-driver) shift; DB_DRIVER=$1;; - --db-waitforsync) shift; DB_WAIT_FOR_SYNC=$1;; - --rpc-port) shift; RPC_PORT=$1;; - --rpc-addr) shift; RPC_ADDRESS=$1;; - --chain-id) shift; CHAINID=$1;; - --period) shift; PERIOD=$1;; - --accounts) shift; ACCOUNTS=$1;; - --save) shift; SAVE=$1;; - --address) shift; ADDRESS=$1;; - --load) shift; LOAD=$1;; - --dir) shift; gethdir=$1;; - *) printf "${0##*/}: internal error: %q\\n" "$1"; exit 1 - esac; shift -done - -chaindir=$gethdir/$RPC_PORT -#while true; do -# if [[ ! -d "$gethdir/$CHAINID" ]]; then break; fi -# CHAINID=$((CHAINID + 1)) -#done - -mkdir -p "$chaindir/config" -#if [ -n "$ADDRESS" ]; then -# balance+=(-n {} -s "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" -i balance \ -# -i "$ADDRESS") -#fi -for i in $(seq 0 "$ACCOUNTS"); do - address+=( "$( - geth 2>/dev/null account new --datadir "$chaindir" --password=<(exit) 2>/dev/null \ - | grep -o -E "0x[A-Fa-f0-9]*" )" ) -# balance+=(-n {} -s "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" -i balance \ -# -i "${address[i]}") - balance+=(' "'"${address[i]}"'": { "balance": "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}') -done - -#ALLOC_CLEAN=$(echo ${ALLOC} | jq .) -EXTRA_DATA="0x3132333400000000000000000000000000000000000000000000000000000000${address[0]#0x}0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" -JSON_VAL='{ - "config": { - "chainId": '"$CHAINID"', - "homesteadBlock": 0, - "eip150Block": 0, - "eip155Block": 0, - "eip158Block": 0, - "byzantiumBlock": 0, - "constantinopleBlock": 0, - "petersburgBlock": 0, - "istanbulBlock": 0, - "clique": { - "period": '"$PERIOD"', - "epoch": 3000 - } - }, - "difficulty": "0x1", - "gaslimit": "0xffffffffffff", - "extraData": "'"$EXTRA_DATA"'", - "alloc": {'"$balance"'} -}' -echo $JSON_VAL | jq . > $chaindir/config/genesis.json - -geth 2>/dev/null --datadir "$chaindir" init "$chaindir/config/genesis.json" - -export ETH_RPC_URL=http://$RPC_ADDRESS:$RPC_PORT - -port=$((RPC_PORT + 30000)) - -geth version -echo >&2 "dapp-testnet: RPC URL: $ETH_RPC_URL" -echo >&2 "dapp-testnet: TCP port: $port" -echo >&2 "dapp-testnet: Chain ID: $CHAINID" -echo >&2 "dapp-testnet: Database: $chaindir" -echo >&2 "dapp-testnet: Geth log: $chaindir/geth.log" - -printf "%s\n" "${address[@]}" > "$chaindir/config/account" -echo "$ETH_RPC_URL" > "$chaindir/config/rpc-url" -echo "$port" > "$chaindir/config/node-port" - -set +m -# Uncomment below once waitforsync has been merged -# geth \ -# 2> >(tee "$chaindir/geth.log" | grep --line-buffered Success | sed 's/^/geth: /' >&2) \ -# --datadir "$chaindir" --networkid "$CHAINID" --port="$port" \ -# --mine --miner.threads=1 --allow-insecure-unlock \ -# --http --http.api "web3,eth,net,debug,personal,statediff" --http.corsdomain '*' --http.vhosts '*' --nodiscover \ -# --http.addr="$RPC_ADDRESS" --http.port="$RPC_PORT" --syncmode=full --gcmode=archive \ -# --statediff --statediff.db.host="$DB_HOST" --statediff.db.port="$DB_PORT" --statediff.db.user="$DB_USER" \ -# --statediff.db.password="$DB_PASSWORD" --statediff.db.name="$DB_NAME" \ -# --statediff.db.nodeid 1 --statediff.db.clientname test1 --statediff.writing="$DB_WRITE" \ -# --statediff.db.type="$DB_TYPE" --statediff.db.driver="$DB_DRIVER" --statediff.waitforsync="$DB_WAIT_FOR_SYNC" \ -# --ws --ws.addr="0.0.0.0" --unlock="$(IFS=,; echo "${address[*]}")" --password=<(exit) & - -geth \ - 2> >(tee "$chaindir/geth.log" | grep --line-buffered Success | sed 's/^/geth: /' >&2) \ - --datadir "$chaindir" --networkid "$CHAINID" --port="$port" \ - --mine --miner.threads=1 --allow-insecure-unlock \ - --http --http.api "web3,eth,net,debug,personal,statediff" --http.corsdomain '*' --http.vhosts '*' --nodiscover \ - --http.addr="$RPC_ADDRESS" --http.port="$RPC_PORT" --syncmode=full --gcmode=archive \ - --statediff --statediff.db.host="$DB_HOST" --statediff.db.port="$DB_PORT" --statediff.db.user="$DB_USER" \ - --statediff.db.password="$DB_PASSWORD" --statediff.db.name="$DB_NAME" \ - --statediff.db.nodeid 1 --statediff.db.clientname test1 --statediff.writing="$DB_WRITE" \ - --statediff.db.type="$DB_TYPE" --statediff.db.driver="$DB_DRIVER" \ - --ws --ws.addr="0.0.0.0" --unlock="$(IFS=,; echo "${address[*]}")" --password=<(exit) & - -gethpid=$! - -clean() { - ( set -x; kill -INT $gethpid; wait ) - if [[ $SAVE ]]; then - echo >&2 "dapp-testnet: saving $gethdir/snapshots/$SAVE" - mkdir -p "$gethdir/snapshots/$SAVE" - cp -r "$chaindir/keystore" "$gethdir/snapshots/$SAVE" - cp -r "$chaindir/config" "$gethdir/snapshots/$SAVE" - geth >/dev/null 2>&1 --datadir "$chaindir" \ - export "$gethdir/snapshots/$SAVE/backup" - fi - ( set -x; rm -rf "$chaindir" ) -} -trap clean EXIT - -until curl -s "$ETH_RPC_URL"; do sleep 1; done - -# UPDATE -#ETH_FROM=$(seth --rpc-url="$ETH_RPC_URL" rpc eth_coinbase) -#export ETH_FROM -export ETH_KEYSTORE=$chaindir/keystore -export ETH_PASSWORD=/dev/null -printf 'dapp-testnet: Account: %s (default)\n' "${address[0]}" >&2 - -[[ "${#address[@]}" -gt 1 ]] && printf 'dapp-testnet: Account: %s\n' "${address[@]:1}" >&2 - -while true; do sleep 3600; done diff --git a/foundry/projects/local-private-network/docker-compose.yml b/foundry/projects/local-private-network/docker-compose.yml deleted file mode 100644 index b5ef47d2015f..000000000000 --- a/foundry/projects/local-private-network/docker-compose.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: "3.2" - -services: - foundry: - restart: unless-stopped - depends_on: - - ipld-eth-db - build: ./ - environment: - DB_USER: vdbm - DB_NAME: vulcanize_testing - DB_HOST: ipld-eth-db - DB_PORT: 5432 - DB_PASSWORD: password - DB_WRITE: "true" - DB_TYPE: postgres - DB_DRIVER: sqlx - DB_WAIT_FOR_SYNC: "true" - ports: - - "127.0.0.1:8545:8545" - - "127.0.0.1:8546:8546" - - ipld-eth-db: - restart: always - image: vulcanize/ipld-eth-db:v3.0.6 - environment: - POSTGRES_USER: "vdbm" - POSTGRES_DB: "vulcanize_testing" - POSTGRES_PASSWORD: "password" - volumes: - - vdb_db_eth_server:/var/lib/postgresql/data - ports: - - "127.0.0.1:8077:5432" - command: ["postgres", "-c", "log_statement=all"] - -volumes: - vdb_db_eth_server: diff --git a/foundry/projects/local-private-network/start-private-network.sh b/foundry/projects/local-private-network/start-private-network.sh deleted file mode 100755 index d4c7d33405c9..000000000000 --- a/foundry/projects/local-private-network/start-private-network.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -ex - -# clean up -trap 'killall geth && rm -rf "$TMPDIR"' EXIT -trap "exit 1" SIGINT SIGTERM - -TMPDIR=$(mktemp -d) -/bin/bash deploy-local-network.sh --rpc-addr 0.0.0.0 --chain-id 4 --db-user $DB_USER --db-password $DB_PASSWORD --db-name $DB_NAME \ - --db-host $DB_HOST --db-port $DB_PORT --db-write $DB_WRITE --dir "$TMPDIR" --address $ADDRESS \ - --db-type $DB_TYPE --db-driver $DB_DRIVER --db-waitforsync $DB_WAIT_FOR_SYNC & -echo "sleeping 90 sec" -# give it a few secs to start up -sleep 90 - -# Run tests -cd stateful -forge build -forge test --fork-url http://localhost:8545 - -# Deploy contracts - -ETH_KEYSTORE_FILES=() -echo "ETH KEYSTORE: $TMPDIR/8545/keystore" -for entry in `ls $TMPDIR/8545/keystore`; do - ETH_KEYSTORE_FILES+=("${TMPDIR}/8545/keystore/${entry}") -done - -echo "ETH_KEYSTORE_FILES: $ETH_KEYSTORE_FILES" -ETH_KEYSTORE_FILE=${ETH_KEYSTORE_FILES[0]} - -if [ "${#ETH_KEYSTORE_FILES[@]}" -eq 1 ]; then - echo "Only one KEYSTORE" -else - echo "WARNING: More than one file in keystore: ${ETH_KEYSTORE_FILES}" -fi - -DEPLOYED_ADDRESS=$(forge create --keystore $ETH_KEYSTORE_FILE --rpc-url http://127.0.0.1:8545 --constructor-args 1 --password "" --legacy /root/stateful/src/Stateful.sol:Stateful | grep "Deployed to:" | cut -d " " -f 3) -echo "Contract has been deployed to: $DEPLOYED_ADDRESS" - -# Call a transaction - -TX_OUT=$(cast send --keystore $ETH_KEYSTORE_FILE --rpc-url http://127.0.0.1:8545 --password "" --legacy $DEPLOYED_ADDRESS "off()") - -echo "TX OUTPUT: $TX_OUT" - - -# Run forever -tail -f /dev/null \ No newline at end of file diff --git a/foundry/projects/local-private-network/stateful/foundry.toml b/foundry/projects/local-private-network/stateful/foundry.toml deleted file mode 100644 index 19903e0b24ae..000000000000 --- a/foundry/projects/local-private-network/stateful/foundry.toml +++ /dev/null @@ -1,7 +0,0 @@ -[default] -src = 'src' -out = 'out' -libs = ['lib'] -remappings = ['ds-test/=lib/ds-test/src/'] - -# See more config options https://github.com/gakonst/foundry/tree/master/config \ No newline at end of file diff --git a/foundry/projects/local-private-network/stateful/src/Stateful.sol b/foundry/projects/local-private-network/stateful/src/Stateful.sol deleted file mode 100644 index e56c15691342..000000000000 --- a/foundry/projects/local-private-network/stateful/src/Stateful.sol +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: UNLICENSED -pragma solidity 0.8.10; - -contract Stateful { - uint x; - - constructor(uint y) public { - x = y; - } - - function off() public { - require(x == 1); - x = 0; - } - - function on() public { - require(x == 0); - x = 1; - } -} \ No newline at end of file diff --git a/foundry/projects/local-private-network/stateful/src/test/Stateful.t.sol b/foundry/projects/local-private-network/stateful/src/test/Stateful.t.sol deleted file mode 100644 index fb2e97e6eb3c..000000000000 --- a/foundry/projects/local-private-network/stateful/src/test/Stateful.t.sol +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: UNLICENSED -pragma solidity 0.8.10; - -import "ds-test/test.sol"; -import {Stateful} from "../Stateful.sol"; - -contract StatefulTest is DSTest { - Stateful contractA; - //contractA A; - uint x; - function setUp() public { - x = 1; - contractA = new Stateful(x); - } - - function testExample() public { - contractA.off(); - } -} diff --git a/foundry/projects/local-private-network/wrapper.sh b/foundry/projects/local-private-network/wrapper.sh deleted file mode 100755 index cc4481876d29..000000000000 --- a/foundry/projects/local-private-network/wrapper.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# This script will run everthing for you. Sit back and enjoy they show. - -set -e - -./compile-geth.sh -docker-compose up --build \ No newline at end of file diff --git a/go.mod b/go.mod index 669d85d16c28..507617709ce0 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,14 @@ module github.com/ethereum/go-ethereum go 1.15 require ( - github.com/Azure/azure-pipeline-go v0.2.2 // indirect - github.com/Azure/azure-storage-blob-go v0.7.0 - github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/VictoriaMetrics/fastcache v1.6.0 github.com/aws/aws-sdk-go-v2 v1.2.0 github.com/aws/aws-sdk-go-v2/config v1.1.1 github.com/aws/aws-sdk-go-v2/credentials v1.1.1 github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1 - github.com/btcsuite/btcd v0.20.1-beta + github.com/btcsuite/btcd/btcec/v2 v2.1.2 github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.14.0 github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f @@ -28,17 +26,18 @@ require ( github.com/georgysavva/scany v0.2.9 github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-stack/stack v1.8.0 + github.com/golang-jwt/jwt/v4 v4.3.0 github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.4 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa - github.com/google/uuid v1.1.5 + github.com/google/uuid v1.2.0 github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v1.3.0 github.com/hashicorp/go-bexpr v0.1.10 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.2.0 - github.com/huin/goupnp v1.0.2 + github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204 github.com/influxdata/influxdb v1.8.3 github.com/influxdata/influxdb-client-go/v2 v2.4.0 github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect @@ -63,6 +62,7 @@ require ( github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 + github.com/pganalyze/pg_query_go/v2 v2.1.0 github.com/prometheus/tsdb v0.7.1 github.com/rjeczalik/notify v0.9.1 github.com/rs/cors v1.7.0 @@ -70,14 +70,15 @@ require ( github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + github.com/thoas/go-funk v0.9.2 github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 - golang.org/x/text v0.3.6 + golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba - golang.org/x/tools v0.1.2 // indirect + golang.org/x/tools v0.1.0 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 diff --git a/go.sum b/go.sum index 136d318067e0..a06c7ed9d692 100644 --- a/go.sum +++ b/go.sum @@ -18,27 +18,12 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-storage-blob-go v0.7.0 h1:MuueVOYkufCxJw5YZzF842DY2MBsp+hLuh2apKY0mck= -github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= -github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0 h1:CxTzQrySOxDnKpLjFJeZAS5Qrv/qFPkgLjx5bOAi//I= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -49,7 +34,6 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIO github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -79,15 +63,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/btcsuite/btcd/btcec/v2 v2.1.2 h1:YoYoC9J0jwfukodSBMzZYUVQ8PTiYg4BnOWiJVzTmLs= +github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0 h1:MSskdM4/xJYcFzy0altH/C/xHopifpWzHUi1JeVI34Q= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= @@ -116,22 +95,27 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48 h1:iZOop7pqsg+56twTopWgwCGxdB5SI2yDO8Ti7eTRliQ= @@ -189,6 +173,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= +github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= @@ -222,6 +208,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= @@ -232,8 +219,8 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= -github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -256,8 +243,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iU github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= -github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204 h1:+EYBkW+dbi3F/atB+LSQZSWh7+HNrV3A/N0y6DSoy9k= +github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -378,7 +365,6 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5f github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -386,7 +372,6 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -401,7 +386,6 @@ github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= @@ -442,9 +426,6 @@ github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -472,6 +453,7 @@ github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjU github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -500,11 +482,9 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -516,6 +496,8 @@ github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChl github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/pganalyze/pg_query_go/v2 v2.1.0 h1:donwPZ4G/X+kMs7j5eYtKjdziqyOLVp3pkUrzb9lDl8= +github.com/pganalyze/pg_query_go/v2 v2.1.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -585,6 +567,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/thoas/go-funk v0.9.2 h1:oKlNYv0AY5nyf9g+/GhMgS/UO2ces0QRdPKwkhY3VCk= +github.com/thoas/go-funk v0.9.2/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -618,7 +602,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -691,12 +674,14 @@ golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -708,7 +693,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -752,6 +736,7 @@ golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= @@ -765,8 +750,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -804,6 +790,7 @@ golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -884,8 +871,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/graphql/graphql.go b/graphql/graphql.go index 16e0eb654d97..cbd76465d668 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/filters" @@ -100,6 +101,14 @@ func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) { } func (a *Account) TransactionCount(ctx context.Context) (hexutil.Uint64, error) { + // Ask transaction pool for the nonce which includes pending transactions + if blockNr, ok := a.blockNrOrHash.Number(); ok && blockNr == rpc.PendingBlockNumber { + nonce, err := a.backend.GetPoolNonce(ctx, a.address) + if err != nil { + return 0, err + } + return hexutil.Uint64(nonce), nil + } state, err := a.getState(ctx) if err != nil { return 0, err @@ -245,6 +254,10 @@ func (t *Transaction) EffectiveGasPrice(ctx context.Context) (*hexutil.Big, erro if err != nil || tx == nil { return nil, err } + // Pending tx + if t.block == nil { + return nil, nil + } header, err := t.block.resolveHeader(ctx) if err != nil || header == nil { return nil, err @@ -285,6 +298,30 @@ func (t *Transaction) MaxPriorityFeePerGas(ctx context.Context) (*hexutil.Big, e } } +func (t *Transaction) EffectiveTip(ctx context.Context) (*hexutil.Big, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return nil, err + } + // Pending tx + if t.block == nil { + return nil, nil + } + header, err := t.block.resolveHeader(ctx) + if err != nil || header == nil { + return nil, err + } + if header.BaseFee == nil { + return (*hexutil.Big)(tx.GasPrice()), nil + } + + tip, err := tx.EffectiveGasTip(header.BaseFee) + if err != nil { + return nil, err + } + return (*hexutil.Big)(tip), nil +} + func (t *Transaction) Value(ctx context.Context) (hexutil.Big, error) { tx, err := t.resolve(ctx) if err != nil || tx == nil { @@ -598,6 +635,22 @@ func (b *Block) BaseFeePerGas(ctx context.Context) (*hexutil.Big, error) { return (*hexutil.Big)(header.BaseFee), nil } +func (b *Block) NextBaseFeePerGas(ctx context.Context) (*hexutil.Big, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return nil, err + } + chaincfg := b.backend.ChainConfig() + if header.BaseFee == nil { + // Make sure next block doesn't enable EIP-1559 + if !chaincfg.IsLondon(new(big.Int).Add(header.Number, common.Big1)) { + return nil, nil + } + } + nextBaseFee := misc.CalcBaseFee(chaincfg, header) + return (*hexutil.Big)(nextBaseFee), nil +} + func (b *Block) Parent(ctx context.Context) (*Block, error) { if _, err := b.resolveHeader(ctx); err != nil { return nil, err diff --git a/graphql/schema.go b/graphql/schema.go index 86060cd2388c..0013e7bae75c 100644 --- a/graphql/schema.go +++ b/graphql/schema.go @@ -69,7 +69,7 @@ const schema string = ` transaction: Transaction! } - #EIP-2718 + #EIP-2718 type AccessTuple{ address: Address! storageKeys : [Bytes32!]! @@ -94,10 +94,12 @@ const schema string = ` value: BigInt! # GasPrice is the price offered to miners for gas, in wei per unit. gasPrice: BigInt! - # MaxFeePerGas is the maximum fee per gas offered to include a transaction, in wei. - maxFeePerGas: BigInt - # MaxPriorityFeePerGas is the maximum miner tip per gas offered to include a transaction, in wei. - maxPriorityFeePerGas: BigInt + # MaxFeePerGas is the maximum fee per gas offered to include a transaction, in wei. + maxFeePerGas: BigInt + # MaxPriorityFeePerGas is the maximum miner tip per gas offered to include a transaction, in wei. + maxPriorityFeePerGas: BigInt + # EffectiveTip is the actual amount of reward going to miner after considering the max fee cap. + effectiveTip: BigInt # Gas is the maximum amount of gas this transaction can consume. gas: Long! # InputData is the data supplied to the target of the transaction. @@ -187,8 +189,10 @@ const schema string = ` gasLimit: Long! # GasUsed is the amount of gas that was used executing transactions in this block. gasUsed: Long! - # BaseFeePerGas is the fee perunit of gas burned by the protocol in this block. - baseFeePerGas: BigInt + # BaseFeePerGas is the fee per unit of gas burned by the protocol in this block. + baseFeePerGas: BigInt + # NextBaseFeePerGas is the fee per unit of gas which needs to be burned in the next block. + nextBaseFeePerGas: BigInt # Timestamp is the unix timestamp at which this block was mined. timestamp: Long! # LogsBloom is a bloom filter that can be used to check if a block may @@ -244,10 +248,10 @@ const schema string = ` gas: Long # GasPrice is the price, in wei, offered for each unit of gas. gasPrice: BigInt - # MaxFeePerGas is the maximum fee per gas offered, in wei. - maxFeePerGas: BigInt - # MaxPriorityFeePerGas is the maximum miner tip per gas offered, in wei. - maxPriorityFeePerGas: BigInt + # MaxFeePerGas is the maximum fee per gas offered, in wei. + maxFeePerGas: BigInt + # MaxPriorityFeePerGas is the maximum miner tip per gas offered, in wei. + maxPriorityFeePerGas: BigInt # Value is the value, in wei, sent along with the call. value: BigInt # Data is the data sent to the callee. diff --git a/graphql/service.go b/graphql/service.go index bcb0a4990d64..29d98ad74683 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -74,7 +74,7 @@ func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) return err } h := handler{Schema: s} - handler := node.NewHTTPHandlerStack(h, cors, vhosts) + handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) stack.RegisterHandler("GraphQL UI", "/graphql/ui", GraphiQL{}) stack.RegisterHandler("GraphQL", "/graphql", handler) diff --git a/internal/build/azure.go b/internal/build/azure.go index 9c9cc2dcc5f1..9d1c4f300a88 100644 --- a/internal/build/azure.go +++ b/internal/build/azure.go @@ -19,10 +19,9 @@ package build import ( "context" "fmt" - "net/url" "os" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" ) // AzureBlobstoreConfig is an authentication and configuration struct containing @@ -49,15 +48,11 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig) if err != nil { return err } - - pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) - - u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account)) - service := azblob.NewServiceURL(*u, pipeline) - - container := service.NewContainerURL(config.Container) - blockblob := container.NewBlockBlobURL(name) - + u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container) + container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil) + if err != nil { + return err + } // Stream the file to upload into the designated blobstore container in, err := os.Open(path) if err != nil { @@ -65,49 +60,41 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig) } defer in.Close() - _, err = blockblob.Upload(context.Background(), in, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}) + blockblob := container.NewBlockBlobClient(name) + _, err = blockblob.Upload(context.Background(), in, nil) return err } // AzureBlobstoreList lists all the files contained within an azure blobstore. -func AzureBlobstoreList(config AzureBlobstoreConfig) ([]azblob.BlobItem, error) { - credential := azblob.NewAnonymousCredential() - if len(config.Token) > 0 { - c, err := azblob.NewSharedKeyCredential(config.Account, config.Token) - if err != nil { - return nil, err - } - credential = c +func AzureBlobstoreList(config AzureBlobstoreConfig) ([]*azblob.BlobItemInternal, error) { + // Create an authenticated client against the Azure cloud + credential, err := azblob.NewSharedKeyCredential(config.Account, config.Token) + if err != nil { + return nil, err } - pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) - - u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account)) - service := azblob.NewServiceURL(*u, pipeline) - - var allBlobs []azblob.BlobItem - // List all the blobs from the container and return them - container := service.NewContainerURL(config.Container) - nextMarker := azblob.Marker{} - for nextMarker.NotDone() { - res, err := container.ListBlobsFlatSegment(context.Background(), nextMarker, azblob.ListBlobsSegmentOptions{ - MaxResults: 5000, // The server only gives max 5K items - }) - if err != nil { - return nil, err - } - allBlobs = append(allBlobs, res.Segment.BlobItems...) - nextMarker = res.NextMarker - + u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container) + container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil) + if err != nil { + return nil, err + } + var maxResults int32 = 5000 + pager := container.ListBlobsFlat(&azblob.ContainerListBlobFlatSegmentOptions{ + Maxresults: &maxResults, + }) + var allBlobs []*azblob.BlobItemInternal + for pager.NextPage(context.Background()) { + res := pager.PageResponse() + allBlobs = append(allBlobs, res.ContainerListBlobFlatSegmentResult.Segment.BlobItems...) } - return allBlobs, nil + return allBlobs, pager.Err() } // AzureBlobstoreDelete iterates over a list of files to delete and removes them // from the blobstore. -func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []azblob.BlobItem) error { +func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []*azblob.BlobItemInternal) error { if *DryRunFlag { for _, blob := range blobs { - fmt.Printf("would delete %s (%s) from %s/%s\n", blob.Name, blob.Properties.LastModified, config.Account, config.Container) + fmt.Printf("would delete %s (%s) from %s/%s\n", *blob.Name, blob.Properties.LastModified, config.Account, config.Container) } return nil } @@ -116,21 +103,18 @@ func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []azblob.BlobItem) if err != nil { return err } - - pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) - - u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account)) - service := azblob.NewServiceURL(*u, pipeline) - - container := service.NewContainerURL(config.Container) - + u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container) + container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil) + if err != nil { + return err + } // Iterate over the blobs and delete them for _, blob := range blobs { - blockblob := container.NewBlockBlobURL(blob.Name) - if _, err := blockblob.Delete(context.Background(), azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}); err != nil { + blockblob := container.NewBlockBlobClient(*blob.Name) + if _, err := blockblob.Delete(context.Background(), &azblob.DeleteBlobOptions{}); err != nil { return err } - fmt.Printf("deleted %s (%s)\n", blob.Name, blob.Properties.LastModified) + fmt.Printf("deleted %s (%s)\n", *blob.Name, blob.Properties.LastModified) } return nil } diff --git a/internal/build/download.go b/internal/build/download.go index 0ed0b5e130d1..efb223b32701 100644 --- a/internal/build/download.go +++ b/internal/build/download.go @@ -58,7 +58,7 @@ func (db *ChecksumDB) Verify(path string) error { } fileHash := hex.EncodeToString(h.Sum(nil)) if !db.findHash(filepath.Base(path), fileHash) { - return fmt.Errorf("invalid file hash %s", fileHash) + return fmt.Errorf("invalid file hash %s for %s", fileHash, filepath.Base(path)) } return nil } diff --git a/internal/build/util.go b/internal/build/util.go index 2bdced82ee2f..cd6db09d006b 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -17,6 +17,7 @@ package build import ( + "bufio" "bytes" "flag" "fmt" @@ -31,6 +32,7 @@ import ( "path/filepath" "strings" "text/template" + "time" ) var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands") @@ -115,7 +117,6 @@ func render(tpl *template.Template, outputFile string, outputPerm os.FileMode, x // the form sftp://[user@]host[:port]. func UploadSFTP(identityFile, host, dir string, files []string) error { sftp := exec.Command("sftp") - sftp.Stdout = nil sftp.Stderr = os.Stderr if identityFile != "" { sftp.Args = append(sftp.Args, "-i", identityFile) @@ -130,6 +131,10 @@ func UploadSFTP(identityFile, host, dir string, files []string) error { if err != nil { return fmt.Errorf("can't create stdin pipe for sftp: %v", err) } + stdout, err := sftp.StdoutPipe() + if err != nil { + return fmt.Errorf("can't create stdout pipe for sftp: %v", err) + } if err := sftp.Start(); err != nil { return err } @@ -137,8 +142,35 @@ func UploadSFTP(identityFile, host, dir string, files []string) error { for _, f := range files { fmt.Fprintln(in, "put", f, path.Join(dir, filepath.Base(f))) } + fmt.Fprintln(in, "exit") + // Some issue with the PPA sftp server makes it so the server does not + // respond properly to a 'bye', 'exit' or 'quit' from the client. + // To work around that, we check the output, and when we see the client + // exit command, we do a hard exit. + // See + // https://github.com/kolban-google/sftp-gcs/issues/23 + // https://github.com/mscdex/ssh2/pull/1111 + aborted := false + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + txt := scanner.Text() + fmt.Println(txt) + if txt == "sftp> exit" { + // Give it .5 seconds to exit (server might be fixed), then + // hard kill it from the outside + time.Sleep(500 * time.Millisecond) + aborted = true + sftp.Process.Kill() + } + } + }() stdin.Close() - return sftp.Wait() + err = sftp.Wait() + if aborted { + return nil + } + return err } // FindMainPackages finds all 'main' packages in the given directory and returns their diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index 43bbcf0201d8..095df03807d7 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -51,7 +51,7 @@ OPTIONS: AppHelpTemplate = `NAME: {{.App.Name}} - {{.App.Usage}} - Copyright 2013-2021 The go-ethereum Authors + Copyright 2013-2022 The go-ethereum Authors USAGE: {{.App.HelpName}} [options]{{if .App.Commands}} [command] [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} @@ -77,7 +77,7 @@ COPYRIGHT: ClefAppHelpTemplate = `NAME: {{.App.Name}} - {{.App.Usage}} - Copyright 2013-2021 The go-ethereum Authors + Copyright 2013-2022 The go-ethereum Authors USAGE: {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} @@ -143,6 +143,7 @@ func FlagCategory(flag cli.Flag, flagGroups []FlagGroup) string { // NewApp creates an app with sane defaults. func NewApp(gitCommit, gitDate, usage string) *cli.App { app := cli.NewApp() + app.EnableBashCompletion = true app.Name = filepath.Base(os.Args[0]) app.Author = "" app.Email = "" diff --git a/les/catalyst/api.go b/les/catalyst/api.go index 5f5193c3bbc9..141df0585b01 100644 --- a/les/catalyst/api.go +++ b/les/catalyst/api.go @@ -34,10 +34,11 @@ func Register(stack *node.Node, backend *les.LightEthereum) error { log.Warn("Catalyst mode enabled", "protocol", "les") stack.RegisterAPIs([]rpc.API{ { - Namespace: "engine", - Version: "1.0", - Service: NewConsensusAPI(backend), - Public: true, + Namespace: "engine", + Version: "1.0", + Service: NewConsensusAPI(backend), + Public: true, + Authenticated: true, }, }) return nil @@ -68,30 +69,31 @@ func NewConsensusAPI(les *les.LightEthereum) *ConsensusAPI { // we return an error since block creation is not supported in les mode func (api *ConsensusAPI) ForkchoiceUpdatedV1(heads beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) { if heads.HeadBlockHash == (common.Hash{}) { - return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: nil}, nil + log.Warn("Forkchoice requested update to zero hash") + return beacon.STATUS_INVALID, nil // TODO(karalabe): Why does someone send us this? } if err := api.checkTerminalTotalDifficulty(heads.HeadBlockHash); err != nil { if header := api.les.BlockChain().GetHeaderByHash(heads.HeadBlockHash); header == nil { // TODO (MariusVanDerWijden) trigger sync - return beacon.SYNCING, nil + return beacon.STATUS_SYNCING, nil } - return beacon.INVALID, err + return beacon.STATUS_INVALID, err } // If the finalized block is set, check if it is in our blockchain if heads.FinalizedBlockHash != (common.Hash{}) { if header := api.les.BlockChain().GetHeaderByHash(heads.FinalizedBlockHash); header == nil { // TODO (MariusVanDerWijden) trigger sync - return beacon.SYNCING, nil + return beacon.STATUS_SYNCING, nil } } // SetHead if err := api.setHead(heads.HeadBlockHash); err != nil { - return beacon.INVALID, err + return beacon.STATUS_INVALID, err } if payloadAttributes != nil { - return beacon.INVALID, errors.New("not supported") + return beacon.STATUS_INVALID, errors.New("not supported") } - return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: nil}, nil + return api.validForkChoiceResponse(), nil } // GetPayloadV1 returns a cached payload by id. It's not supported in les mode. @@ -100,7 +102,7 @@ func (api *ConsensusAPI) GetPayloadV1(payloadID beacon.PayloadID) (*beacon.Execu } // ExecutePayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) ExecutePayloadV1(params beacon.ExecutableDataV1) (beacon.ExecutePayloadResponse, error) { +func (api *ConsensusAPI) ExecutePayloadV1(params beacon.ExecutableDataV1) (beacon.PayloadStatusV1, error) { block, err := beacon.ExecutableDataToBlock(params) if err != nil { return api.invalid(), err @@ -113,7 +115,7 @@ func (api *ConsensusAPI) ExecutePayloadV1(params beacon.ExecutableDataV1) (beaco } */ // TODO (MariusVanDerWijden) we should return nil here not empty hash - return beacon.ExecutePayloadResponse{Status: beacon.SYNCING.Status, LatestValidHash: common.Hash{}}, nil + return beacon.PayloadStatusV1{Status: beacon.SYNCING, LatestValidHash: nil}, nil } parent := api.les.BlockChain().GetHeaderByHash(params.ParentHash) if parent == nil { @@ -130,12 +132,21 @@ func (api *ConsensusAPI) ExecutePayloadV1(params beacon.ExecutableDataV1) (beaco if merger := api.les.Merger(); !merger.TDDReached() { merger.ReachTTD() } - return beacon.ExecutePayloadResponse{Status: beacon.VALID.Status, LatestValidHash: block.Hash()}, nil + hash := block.Hash() + return beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &hash}, nil +} + +func (api *ConsensusAPI) validForkChoiceResponse() beacon.ForkChoiceResponse { + currentHash := api.les.BlockChain().CurrentHeader().Hash() + return beacon.ForkChoiceResponse{ + PayloadStatus: beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: ¤tHash}, + } } // invalid returns a response "INVALID" with the latest valid hash set to the current head. -func (api *ConsensusAPI) invalid() beacon.ExecutePayloadResponse { - return beacon.ExecutePayloadResponse{Status: beacon.INVALID.Status, LatestValidHash: api.les.BlockChain().CurrentHeader().Hash()} +func (api *ConsensusAPI) invalid() beacon.PayloadStatusV1 { + currentHash := api.les.BlockChain().CurrentHeader().Hash() + return beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: ¤tHash} } func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error { diff --git a/les/peer.go b/les/peer.go index c6c672942b59..499429739d23 100644 --- a/les/peer.go +++ b/les/peer.go @@ -213,7 +213,7 @@ func (p *peerCommons) sendReceiveHandshake(sendList keyValueList) (keyValueList, ) // Send out own handshake in a new thread go func() { - errc <- p2p.Send(p.rw, StatusMsg, sendList) + errc <- p2p.Send(p.rw, StatusMsg, &sendList) }() go func() { // In the mean time retrieve the remote status message @@ -421,7 +421,7 @@ func sendRequest(w p2p.MsgWriter, msgcode, reqID uint64, data interface{}) error ReqID uint64 Data interface{} } - return p2p.Send(w, msgcode, req{reqID, data}) + return p2p.Send(w, msgcode, &req{reqID, data}) } func (p *serverPeer) sendRequest(msgcode, reqID uint64, data interface{}, amount int) error { @@ -871,7 +871,7 @@ func (r *reply) send(bv uint64) error { ReqID, BV uint64 Data rlp.RawValue } - return p2p.Send(r.w, r.msgcode, resp{r.reqID, bv, r.data}) + return p2p.Send(r.w, r.msgcode, &resp{r.reqID, bv, r.data}) } // size returns the RLP encoded size of the message data diff --git a/les/test_helper.go b/les/test_helper.go index 10367ea800c4..480d249dca40 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -356,7 +356,7 @@ func (p *testPeer) handshakeWithServer(t *testing.T, td *big.Int, head common.Ha if err := p2p.ExpectMsg(p.app, StatusMsg, nil); err != nil { t.Fatalf("status recv: %v", err) } - if err := p2p.Send(p.app, StatusMsg, sendList); err != nil { + if err := p2p.Send(p.app, StatusMsg, &sendList); err != nil { t.Fatalf("status send: %v", err) } } @@ -389,7 +389,7 @@ func (p *testPeer) handshakeWithClient(t *testing.T, td *big.Int, head common.Ha if err := p2p.ExpectMsg(p.app, StatusMsg, nil); err != nil { t.Fatalf("status recv: %v", err) } - if err := p2p.Send(p.app, StatusMsg, sendList); err != nil { + if err := p2p.Send(p.app, StatusMsg, &sendList); err != nil { t.Fatalf("status send: %v", err) } } diff --git a/les/vflux/client/fillset_test.go b/les/vflux/client/fillset_test.go index 58240682c60d..ca5af8f07ecc 100644 --- a/les/vflux/client/fillset_test.go +++ b/les/vflux/client/fillset_test.go @@ -34,16 +34,20 @@ type testIter struct { } func (i *testIter) Next() bool { - i.waitCh <- struct{}{} + if _, ok := <-i.waitCh; !ok { + return false + } i.node = <-i.nodeCh - return i.node != nil + return true } func (i *testIter) Node() *enode.Node { return i.node } -func (i *testIter) Close() {} +func (i *testIter) Close() { + close(i.waitCh) +} func (i *testIter) push() { var id enode.ID @@ -53,7 +57,7 @@ func (i *testIter) push() { func (i *testIter) waiting(timeout time.Duration) bool { select { - case <-i.waitCh: + case i.waitCh <- struct{}{}: return true case <-time.After(timeout): return false diff --git a/les/vflux/server/clientpool.go b/les/vflux/server/clientpool.go index 87d783ebab2a..805de2d41b03 100644 --- a/les/vflux/server/clientpool.go +++ b/les/vflux/server/clientpool.go @@ -34,7 +34,7 @@ import ( var ( ErrNotConnected = errors.New("client not connected") ErrNoPriority = errors.New("priority too low to raise capacity") - ErrCantFindMaximum = errors.New("Unable to find maximum allowed capacity") + ErrCantFindMaximum = errors.New("unable to find maximum allowed capacity") ) // ClientPool implements a client database that assigns a priority to each client @@ -177,7 +177,7 @@ func (cp *ClientPool) Unregister(peer clientPeer) { cp.ns.SetField(peer.Node(), cp.setup.clientField, nil) } -// setConnectedBias sets the connection bias, which is applied to already connected clients +// SetConnectedBias sets the connection bias, which is applied to already connected clients // So that already connected client won't be kicked out very soon and we can ensure all // connected clients can have enough time to request or sync some data. func (cp *ClientPool) SetConnectedBias(bias time.Duration) { diff --git a/light/lightchain.go b/light/lightchain.go index 61309ce35601..0cc88b46e7e0 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -419,6 +419,9 @@ func (lc *LightChain) SetChainHead(header *types.Header) error { // In the case of a light chain, InsertHeaderChain also creates and posts light // chain events when necessary. func (lc *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) { + if len(chain) == 0 { + return 0, nil + } if atomic.LoadInt32(&lc.disableCheckFreq) == 1 { checkFreq = 0 } diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index 9fa63281c659..ccb7279b047e 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -173,7 +173,7 @@ func (n *ethNode) insertBlock(eb beacon.ExecutableDataV1) error { } switch n.typ { case eth2NormalNode, eth2MiningNode: - newResp, err := n.api.ExecutePayloadV1(eb) + newResp, err := n.api.NewPayloadV1(eb) if err != nil { return err } else if newResp.Status != "VALID" { diff --git a/mobile/geth.go b/mobile/geth.go index bad9e0589f92..709b68cbded8 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -220,14 +220,6 @@ func (n *Node) Start() error { return n.node.Start() } -// Stop terminates a running node along with all its services. If the node was not started, -// an error is returned. It is not possible to restart a stopped node. -// -// Deprecated: use Close() -func (n *Node) Stop() error { - return n.node.Close() -} - // GetEthereumClient retrieves a client to access the Ethereum subsystem. func (n *Node) GetEthereumClient() (client *EthereumClient, _ error) { rpc, err := n.node.Attach() diff --git a/node/api.go b/node/api.go index a685ecd6b334..1b32399f635c 100644 --- a/node/api.go +++ b/node/api.go @@ -274,11 +274,12 @@ func (api *privateAdminAPI) StartWS(host *string, port *int, allowedOrigins *str } // Enable WebSocket on the server. - server := api.node.wsServerForPort(*port) + server := api.node.wsServerForPort(*port, false) if err := server.setListenAddr(*host, *port); err != nil { return false, err } - if err := server.enableWS(api.node.rpcAPIs, config); err != nil { + openApis, _ := api.node.GetAPIs() + if err := server.enableWS(openApis, config); err != nil { return false, err } if err := server.start(); err != nil { diff --git a/node/config.go b/node/config.go index 26f00cd678ab..853190c95f71 100644 --- a/node/config.go +++ b/node/config.go @@ -36,6 +36,7 @@ import ( const ( datadirPrivateKey = "nodekey" // Path within the datadir to the node's private key + datadirJWTKey = "jwtsecret" // Path within the datadir to the node's jwt secret datadirDefaultKeyStore = "keystore" // Path within the datadir to the keystore datadirStaticNodes = "static-nodes.json" // Path within the datadir to the static node list datadirTrustedNodes = "trusted-nodes.json" // Path within the datadir to the trusted node list @@ -138,6 +139,16 @@ type Config struct { // HTTPPathPrefix specifies a path prefix on which http-rpc is to be served. HTTPPathPrefix string `toml:",omitempty"` + // AuthAddr is the listening address on which authenticated APIs are provided. + AuthAddr string `toml:",omitempty"` + + // AuthPort is the port number on which authenticated APIs are provided. + AuthPort int `toml:",omitempty"` + + // AuthVirtualHosts is the list of virtual hostnames which are allowed on incoming requests + // for the authenticated api. This is by default {'localhost'}. + AuthVirtualHosts []string `toml:",omitempty"` + // WSHost is the host interface on which to start the websocket RPC server. If // this field is empty, no websocket API endpoint will be started. WSHost string @@ -190,6 +201,9 @@ type Config struct { // AllowUnprotectedTxs allows non EIP-155 protected transactions to be send over RPC. AllowUnprotectedTxs bool `toml:",omitempty"` + + // JWTSecret is the hex-encoded jwt secret. + JWTSecret string `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into @@ -248,7 +262,7 @@ func (c *Config) HTTPEndpoint() string { // DefaultHTTPEndpoint returns the HTTP endpoint used by default. func DefaultHTTPEndpoint() string { - config := &Config{HTTPHost: DefaultHTTPHost, HTTPPort: DefaultHTTPPort} + config := &Config{HTTPHost: DefaultHTTPHost, HTTPPort: DefaultHTTPPort, AuthPort: DefaultAuthPort} return config.HTTPEndpoint() } diff --git a/node/defaults.go b/node/defaults.go index c685dde5d127..fd0277e29dc9 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -34,12 +34,25 @@ const ( DefaultWSPort = 8546 // Default TCP port for the websocket RPC server DefaultGraphQLHost = "localhost" // Default host interface for the GraphQL server DefaultGraphQLPort = 8547 // Default TCP port for the GraphQL server + DefaultAuthHost = "localhost" // Default host interface for the authenticated apis + DefaultAuthPort = 8551 // Default port for the authenticated apis +) + +var ( + DefaultAuthCors = []string{"localhost"} // Default cors domain for the authenticated apis + DefaultAuthVhosts = []string{"localhost"} // Default virtual hosts for the authenticated apis + DefaultAuthOrigins = []string{"localhost"} // Default origins for the authenticated apis + DefaultAuthPrefix = "" // Default prefix for the authenticated apis + DefaultAuthModules = []string{"eth", "engine"} ) // DefaultConfig contains reasonable default settings. var DefaultConfig = Config{ DataDir: DefaultDataDir(), HTTPPort: DefaultHTTPPort, + AuthAddr: DefaultAuthHost, + AuthPort: DefaultAuthPort, + AuthVirtualHosts: DefaultAuthVhosts, HTTPModules: []string{"net", "web3"}, HTTPVirtualHosts: []string{"localhost"}, HTTPTimeouts: rpc.DefaultHTTPTimeouts, diff --git a/node/endpoints.go b/node/endpoints.go index 1f85a5213168..166e39adb46f 100644 --- a/node/endpoints.go +++ b/node/endpoints.go @@ -60,8 +60,10 @@ func checkModuleAvailability(modules []string, apis []rpc.API) (bad, available [ } } for _, name := range modules { - if _, ok := availableSet[name]; !ok && name != rpc.MetadataApi { - bad = append(bad, name) + if _, ok := availableSet[name]; !ok { + if name != rpc.MetadataApi && name != rpc.EngineApi { + bad = append(bad, name) + } } } return bad, available diff --git a/node/jwt_handler.go b/node/jwt_handler.go new file mode 100644 index 000000000000..28d5b87c60bc --- /dev/null +++ b/node/jwt_handler.go @@ -0,0 +1,78 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package node + +import ( + "net/http" + "strings" + "time" + + "github.com/golang-jwt/jwt/v4" +) + +type jwtHandler struct { + keyFunc func(token *jwt.Token) (interface{}, error) + next http.Handler +} + +// newJWTHandler creates a http.Handler with jwt authentication support. +func newJWTHandler(secret []byte, next http.Handler) http.Handler { + return &jwtHandler{ + keyFunc: func(token *jwt.Token) (interface{}, error) { + return secret, nil + }, + next: next, + } +} + +// ServeHTTP implements http.Handler +func (handler *jwtHandler) ServeHTTP(out http.ResponseWriter, r *http.Request) { + var ( + strToken string + claims jwt.RegisteredClaims + ) + if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "Bearer ") { + strToken = strings.TrimPrefix(auth, "Bearer ") + } + if len(strToken) == 0 { + http.Error(out, "missing token", http.StatusForbidden) + return + } + // We explicitly set only HS256 allowed, and also disables the + // claim-check: the RegisteredClaims internally requires 'iat' to + // be no later than 'now', but we allow for a bit of drift. + token, err := jwt.ParseWithClaims(strToken, &claims, handler.keyFunc, + jwt.WithValidMethods([]string{"HS256"}), + jwt.WithoutClaimsValidation()) + + switch { + case err != nil: + http.Error(out, err.Error(), http.StatusForbidden) + case !token.Valid: + http.Error(out, "invalid token", http.StatusForbidden) + case !claims.VerifyExpiresAt(time.Now(), false): // optional + http.Error(out, "token is expired", http.StatusForbidden) + case claims.IssuedAt == nil: + http.Error(out, "missing issued-at", http.StatusForbidden) + case time.Since(claims.IssuedAt.Time) > 5*time.Second: + http.Error(out, "stale token", http.StatusForbidden) + case time.Until(claims.IssuedAt.Time) > 5*time.Second: + http.Error(out, "future token", http.StatusForbidden) + default: + handler.next.ServeHTTP(out, r) + } +} diff --git a/node/node.go b/node/node.go index ceab1c90902b..7c540306db2b 100644 --- a/node/node.go +++ b/node/node.go @@ -17,6 +17,7 @@ package node import ( + crand "crypto/rand" "errors" "fmt" "net/http" @@ -27,6 +28,8 @@ import ( "sync" "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -55,6 +58,8 @@ type Node struct { rpcAPIs []rpc.API // List of APIs currently provided by the node http *httpServer // ws *httpServer // + httpAuth *httpServer // + wsAuth *httpServer // ipc *ipcServer // Stores information about the ipc http server inprocHandler *rpc.Server // In-process RPC request handler to process the API requests @@ -147,7 +152,9 @@ func New(conf *Config) (*Node, error) { // Configure RPC servers. node.http = newHTTPServer(node.log, conf.HTTPTimeouts) + node.httpAuth = newHTTPServer(node.log, conf.HTTPTimeouts) node.ws = newHTTPServer(node.log, rpc.DefaultHTTPTimeouts) + node.wsAuth = newHTTPServer(node.log, rpc.DefaultHTTPTimeouts) node.ipc = newIPCServer(node.log, conf.IPCEndpoint()) return node, nil @@ -335,7 +342,41 @@ func (n *Node) closeDataDir() { } } -// configureRPC is a helper method to configure all the various RPC endpoints during node +// obtainJWTSecret loads the jwt-secret, either from the provided config, +// or from the default location. If neither of those are present, it generates +// a new secret and stores to the default location. +func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { + fileName := cliParam + if len(fileName) == 0 { + // no path provided, use default + fileName = n.ResolvePath(datadirJWTKey) + } + // try reading from file + log.Debug("Reading JWT secret", "path", fileName) + if data, err := os.ReadFile(fileName); err == nil { + jwtSecret := common.FromHex(strings.TrimSpace(string(data))) + if len(jwtSecret) == 32 { + return jwtSecret, nil + } + log.Error("Invalid JWT secret", "path", fileName, "length", len(jwtSecret)) + return nil, errors.New("invalid JWT secret") + } + // Need to generate one + jwtSecret := make([]byte, 32) + crand.Read(jwtSecret) + // if we're in --dev mode, don't bother saving, just show it + if fileName == "" { + log.Info("Generated ephemeral JWT secret", "secret", hexutil.Encode(jwtSecret)) + return jwtSecret, nil + } + if err := os.WriteFile(fileName, []byte(hexutil.Encode(jwtSecret)), 0600); err != nil { + return nil, err + } + log.Info("Generated JWT secret", "path", fileName) + return jwtSecret, nil +} + +// startRPC is a helper method to configure all the various RPC endpoints during node // startup. It's not meant to be called at any time afterwards as it makes certain // assumptions about the state of the node. func (n *Node) startRPC() error { @@ -349,55 +390,125 @@ func (n *Node) startRPC() error { return err } } + var ( + servers []*httpServer + open, all = n.GetAPIs() + ) - // Configure HTTP. - if n.config.HTTPHost != "" { - config := httpConfig{ + initHttp := func(server *httpServer, apis []rpc.API, port int) error { + if err := server.setListenAddr(n.config.HTTPHost, port); err != nil { + return err + } + if err := server.enableRPC(apis, httpConfig{ CorsAllowedOrigins: n.config.HTTPCors, Vhosts: n.config.HTTPVirtualHosts, Modules: n.config.HTTPModules, prefix: n.config.HTTPPathPrefix, + }); err != nil { + return err } - if err := n.http.setListenAddr(n.config.HTTPHost, n.config.HTTPPort); err != nil { + servers = append(servers, server) + return nil + } + + initWS := func(apis []rpc.API, port int) error { + server := n.wsServerForPort(port, false) + if err := server.setListenAddr(n.config.WSHost, port); err != nil { return err } - if err := n.http.enableRPC(n.rpcAPIs, config); err != nil { + if err := server.enableWS(n.rpcAPIs, wsConfig{ + Modules: n.config.WSModules, + Origins: n.config.WSOrigins, + prefix: n.config.WSPathPrefix, + }); err != nil { return err } + servers = append(servers, server) + return nil } + initAuth := func(apis []rpc.API, port int, secret []byte) error { + // Enable auth via HTTP + server := n.httpAuth + if err := server.setListenAddr(n.config.AuthAddr, port); err != nil { + return err + } + if err := server.enableRPC(apis, httpConfig{ + CorsAllowedOrigins: DefaultAuthCors, + Vhosts: n.config.AuthVirtualHosts, + Modules: DefaultAuthModules, + prefix: DefaultAuthPrefix, + jwtSecret: secret, + }); err != nil { + return err + } + servers = append(servers, server) + // Enable auth via WS + server = n.wsServerForPort(port, true) + if err := server.setListenAddr(n.config.AuthAddr, port); err != nil { + return err + } + if err := server.enableWS(apis, wsConfig{ + Modules: DefaultAuthModules, + Origins: DefaultAuthOrigins, + prefix: DefaultAuthPrefix, + jwtSecret: secret, + }); err != nil { + return err + } + servers = append(servers, server) + return nil + } + + // Set up HTTP. + if n.config.HTTPHost != "" { + // Configure legacy unauthenticated HTTP. + if err := initHttp(n.http, open, n.config.HTTPPort); err != nil { + return err + } + } // Configure WebSocket. if n.config.WSHost != "" { - server := n.wsServerForPort(n.config.WSPort) - config := wsConfig{ - Modules: n.config.WSModules, - Origins: n.config.WSOrigins, - prefix: n.config.WSPathPrefix, + // legacy unauthenticated + if err := initWS(open, n.config.WSPort); err != nil { + return err } - if err := server.setListenAddr(n.config.WSHost, n.config.WSPort); err != nil { + } + // Configure authenticated API + if len(open) != len(all) { + jwtSecret, err := n.obtainJWTSecret(n.config.JWTSecret) + if err != nil { return err } - if err := server.enableWS(n.rpcAPIs, config); err != nil { + if err := initAuth(all, n.config.AuthPort, jwtSecret); err != nil { return err } } - - if err := n.http.start(); err != nil { - return err + // Start the servers + for _, server := range servers { + if err := server.start(); err != nil { + return err + } } - return n.ws.start() + return nil } -func (n *Node) wsServerForPort(port int) *httpServer { - if n.config.HTTPHost == "" || n.http.port == port { - return n.http +func (n *Node) wsServerForPort(port int, authenticated bool) *httpServer { + httpServer, wsServer := n.http, n.ws + if authenticated { + httpServer, wsServer = n.httpAuth, n.wsAuth } - return n.ws + if n.config.HTTPHost == "" || httpServer.port == port { + return httpServer + } + return wsServer } func (n *Node) stopRPC() { n.http.stop() n.ws.stop() + n.httpAuth.stop() + n.wsAuth.stop() n.ipc.stop() n.stopInProc() } @@ -458,6 +569,17 @@ func (n *Node) RegisterAPIs(apis []rpc.API) { n.rpcAPIs = append(n.rpcAPIs, apis...) } +// GetAPIs return two sets of APIs, both the ones that do not require +// authentication, and the complete set +func (n *Node) GetAPIs() (unauthenticated, all []rpc.API) { + for _, api := range n.rpcAPIs { + if !api.Authenticated { + unauthenticated = append(unauthenticated, api) + } + } + return unauthenticated, n.rpcAPIs +} + // RegisterHandler mounts a handler on the given path on the canonical HTTP server. // // The name of the handler is shown in a log message when the HTTP server starts diff --git a/node/node_test.go b/node/node_test.go index 25cfa9d38d78..84f61f0c44c4 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -577,13 +577,13 @@ func (test rpcPrefixTest) check(t *testing.T, node *Node) { } } for _, path := range test.wantWS { - err := wsRequest(t, wsBase+path, "") + err := wsRequest(t, wsBase+path) if err != nil { t.Errorf("Error: %s: WebSocket connection failed: %v", path, err) } } for _, path := range test.wantNoWS { - err := wsRequest(t, wsBase+path, "") + err := wsRequest(t, wsBase+path) if err == nil { t.Errorf("Error: %s: WebSocket connection succeeded for path in wantNoWS", path) } diff --git a/node/rpcstack.go b/node/rpcstack.go index 2c55a070b229..d9c41cca5781 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -40,13 +40,15 @@ type httpConfig struct { CorsAllowedOrigins []string Vhosts []string prefix string // path prefix on which to mount http handler + jwtSecret []byte // optional JWT secret } // wsConfig is the JSON-RPC/Websocket configuration type wsConfig struct { - Origins []string - Modules []string - prefix string // path prefix on which to mount ws handler + Origins []string + Modules []string + prefix string // path prefix on which to mount ws handler + jwtSecret []byte // optional JWT secret } type rpcHandler struct { @@ -157,7 +159,7 @@ func (h *httpServer) start() error { } // Log http endpoint. h.log.Info("HTTP server started", - "endpoint", listener.Addr(), + "endpoint", listener.Addr(), "auth", (h.httpConfig.jwtSecret != nil), "prefix", h.httpConfig.prefix, "cors", strings.Join(h.httpConfig.CorsAllowedOrigins, ","), "vhosts", strings.Join(h.httpConfig.Vhosts, ","), @@ -285,7 +287,7 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { } h.httpConfig = config h.httpHandler.Store(&rpcHandler{ - Handler: NewHTTPHandlerStack(srv, config.CorsAllowedOrigins, config.Vhosts), + Handler: NewHTTPHandlerStack(srv, config.CorsAllowedOrigins, config.Vhosts, config.jwtSecret), server: srv, }) return nil @@ -309,7 +311,6 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { if h.wsAllowed() { return fmt.Errorf("JSON-RPC over WebSocket is already enabled") } - // Create RPC server and handler. srv := rpc.NewServer() if err := RegisterApis(apis, config.Modules, srv, false); err != nil { @@ -317,7 +318,7 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { } h.wsConfig = config h.wsHandler.Store(&rpcHandler{ - Handler: srv.WebsocketHandler(config.Origins), + Handler: NewWSHandlerStack(srv.WebsocketHandler(config.Origins), config.jwtSecret), server: srv, }) return nil @@ -362,13 +363,24 @@ func isWebsocket(r *http.Request) bool { } // NewHTTPHandlerStack returns wrapped http-related handlers -func NewHTTPHandlerStack(srv http.Handler, cors []string, vhosts []string) http.Handler { +func NewHTTPHandlerStack(srv http.Handler, cors []string, vhosts []string, jwtSecret []byte) http.Handler { // Wrap the CORS-handler within a host-handler handler := newCorsHandler(srv, cors) handler = newVHostHandler(vhosts, handler) + if len(jwtSecret) != 0 { + handler = newJWTHandler(jwtSecret, handler) + } return newGzipHandler(handler) } +// NewWSHandlerStack returns a wrapped ws-related handler. +func NewWSHandlerStack(srv http.Handler, jwtSecret []byte) http.Handler { + if len(jwtSecret) != 0 { + return newJWTHandler(jwtSecret, srv) + } + return srv +} + func newCorsHandler(srv http.Handler, allowedOrigins []string) http.Handler { // disable CORS support if user has not specified a custom CORS configuration if len(allowedOrigins) == 0 { diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index f92f0ba39693..60fcab5a9001 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -24,10 +24,12 @@ import ( "strconv" "strings" "testing" + "time" "github.com/ethereum/go-ethereum/internal/testlog" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + "github.com/golang-jwt/jwt/v4" "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" ) @@ -146,12 +148,12 @@ func TestWebsocketOrigins(t *testing.T) { srv := createAndStartServer(t, &httpConfig{}, true, &wsConfig{Origins: splitAndTrim(tc.spec)}) url := fmt.Sprintf("ws://%v", srv.listenAddr()) for _, origin := range tc.expOk { - if err := wsRequest(t, url, origin); err != nil { + if err := wsRequest(t, url, "Origin", origin); err != nil { t.Errorf("spec '%v', origin '%v': expected ok, got %v", tc.spec, origin, err) } } for _, origin := range tc.expFail { - if err := wsRequest(t, url, origin); err == nil { + if err := wsRequest(t, url, "Origin", origin); err == nil { t.Errorf("spec '%v', origin '%v': expected not to allow, got ok", tc.spec, origin) } } @@ -243,13 +245,18 @@ func createAndStartServer(t *testing.T, conf *httpConfig, ws bool, wsConf *wsCon } // wsRequest attempts to open a WebSocket connection to the given URL. -func wsRequest(t *testing.T, url, browserOrigin string) error { +func wsRequest(t *testing.T, url string, extraHeaders ...string) error { t.Helper() - t.Logf("checking WebSocket on %s (origin %q)", url, browserOrigin) + //t.Logf("checking WebSocket on %s (origin %q)", url, browserOrigin) headers := make(http.Header) - if browserOrigin != "" { - headers.Set("Origin", browserOrigin) + // Apply extra headers. + if len(extraHeaders)%2 != 0 { + panic("odd extraHeaders length") + } + for i := 0; i < len(extraHeaders); i += 2 { + key, value := extraHeaders[i], extraHeaders[i+1] + headers.Set(key, value) } conn, _, err := websocket.DefaultDialer.Dial(url, headers) if conn != nil { @@ -291,3 +298,79 @@ func rpcRequest(t *testing.T, url string, extraHeaders ...string) *http.Response } return resp } + +type testClaim map[string]interface{} + +func (testClaim) Valid() error { + return nil +} + +func TestJWT(t *testing.T) { + var secret = []byte("secret") + issueToken := func(secret []byte, method jwt.SigningMethod, input map[string]interface{}) string { + if method == nil { + method = jwt.SigningMethodHS256 + } + ss, _ := jwt.NewWithClaims(method, testClaim(input)).SignedString(secret) + return ss + } + expOk := []string{ + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 4})), + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 4})), + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ + "iat": time.Now().Unix(), + "exp": time.Now().Unix() + 2, + })), + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ + "iat": time.Now().Unix(), + "bar": "baz", + })), + } + expFail := []string{ + // future + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 6})), + // stale + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 6})), + // wrong algo + fmt.Sprintf("Bearer %v", issueToken(secret, jwt.SigningMethodHS512, testClaim{"iat": time.Now().Unix() + 4})), + // expired + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix(), "exp": time.Now().Unix()})), + // missing mandatory iat + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{})), + // wrong secret + fmt.Sprintf("Bearer %v", issueToken([]byte("wrong"), nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("Bearer %v", issueToken([]byte{}, nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("Bearer %v", issueToken(nil, nil, testClaim{"iat": time.Now().Unix()})), + // Various malformed syntax + fmt.Sprintf("%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("Bearer: %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("Bearer:%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("Bearer\t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + fmt.Sprintf("Bearer \t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + } + srv := createAndStartServer(t, &httpConfig{jwtSecret: []byte("secret")}, + true, &wsConfig{Origins: []string{"*"}, jwtSecret: []byte("secret")}) + wsUrl := fmt.Sprintf("ws://%v", srv.listenAddr()) + htUrl := fmt.Sprintf("http://%v", srv.listenAddr()) + + for i, token := range expOk { + if err := wsRequest(t, wsUrl, "Authorization", token); err != nil { + t.Errorf("test %d-ws, token '%v': expected ok, got %v", i, token, err) + } + if resp := rpcRequest(t, htUrl, "Authorization", token); resp.StatusCode != 200 { + t.Errorf("test %d-http, token '%v': expected ok, got %v", i, token, resp.StatusCode) + } + } + for i, token := range expFail { + if err := wsRequest(t, wsUrl, "Authorization", token); err == nil { + t.Errorf("tc %d-ws, token '%v': expected not to allow, got ok", i, token) + } + if resp := rpcRequest(t, htUrl, "Authorization", token); resp.StatusCode != 403 { + t.Errorf("tc %d-http, token '%v': expected not to allow, got %v", i, token, resp.StatusCode) + } + } + srv.stop() +} diff --git a/p2p/peer.go b/p2p/peer.go index 8f564e776de5..257027a5b74d 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -332,11 +332,11 @@ func (p *Peer) handle(msg Msg) error { msg.Discard() go SendItems(p.rw, pongMsg) case msg.Code == discMsg: - var reason [1]DiscReason // This is the last message. We don't need to discard or // check errors because, the connection will be closed after it. - rlp.Decode(msg.Payload, &reason) - return reason[0] + var m struct{ R DiscReason } + rlp.Decode(msg.Payload, &m) + return m.R case msg.Code < baseProtocolLength: // ignore other base protocol messages return msg.Discard() diff --git a/p2p/peer_error.go b/p2p/peer_error.go index 393cc86b0970..aad1a65c7ac3 100644 --- a/p2p/peer_error.go +++ b/p2p/peer_error.go @@ -54,7 +54,7 @@ func (pe *peerError) Error() string { var errProtocolReturned = errors.New("protocol returned") -type DiscReason uint +type DiscReason uint8 const ( DiscRequested DiscReason = iota diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index 6d7f0b6d7a31..f5172f3f23db 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -141,7 +141,7 @@ func (t *testService) Stop() error { // message with the given code func (t *testService) handshake(rw p2p.MsgReadWriter, code uint64) error { errc := make(chan error, 2) - go func() { errc <- p2p.Send(rw, code, struct{}{}) }() + go func() { errc <- p2p.SendItems(rw, code) }() go func() { errc <- p2p.ExpectMsg(rw, code, struct{}{}) }() for i := 0; i < 2; i++ { if err := <-errc; err != nil { diff --git a/params/bootnodes.go b/params/bootnodes.go index e3b5570d55b5..ed52e8ee64b1 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -24,12 +24,12 @@ var MainnetBootnodes = []string{ // Ethereum Foundation Go Bootnodes "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", // bootnode-aws-ap-southeast-1-001 "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", // bootnode-aws-us-east-1-001 - "enode://ca6de62fce278f96aea6ec5a2daadb877e51651247cb96ee310a318def462913b653963c155a0ef6c7d50048bba6e6cea881130857413d9f50a621546b590758@34.255.23.113:30303", // bootnode-aws-eu-west-1-001 - "enode://279944d8dcd428dffaa7436f25ca0ca43ae19e7bcf94a8fb7d1641651f92d121e972ac2e8f381414b80cc8e5555811c2ec6e1a99bb009b3f53c4c69923e11bd8@35.158.244.151:30303", // bootnode-aws-eu-central-1-001 "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", // bootnode-azure-australiaeast-001 "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", // bootnode-azure-brazilsouth-001 "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", // bootnode-azure-koreasouth-001 "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", // bootnode-azure-westus-001 + "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", // bootnode-hetzner-hel + "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", // bootnode-hetzner-fsn } // RopstenBootnodes are the enode URLs of the P2P bootstrap nodes running on the @@ -76,6 +76,13 @@ var GoerliBootnodes = []string{ "enode://a59e33ccd2b3e52d578f1fbd70c6f9babda2650f0760d6ff3b37742fdcdfdb3defba5d56d315b40c46b70198c7621e63ffa3f987389c7118634b0fefbbdfa7fd@51.15.119.157:40303", } +var KilnBootnodes = []string{ + "enode://c354db99124f0faf677ff0e75c3cbbd568b2febc186af664e0c51ac435609badedc67a18a63adb64dacc1780a28dcefebfc29b83fd1a3f4aa3c0eb161364cf94@164.92.130.5:30303", + "enode://d41af1662434cad0a88fe3c7c92375ec5719f4516ab6d8cb9695e0e2e815382c767038e72c224e04040885157da47422f756c040a9072676c6e35c5b1a383cce@138.68.66.103:30303", + "enode://91a745c3fb069f6b99cad10b75c463d527711b106b622756e9ef9f12d2631b6cb885f831d1c8731b9bc7177cae5e1ea1f1be087f86d7d30b590a91f22bc041b0@165.232.180.230:30303", + "enode://b74bd2e8a9f0c53f0c93bcce80818f2f19439fd807af5c7fbc3efb10130c6ee08be8f3aaec7dc0a057ad7b2a809c8f34dc62431e9b6954b07a6548cc59867884@164.92.140.200:30303", +} + var V5Bootnodes = []string{ // Teku team's bootnode "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", diff --git a/params/config.go b/params/config.go index 7f52472ec9dc..aee5b7e1c855 100644 --- a/params/config.go +++ b/params/config.go @@ -32,6 +32,7 @@ var ( SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") + KilnGenesisHash = common.HexToHash("0x51c7fe41be669f69c45c33a56982cbde405313342d9e2b00d7c91a7b284dd4f8") ) // TrustedCheckpoints associates each known checkpoint with the genesis hash of @@ -387,7 +388,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Berlin: %v, London: %v, Arrow Glacier: %v, MergeFork: %v, Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Berlin: %v, London: %v, Arrow Glacier: %v, MergeFork: %v, Terminal TD: %v, Engine: %v}", c.ChainID, c.HomesteadBlock, c.DAOForkBlock, @@ -404,6 +405,7 @@ func (c *ChainConfig) String() string { c.LondonBlock, c.ArrowGlacierBlock, c.MergeForkBlock, + c.TerminalTotalDifficulty, engine, ) } diff --git a/params/version.go b/params/version.go index e94d2aed4bef..770db7939aef 100644 --- a/params/version.go +++ b/params/version.go @@ -23,8 +23,8 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 16 // Patch version component of the current release - VersionMeta = "statediff-3.0.2" // Version metadata to append to the version string + VersionPatch = 17 // Patch version component of the current release + VersionMeta = "statediff-3.2.0" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/rlp/decode.go b/rlp/decode.go index 5f2e5ad5fea0..9214dbfb3720 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -27,6 +27,8 @@ import ( "reflect" "strings" "sync" + + "github.com/ethereum/go-ethereum/rlp/internal/rlpstruct" ) //lint:ignore ST1012 EOL is not an error. @@ -148,7 +150,7 @@ var ( bigInt = reflect.TypeOf(big.Int{}) ) -func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { +func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error) { kind := typ.Kind() switch { case typ == rawValueType: @@ -220,55 +222,20 @@ func decodeBigIntNoPtr(s *Stream, val reflect.Value) error { } func decodeBigInt(s *Stream, val reflect.Value) error { - var buffer []byte - kind, size, err := s.Kind() - switch { - case err != nil: - return wrapStreamError(err, val.Type()) - case kind == List: - return wrapStreamError(ErrExpectedString, val.Type()) - case kind == Byte: - buffer = s.uintbuf[:1] - buffer[0] = s.byteval - s.kind = -1 // re-arm Kind - case size == 0: - // Avoid zero-length read. - s.kind = -1 - case size <= uint64(len(s.uintbuf)): - // For integers smaller than s.uintbuf, allocating a buffer - // can be avoided. - buffer = s.uintbuf[:size] - if err := s.readFull(buffer); err != nil { - return wrapStreamError(err, val.Type()) - } - // Reject inputs where single byte encoding should have been used. - if size == 1 && buffer[0] < 128 { - return wrapStreamError(ErrCanonSize, val.Type()) - } - default: - // For large integers, a temporary buffer is needed. - buffer = make([]byte, size) - if err := s.readFull(buffer); err != nil { - return wrapStreamError(err, val.Type()) - } - } - - // Reject leading zero bytes. - if len(buffer) > 0 && buffer[0] == 0 { - return wrapStreamError(ErrCanonInt, val.Type()) - } - - // Set the integer bytes. i := val.Interface().(*big.Int) if i == nil { i = new(big.Int) val.Set(reflect.ValueOf(i)) } - i.SetBytes(buffer) + + err := s.decodeBigInt(i) + if err != nil { + return wrapStreamError(err, val.Type()) + } return nil } -func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { +func makeListDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) { etype := typ.Elem() if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { if typ.Kind() == reflect.Array { @@ -276,7 +243,7 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { } return decodeByteSlice, nil } - etypeinfo := theTC.infoWhileGenerating(etype, tags{}) + etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{}) if etypeinfo.decoderErr != nil { return nil, etypeinfo.decoderErr } @@ -286,7 +253,7 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { dec = func(s *Stream, val reflect.Value) error { return decodeListArray(s, val, etypeinfo.decoder) } - case tag.tail: + case tag.Tail: // A slice with "tail" tag can occur as the last field // of a struct and is supposed to swallow all remaining // list elements. The struct decoder already called s.List, @@ -451,16 +418,16 @@ func zeroFields(structval reflect.Value, fields []field) { } // makePtrDecoder creates a decoder that decodes into the pointer's element type. -func makePtrDecoder(typ reflect.Type, tag tags) (decoder, error) { +func makePtrDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) { etype := typ.Elem() - etypeinfo := theTC.infoWhileGenerating(etype, tags{}) + etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{}) switch { case etypeinfo.decoderErr != nil: return nil, etypeinfo.decoderErr - case !tag.nilOK: + case !tag.NilOK: return makeSimplePtrDecoder(etype, etypeinfo), nil default: - return makeNilPtrDecoder(etype, etypeinfo, tag.nilKind), nil + return makeNilPtrDecoder(etype, etypeinfo, tag), nil } } @@ -481,9 +448,13 @@ func makeSimplePtrDecoder(etype reflect.Type, etypeinfo *typeinfo) decoder { // values are decoded into a value of the element type, just like makePtrDecoder does. // // This decoder is used for pointer-typed struct fields with struct tag "nil". -func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, nilKind Kind) decoder { +func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, ts rlpstruct.Tags) decoder { typ := reflect.PtrTo(etype) nilPtr := reflect.Zero(typ) + + // Determine the value kind that results in nil pointer. + nilKind := typeNilKind(etype, ts) + return func(s *Stream, val reflect.Value) (err error) { kind, size, err := s.Kind() if err != nil { @@ -659,6 +630,37 @@ func (s *Stream) Bytes() ([]byte, error) { } } +// ReadBytes decodes the next RLP value and stores the result in b. +// The value size must match len(b) exactly. +func (s *Stream) ReadBytes(b []byte) error { + kind, size, err := s.Kind() + if err != nil { + return err + } + switch kind { + case Byte: + if len(b) != 1 { + return fmt.Errorf("input value has wrong size 1, want %d", len(b)) + } + b[0] = s.byteval + s.kind = -1 // rearm Kind + return nil + case String: + if uint64(len(b)) != size { + return fmt.Errorf("input value has wrong size %d, want %d", size, len(b)) + } + if err = s.readFull(b); err != nil { + return err + } + if size == 1 && b[0] < 128 { + return ErrCanonSize + } + return nil + default: + return ErrExpectedString + } +} + // Raw reads a raw encoded value including RLP type information. func (s *Stream) Raw() ([]byte, error) { kind, size, err := s.Kind() @@ -687,10 +689,31 @@ func (s *Stream) Raw() ([]byte, error) { // Uint reads an RLP string of up to 8 bytes and returns its contents // as an unsigned integer. If the input does not contain an RLP string, the // returned error will be ErrExpectedString. +// +// Deprecated: use s.Uint64 instead. func (s *Stream) Uint() (uint64, error) { return s.uint(64) } +func (s *Stream) Uint64() (uint64, error) { + return s.uint(64) +} + +func (s *Stream) Uint32() (uint32, error) { + i, err := s.uint(32) + return uint32(i), err +} + +func (s *Stream) Uint16() (uint16, error) { + i, err := s.uint(16) + return uint16(i), err +} + +func (s *Stream) Uint8() (uint8, error) { + i, err := s.uint(8) + return uint8(i), err +} + func (s *Stream) uint(maxbits int) (uint64, error) { kind, size, err := s.Kind() if err != nil { @@ -781,6 +804,65 @@ func (s *Stream) ListEnd() error { return nil } +// MoreDataInList reports whether the current list context contains +// more data to be read. +func (s *Stream) MoreDataInList() bool { + _, listLimit := s.listLimit() + return listLimit > 0 +} + +// BigInt decodes an arbitrary-size integer value. +func (s *Stream) BigInt() (*big.Int, error) { + i := new(big.Int) + if err := s.decodeBigInt(i); err != nil { + return nil, err + } + return i, nil +} + +func (s *Stream) decodeBigInt(dst *big.Int) error { + var buffer []byte + kind, size, err := s.Kind() + switch { + case err != nil: + return err + case kind == List: + return ErrExpectedString + case kind == Byte: + buffer = s.uintbuf[:1] + buffer[0] = s.byteval + s.kind = -1 // re-arm Kind + case size == 0: + // Avoid zero-length read. + s.kind = -1 + case size <= uint64(len(s.uintbuf)): + // For integers smaller than s.uintbuf, allocating a buffer + // can be avoided. + buffer = s.uintbuf[:size] + if err := s.readFull(buffer); err != nil { + return err + } + // Reject inputs where single byte encoding should have been used. + if size == 1 && buffer[0] < 128 { + return ErrCanonSize + } + default: + // For large integers, a temporary buffer is needed. + buffer = make([]byte, size) + if err := s.readFull(buffer); err != nil { + return err + } + } + + // Reject leading zero bytes. + if len(buffer) > 0 && buffer[0] == 0 { + return ErrCanonInt + } + // Set the integer bytes. + dst.SetBytes(buffer) + return nil +} + // Decode decodes a value and stores the result in the value pointed // to by val. Please see the documentation for the Decode function // to learn about the decoding rules. diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 7c3dafeac44d..babdf3891bba 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -286,6 +286,47 @@ func TestStreamRaw(t *testing.T) { } } +func TestStreamReadBytes(t *testing.T) { + tests := []struct { + input string + size int + err string + }{ + // kind List + {input: "C0", size: 1, err: "rlp: expected String or Byte"}, + // kind Byte + {input: "04", size: 0, err: "input value has wrong size 1, want 0"}, + {input: "04", size: 1}, + {input: "04", size: 2, err: "input value has wrong size 1, want 2"}, + // kind String + {input: "820102", size: 0, err: "input value has wrong size 2, want 0"}, + {input: "820102", size: 1, err: "input value has wrong size 2, want 1"}, + {input: "820102", size: 2}, + {input: "820102", size: 3, err: "input value has wrong size 2, want 3"}, + } + + for _, test := range tests { + test := test + name := fmt.Sprintf("input_%s/size_%d", test.input, test.size) + t.Run(name, func(t *testing.T) { + s := NewStream(bytes.NewReader(unhex(test.input)), 0) + b := make([]byte, test.size) + err := s.ReadBytes(b) + if test.err == "" { + if err != nil { + t.Errorf("unexpected error %q", err) + } + } else { + if err == nil { + t.Errorf("expected error, got nil") + } else if err.Error() != test.err { + t.Errorf("wrong error %q", err) + } + } + }) + } +} + func TestDecodeErrors(t *testing.T) { r := bytes.NewReader(nil) @@ -990,7 +1031,7 @@ func TestInvalidOptionalField(t *testing.T) { v interface{} err string }{ - {v: new(invalid1), err: `rlp: struct field rlp.invalid1.B needs "optional" tag`}, + {v: new(invalid1), err: `rlp: invalid struct tag "" for rlp.invalid1.B (must be optional because preceding field "A" is optional)`}, {v: new(invalid2), err: `rlp: invalid struct tag "optional" for rlp.invalid2.T (also has "tail" tag)`}, {v: new(invalid3), err: `rlp: invalid struct tag "tail" for rlp.invalid3.T (also has "optional" tag)`}, } diff --git a/rlp/encbuffer.go b/rlp/encbuffer.go new file mode 100644 index 000000000000..289e7448c6ca --- /dev/null +++ b/rlp/encbuffer.go @@ -0,0 +1,382 @@ +package rlp + +import ( + "io" + "math/big" + "reflect" + "sync" +) + +type encBuffer struct { + str []byte // string data, contains everything except list headers + lheads []listhead // all list headers + lhsize int // sum of sizes of all encoded list headers + sizebuf [9]byte // auxiliary buffer for uint encoding +} + +// The global encBuffer pool. +var encBufferPool = sync.Pool{ + New: func() interface{} { return new(encBuffer) }, +} + +func getEncBuffer() *encBuffer { + buf := encBufferPool.Get().(*encBuffer) + buf.reset() + return buf +} + +func (buf *encBuffer) reset() { + buf.lhsize = 0 + buf.str = buf.str[:0] + buf.lheads = buf.lheads[:0] +} + +// size returns the length of the encoded data. +func (buf *encBuffer) size() int { + return len(buf.str) + buf.lhsize +} + +// makeBytes creates the encoder output. +func (w *encBuffer) makeBytes() []byte { + out := make([]byte, w.size()) + w.copyTo(out) + return out +} + +func (w *encBuffer) copyTo(dst []byte) { + strpos := 0 + pos := 0 + for _, head := range w.lheads { + // write string data before header + n := copy(dst[pos:], w.str[strpos:head.offset]) + pos += n + strpos += n + // write the header + enc := head.encode(dst[pos:]) + pos += len(enc) + } + // copy string data after the last list header + copy(dst[pos:], w.str[strpos:]) +} + +// writeTo writes the encoder output to w. +func (buf *encBuffer) writeTo(w io.Writer) (err error) { + strpos := 0 + for _, head := range buf.lheads { + // write string data before header + if head.offset-strpos > 0 { + n, err := w.Write(buf.str[strpos:head.offset]) + strpos += n + if err != nil { + return err + } + } + // write the header + enc := head.encode(buf.sizebuf[:]) + if _, err = w.Write(enc); err != nil { + return err + } + } + if strpos < len(buf.str) { + // write string data after the last list header + _, err = w.Write(buf.str[strpos:]) + } + return err +} + +// Write implements io.Writer and appends b directly to the output. +func (buf *encBuffer) Write(b []byte) (int, error) { + buf.str = append(buf.str, b...) + return len(b), nil +} + +// writeBool writes b as the integer 0 (false) or 1 (true). +func (buf *encBuffer) writeBool(b bool) { + if b { + buf.str = append(buf.str, 0x01) + } else { + buf.str = append(buf.str, 0x80) + } +} + +func (buf *encBuffer) writeUint64(i uint64) { + if i == 0 { + buf.str = append(buf.str, 0x80) + } else if i < 128 { + // fits single byte + buf.str = append(buf.str, byte(i)) + } else { + s := putint(buf.sizebuf[1:], i) + buf.sizebuf[0] = 0x80 + byte(s) + buf.str = append(buf.str, buf.sizebuf[:s+1]...) + } +} + +func (buf *encBuffer) writeBytes(b []byte) { + if len(b) == 1 && b[0] <= 0x7F { + // fits single byte, no string header + buf.str = append(buf.str, b[0]) + } else { + buf.encodeStringHeader(len(b)) + buf.str = append(buf.str, b...) + } +} + +func (buf *encBuffer) writeString(s string) { + buf.writeBytes([]byte(s)) +} + +// wordBytes is the number of bytes in a big.Word +const wordBytes = (32 << (uint64(^big.Word(0)) >> 63)) / 8 + +// writeBigInt writes i as an integer. +func (w *encBuffer) writeBigInt(i *big.Int) { + bitlen := i.BitLen() + if bitlen <= 64 { + w.writeUint64(i.Uint64()) + return + } + // Integer is larger than 64 bits, encode from i.Bits(). + // The minimal byte length is bitlen rounded up to the next + // multiple of 8, divided by 8. + length := ((bitlen + 7) & -8) >> 3 + w.encodeStringHeader(length) + w.str = append(w.str, make([]byte, length)...) + index := length + buf := w.str[len(w.str)-length:] + for _, d := range i.Bits() { + for j := 0; j < wordBytes && index > 0; j++ { + index-- + buf[index] = byte(d) + d >>= 8 + } + } +} + +// list adds a new list header to the header stack. It returns the index of the header. +// Call listEnd with this index after encoding the content of the list. +func (buf *encBuffer) list() int { + buf.lheads = append(buf.lheads, listhead{offset: len(buf.str), size: buf.lhsize}) + return len(buf.lheads) - 1 +} + +func (buf *encBuffer) listEnd(index int) { + lh := &buf.lheads[index] + lh.size = buf.size() - lh.offset - lh.size + if lh.size < 56 { + buf.lhsize++ // length encoded into kind tag + } else { + buf.lhsize += 1 + intsize(uint64(lh.size)) + } +} + +func (buf *encBuffer) encode(val interface{}) error { + rval := reflect.ValueOf(val) + writer, err := cachedWriter(rval.Type()) + if err != nil { + return err + } + return writer(rval, buf) +} + +func (buf *encBuffer) encodeStringHeader(size int) { + if size < 56 { + buf.str = append(buf.str, 0x80+byte(size)) + } else { + sizesize := putint(buf.sizebuf[1:], uint64(size)) + buf.sizebuf[0] = 0xB7 + byte(sizesize) + buf.str = append(buf.str, buf.sizebuf[:sizesize+1]...) + } +} + +// encReader is the io.Reader returned by EncodeToReader. +// It releases its encbuf at EOF. +type encReader struct { + buf *encBuffer // the buffer we're reading from. this is nil when we're at EOF. + lhpos int // index of list header that we're reading + strpos int // current position in string buffer + piece []byte // next piece to be read +} + +func (r *encReader) Read(b []byte) (n int, err error) { + for { + if r.piece = r.next(); r.piece == nil { + // Put the encode buffer back into the pool at EOF when it + // is first encountered. Subsequent calls still return EOF + // as the error but the buffer is no longer valid. + if r.buf != nil { + encBufferPool.Put(r.buf) + r.buf = nil + } + return n, io.EOF + } + nn := copy(b[n:], r.piece) + n += nn + if nn < len(r.piece) { + // piece didn't fit, see you next time. + r.piece = r.piece[nn:] + return n, nil + } + r.piece = nil + } +} + +// next returns the next piece of data to be read. +// it returns nil at EOF. +func (r *encReader) next() []byte { + switch { + case r.buf == nil: + return nil + + case r.piece != nil: + // There is still data available for reading. + return r.piece + + case r.lhpos < len(r.buf.lheads): + // We're before the last list header. + head := r.buf.lheads[r.lhpos] + sizebefore := head.offset - r.strpos + if sizebefore > 0 { + // String data before header. + p := r.buf.str[r.strpos:head.offset] + r.strpos += sizebefore + return p + } + r.lhpos++ + return head.encode(r.buf.sizebuf[:]) + + case r.strpos < len(r.buf.str): + // String data at the end, after all list headers. + p := r.buf.str[r.strpos:] + r.strpos = len(r.buf.str) + return p + + default: + return nil + } +} + +func encBufferFromWriter(w io.Writer) *encBuffer { + switch w := w.(type) { + case EncoderBuffer: + return w.buf + case *EncoderBuffer: + return w.buf + case *encBuffer: + return w + default: + return nil + } +} + +// EncoderBuffer is a buffer for incremental encoding. +// +// The zero value is NOT ready for use. To get a usable buffer, +// create it using NewEncoderBuffer or call Reset. +type EncoderBuffer struct { + buf *encBuffer + dst io.Writer + + ownBuffer bool +} + +// NewEncoderBuffer creates an encoder buffer. +func NewEncoderBuffer(dst io.Writer) EncoderBuffer { + var w EncoderBuffer + w.Reset(dst) + return w +} + +// Reset truncates the buffer and sets the output destination. +func (w *EncoderBuffer) Reset(dst io.Writer) { + if w.buf != nil && !w.ownBuffer { + panic("can't Reset derived EncoderBuffer") + } + + // If the destination writer has an *encBuffer, use it. + // Note that w.ownBuffer is left false here. + if dst != nil { + if outer := encBufferFromWriter(dst); outer != nil { + *w = EncoderBuffer{outer, nil, false} + return + } + } + + // Get a fresh buffer. + if w.buf == nil { + w.buf = encBufferPool.Get().(*encBuffer) + w.ownBuffer = true + } + w.buf.reset() + w.dst = dst +} + +// Flush writes encoded RLP data to the output writer. This can only be called once. +// If you want to re-use the buffer after Flush, you must call Reset. +func (w *EncoderBuffer) Flush() error { + var err error + if w.dst != nil { + err = w.buf.writeTo(w.dst) + } + // Release the internal buffer. + if w.ownBuffer { + encBufferPool.Put(w.buf) + } + *w = EncoderBuffer{} + return err +} + +// ToBytes returns the encoded bytes. +func (w *EncoderBuffer) ToBytes() []byte { + return w.buf.makeBytes() +} + +// AppendToBytes appends the encoded bytes to dst. +func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte { + size := w.buf.size() + out := append(dst, make([]byte, size)...) + w.buf.copyTo(out[len(dst):]) + return out +} + +// Write appends b directly to the encoder output. +func (w EncoderBuffer) Write(b []byte) (int, error) { + return w.buf.Write(b) +} + +// WriteBool writes b as the integer 0 (false) or 1 (true). +func (w EncoderBuffer) WriteBool(b bool) { + w.buf.writeBool(b) +} + +// WriteUint64 encodes an unsigned integer. +func (w EncoderBuffer) WriteUint64(i uint64) { + w.buf.writeUint64(i) +} + +// WriteBigInt encodes a big.Int as an RLP string. +// Note: Unlike with Encode, the sign of i is ignored. +func (w EncoderBuffer) WriteBigInt(i *big.Int) { + w.buf.writeBigInt(i) +} + +// WriteBytes encodes b as an RLP string. +func (w EncoderBuffer) WriteBytes(b []byte) { + w.buf.writeBytes(b) +} + +// WriteBytes encodes s as an RLP string. +func (w EncoderBuffer) WriteString(s string) { + w.buf.writeString(s) +} + +// List starts a list. It returns an internal index. Call EndList with +// this index after encoding the content to finish the list. +func (w EncoderBuffer) List() int { + return w.buf.list() +} + +// ListEnd finishes the given list. +func (w EncoderBuffer) ListEnd(index int) { + w.buf.listEnd(index) +} diff --git a/rlp/encbuffer_example_test.go b/rlp/encbuffer_example_test.go new file mode 100644 index 000000000000..ee15d82a77b9 --- /dev/null +++ b/rlp/encbuffer_example_test.go @@ -0,0 +1,45 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp_test + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/rlp" +) + +func ExampleEncoderBuffer() { + var w bytes.Buffer + + // Encode [4, [5, 6]] to w. + buf := rlp.NewEncoderBuffer(&w) + l1 := buf.List() + buf.WriteUint64(4) + l2 := buf.List() + buf.WriteUint64(5) + buf.WriteUint64(6) + buf.ListEnd(l2) + buf.ListEnd(l1) + + if err := buf.Flush(); err != nil { + panic(err) + } + fmt.Printf("%X\n", w.Bytes()) + // Output: + // C404C20506 +} diff --git a/rlp/encode.go b/rlp/encode.go index 1623e97a3e9e..b96505f56dfe 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -17,11 +17,13 @@ package rlp import ( + "errors" "fmt" "io" "math/big" "reflect" - "sync" + + "github.com/ethereum/go-ethereum/rlp/internal/rlpstruct" ) var ( @@ -31,6 +33,8 @@ var ( EmptyList = []byte{0xC0} ) +var ErrNegativeBigInt = errors.New("rlp: cannot encode negative big.Int") + // Encoder is implemented by types that require custom // encoding rules or want to encode private fields. type Encoder interface { @@ -51,30 +55,29 @@ type Encoder interface { // // Please see package-level documentation of encoding rules. func Encode(w io.Writer, val interface{}) error { - if outer, ok := w.(*encbuf); ok { - // Encode was called by some type's EncodeRLP. - // Avoid copying by writing to the outer encbuf directly. - return outer.encode(val) + // Optimization: reuse *encBuffer when called by EncodeRLP. + if buf := encBufferFromWriter(w); buf != nil { + return buf.encode(val) } - eb := encbufPool.Get().(*encbuf) - defer encbufPool.Put(eb) - eb.reset() - if err := eb.encode(val); err != nil { + + buf := getEncBuffer() + defer encBufferPool.Put(buf) + if err := buf.encode(val); err != nil { return err } - return eb.toWriter(w) + return buf.writeTo(w) } // EncodeToBytes returns the RLP encoding of val. // Please see package-level documentation for the encoding rules. func EncodeToBytes(val interface{}) ([]byte, error) { - eb := encbufPool.Get().(*encbuf) - defer encbufPool.Put(eb) - eb.reset() - if err := eb.encode(val); err != nil { + buf := getEncBuffer() + defer encBufferPool.Put(buf) + + if err := buf.encode(val); err != nil { return nil, err } - return eb.toBytes(), nil + return buf.makeBytes(), nil } // EncodeToReader returns a reader from which the RLP encoding of val @@ -83,12 +86,15 @@ func EncodeToBytes(val interface{}) ([]byte, error) { // // Please see the documentation of Encode for the encoding rules. func EncodeToReader(val interface{}) (size int, r io.Reader, err error) { - eb := encbufPool.Get().(*encbuf) - eb.reset() - if err := eb.encode(val); err != nil { + buf := getEncBuffer() + if err := buf.encode(val); err != nil { + encBufferPool.Put(buf) return 0, nil, err } - return eb.size(), &encReader{buf: eb}, nil + // Note: can't put the reader back into the pool here + // because it is held by encReader. The reader puts it + // back when it has been fully consumed. + return buf.size(), &encReader{buf: buf}, nil } type listhead struct { @@ -123,207 +129,10 @@ func puthead(buf []byte, smalltag, largetag byte, size uint64) int { return sizesize + 1 } -type encbuf struct { - str []byte // string data, contains everything except list headers - lheads []listhead // all list headers - lhsize int // sum of sizes of all encoded list headers - sizebuf [9]byte // auxiliary buffer for uint encoding -} - -// encbufs are pooled. -var encbufPool = sync.Pool{ - New: func() interface{} { return new(encbuf) }, -} - -func (w *encbuf) reset() { - w.lhsize = 0 - w.str = w.str[:0] - w.lheads = w.lheads[:0] -} - -// encbuf implements io.Writer so it can be passed it into EncodeRLP. -func (w *encbuf) Write(b []byte) (int, error) { - w.str = append(w.str, b...) - return len(b), nil -} - -func (w *encbuf) encode(val interface{}) error { - rval := reflect.ValueOf(val) - writer, err := cachedWriter(rval.Type()) - if err != nil { - return err - } - return writer(rval, w) -} - -func (w *encbuf) encodeStringHeader(size int) { - if size < 56 { - w.str = append(w.str, 0x80+byte(size)) - } else { - sizesize := putint(w.sizebuf[1:], uint64(size)) - w.sizebuf[0] = 0xB7 + byte(sizesize) - w.str = append(w.str, w.sizebuf[:sizesize+1]...) - } -} - -func (w *encbuf) encodeString(b []byte) { - if len(b) == 1 && b[0] <= 0x7F { - // fits single byte, no string header - w.str = append(w.str, b[0]) - } else { - w.encodeStringHeader(len(b)) - w.str = append(w.str, b...) - } -} - -func (w *encbuf) encodeUint(i uint64) { - if i == 0 { - w.str = append(w.str, 0x80) - } else if i < 128 { - // fits single byte - w.str = append(w.str, byte(i)) - } else { - s := putint(w.sizebuf[1:], i) - w.sizebuf[0] = 0x80 + byte(s) - w.str = append(w.str, w.sizebuf[:s+1]...) - } -} - -// list adds a new list header to the header stack. It returns the index -// of the header. The caller must call listEnd with this index after encoding -// the content of the list. -func (w *encbuf) list() int { - w.lheads = append(w.lheads, listhead{offset: len(w.str), size: w.lhsize}) - return len(w.lheads) - 1 -} - -func (w *encbuf) listEnd(index int) { - lh := &w.lheads[index] - lh.size = w.size() - lh.offset - lh.size - if lh.size < 56 { - w.lhsize++ // length encoded into kind tag - } else { - w.lhsize += 1 + intsize(uint64(lh.size)) - } -} - -func (w *encbuf) size() int { - return len(w.str) + w.lhsize -} - -func (w *encbuf) toBytes() []byte { - out := make([]byte, w.size()) - strpos := 0 - pos := 0 - for _, head := range w.lheads { - // write string data before header - n := copy(out[pos:], w.str[strpos:head.offset]) - pos += n - strpos += n - // write the header - enc := head.encode(out[pos:]) - pos += len(enc) - } - // copy string data after the last list header - copy(out[pos:], w.str[strpos:]) - return out -} - -func (w *encbuf) toWriter(out io.Writer) (err error) { - strpos := 0 - for _, head := range w.lheads { - // write string data before header - if head.offset-strpos > 0 { - n, err := out.Write(w.str[strpos:head.offset]) - strpos += n - if err != nil { - return err - } - } - // write the header - enc := head.encode(w.sizebuf[:]) - if _, err = out.Write(enc); err != nil { - return err - } - } - if strpos < len(w.str) { - // write string data after the last list header - _, err = out.Write(w.str[strpos:]) - } - return err -} - -// encReader is the io.Reader returned by EncodeToReader. -// It releases its encbuf at EOF. -type encReader struct { - buf *encbuf // the buffer we're reading from. this is nil when we're at EOF. - lhpos int // index of list header that we're reading - strpos int // current position in string buffer - piece []byte // next piece to be read -} - -func (r *encReader) Read(b []byte) (n int, err error) { - for { - if r.piece = r.next(); r.piece == nil { - // Put the encode buffer back into the pool at EOF when it - // is first encountered. Subsequent calls still return EOF - // as the error but the buffer is no longer valid. - if r.buf != nil { - encbufPool.Put(r.buf) - r.buf = nil - } - return n, io.EOF - } - nn := copy(b[n:], r.piece) - n += nn - if nn < len(r.piece) { - // piece didn't fit, see you next time. - r.piece = r.piece[nn:] - return n, nil - } - r.piece = nil - } -} - -// next returns the next piece of data to be read. -// it returns nil at EOF. -func (r *encReader) next() []byte { - switch { - case r.buf == nil: - return nil - - case r.piece != nil: - // There is still data available for reading. - return r.piece - - case r.lhpos < len(r.buf.lheads): - // We're before the last list header. - head := r.buf.lheads[r.lhpos] - sizebefore := head.offset - r.strpos - if sizebefore > 0 { - // String data before header. - p := r.buf.str[r.strpos:head.offset] - r.strpos += sizebefore - return p - } - r.lhpos++ - return head.encode(r.buf.sizebuf[:]) - - case r.strpos < len(r.buf.str): - // String data at the end, after all list headers. - p := r.buf.str[r.strpos:] - r.strpos = len(r.buf.str) - return p - - default: - return nil - } -} - var encoderInterface = reflect.TypeOf(new(Encoder)).Elem() // makeWriter creates a writer function for the given type. -func makeWriter(typ reflect.Type, ts tags) (writer, error) { +func makeWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { kind := typ.Kind() switch { case typ == rawValueType: @@ -357,71 +166,45 @@ func makeWriter(typ reflect.Type, ts tags) (writer, error) { } } -func writeRawValue(val reflect.Value, w *encbuf) error { +func writeRawValue(val reflect.Value, w *encBuffer) error { w.str = append(w.str, val.Bytes()...) return nil } -func writeUint(val reflect.Value, w *encbuf) error { - w.encodeUint(val.Uint()) +func writeUint(val reflect.Value, w *encBuffer) error { + w.writeUint64(val.Uint()) return nil } -func writeBool(val reflect.Value, w *encbuf) error { - if val.Bool() { - w.str = append(w.str, 0x01) - } else { - w.str = append(w.str, 0x80) - } +func writeBool(val reflect.Value, w *encBuffer) error { + w.writeBool(val.Bool()) return nil } -func writeBigIntPtr(val reflect.Value, w *encbuf) error { +func writeBigIntPtr(val reflect.Value, w *encBuffer) error { ptr := val.Interface().(*big.Int) if ptr == nil { w.str = append(w.str, 0x80) return nil } - return writeBigInt(ptr, w) + if ptr.Sign() == -1 { + return ErrNegativeBigInt + } + w.writeBigInt(ptr) + return nil } -func writeBigIntNoPtr(val reflect.Value, w *encbuf) error { +func writeBigIntNoPtr(val reflect.Value, w *encBuffer) error { i := val.Interface().(big.Int) - return writeBigInt(&i, w) -} - -// wordBytes is the number of bytes in a big.Word -const wordBytes = (32 << (uint64(^big.Word(0)) >> 63)) / 8 - -func writeBigInt(i *big.Int, w *encbuf) error { if i.Sign() == -1 { - return fmt.Errorf("rlp: cannot encode negative *big.Int") - } - bitlen := i.BitLen() - if bitlen <= 64 { - w.encodeUint(i.Uint64()) - return nil - } - // Integer is larger than 64 bits, encode from i.Bits(). - // The minimal byte length is bitlen rounded up to the next - // multiple of 8, divided by 8. - length := ((bitlen + 7) & -8) >> 3 - w.encodeStringHeader(length) - w.str = append(w.str, make([]byte, length)...) - index := length - buf := w.str[len(w.str)-length:] - for _, d := range i.Bits() { - for j := 0; j < wordBytes && index > 0; j++ { - index-- - buf[index] = byte(d) - d >>= 8 - } + return ErrNegativeBigInt } + w.writeBigInt(&i) return nil } -func writeBytes(val reflect.Value, w *encbuf) error { - w.encodeString(val.Bytes()) +func writeBytes(val reflect.Value, w *encBuffer) error { + w.writeBytes(val.Bytes()) return nil } @@ -433,7 +216,7 @@ func makeByteArrayWriter(typ reflect.Type) writer { return writeLengthOneByteArray default: length := typ.Len() - return func(val reflect.Value, w *encbuf) error { + return func(val reflect.Value, w *encBuffer) error { if !val.CanAddr() { // Getting the byte slice of val requires it to be addressable. Make it // addressable by copying. @@ -449,12 +232,12 @@ func makeByteArrayWriter(typ reflect.Type) writer { } } -func writeLengthZeroByteArray(val reflect.Value, w *encbuf) error { +func writeLengthZeroByteArray(val reflect.Value, w *encBuffer) error { w.str = append(w.str, 0x80) return nil } -func writeLengthOneByteArray(val reflect.Value, w *encbuf) error { +func writeLengthOneByteArray(val reflect.Value, w *encBuffer) error { b := byte(val.Index(0).Uint()) if b <= 0x7f { w.str = append(w.str, b) @@ -464,7 +247,7 @@ func writeLengthOneByteArray(val reflect.Value, w *encbuf) error { return nil } -func writeString(val reflect.Value, w *encbuf) error { +func writeString(val reflect.Value, w *encBuffer) error { s := val.String() if len(s) == 1 && s[0] <= 0x7f { // fits single byte, no string header @@ -476,7 +259,7 @@ func writeString(val reflect.Value, w *encbuf) error { return nil } -func writeInterface(val reflect.Value, w *encbuf) error { +func writeInterface(val reflect.Value, w *encBuffer) error { if val.IsNil() { // Write empty list. This is consistent with the previous RLP // encoder that we had and should therefore avoid any @@ -492,17 +275,17 @@ func writeInterface(val reflect.Value, w *encbuf) error { return writer(eval, w) } -func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) +func makeSliceWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{}) if etypeinfo.writerErr != nil { return nil, etypeinfo.writerErr } var wfn writer - if ts.tail { + if ts.Tail { // This is for struct tail slices. // w.list is not called for them. - wfn = func(val reflect.Value, w *encbuf) error { + wfn = func(val reflect.Value, w *encBuffer) error { vlen := val.Len() for i := 0; i < vlen; i++ { if err := etypeinfo.writer(val.Index(i), w); err != nil { @@ -513,7 +296,7 @@ func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { } } else { // This is for regular slices and arrays. - wfn = func(val reflect.Value, w *encbuf) error { + wfn = func(val reflect.Value, w *encBuffer) error { vlen := val.Len() if vlen == 0 { w.str = append(w.str, 0xC0) @@ -547,7 +330,7 @@ func makeStructWriter(typ reflect.Type) (writer, error) { firstOptionalField := firstOptionalField(fields) if firstOptionalField == len(fields) { // This is the writer function for structs without any optional fields. - writer = func(val reflect.Value, w *encbuf) error { + writer = func(val reflect.Value, w *encBuffer) error { lh := w.list() for _, f := range fields { if err := f.info.writer(val.Field(f.index), w); err != nil { @@ -560,7 +343,7 @@ func makeStructWriter(typ reflect.Type) (writer, error) { } else { // If there are any "optional" fields, the writer needs to perform additional // checks to determine the output list length. - writer = func(val reflect.Value, w *encbuf) error { + writer = func(val reflect.Value, w *encBuffer) error { lastField := len(fields) - 1 for ; lastField >= firstOptionalField; lastField-- { if !val.Field(fields[lastField].index).IsZero() { @@ -580,33 +363,18 @@ func makeStructWriter(typ reflect.Type) (writer, error) { return writer, nil } -// nilEncoding returns the encoded value of a nil pointer. -func nilEncoding(typ reflect.Type, ts tags) uint8 { - var nilKind Kind - if ts.nilOK { - nilKind = ts.nilKind // use struct tag if provided - } else { - nilKind = defaultNilKind(typ.Elem()) +func makePtrWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + nilEncoding := byte(0xC0) + if typeNilKind(typ.Elem(), ts) == String { + nilEncoding = 0x80 } - switch nilKind { - case String: - return 0x80 - case List: - return 0xC0 - default: - panic(fmt.Errorf("rlp: invalid nil kind %d", nilKind)) - } -} - -func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{}) if etypeinfo.writerErr != nil { return nil, etypeinfo.writerErr } - nilEncoding := nilEncoding(typ, ts) - writer := func(val reflect.Value, w *encbuf) error { + writer := func(val reflect.Value, w *encBuffer) error { if ev := val.Elem(); ev.IsValid() { return etypeinfo.writer(ev, w) } @@ -618,11 +386,11 @@ func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { func makeEncoderWriter(typ reflect.Type) writer { if typ.Implements(encoderInterface) { - return func(val reflect.Value, w *encbuf) error { + return func(val reflect.Value, w *encBuffer) error { return val.Interface().(Encoder).EncodeRLP(w) } } - w := func(val reflect.Value, w *encbuf) error { + w := func(val reflect.Value, w *encBuffer) error { if !val.CanAddr() { // package json simply doesn't call MarshalJSON for this case, but encodes the // value as if it didn't implement the interface. We don't want to handle it that diff --git a/rlp/encode_test.go b/rlp/encode_test.go index a63743440d39..1d715e3776c3 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -145,7 +145,8 @@ var encTests = []encTest{ {val: *big.NewInt(0xFFFFFF), output: "83FFFFFF"}, // negative ints are not supported - {val: big.NewInt(-1), error: "rlp: cannot encode negative *big.Int"}, + {val: big.NewInt(-1), error: "rlp: cannot encode negative big.Int"}, + {val: *big.NewInt(-1), error: "rlp: cannot encode negative big.Int"}, // byte arrays {val: [0]byte{}, output: "80"}, @@ -398,6 +399,21 @@ func TestEncodeToBytes(t *testing.T) { runEncTests(t, EncodeToBytes) } +func TestEncodeAppendToBytes(t *testing.T) { + buffer := make([]byte, 20) + runEncTests(t, func(val interface{}) ([]byte, error) { + w := NewEncoderBuffer(nil) + defer w.Flush() + + err := Encode(w, val) + if err != nil { + return nil, err + } + output := w.AppendToBytes(buffer[:0]) + return output, nil + }) +} + func TestEncodeToReader(t *testing.T) { runEncTests(t, func(val interface{}) ([]byte, error) { _, r, err := EncodeToReader(val) diff --git a/rlp/encoder_example_test.go b/rlp/encoder_example_test.go index 42c1c5c89064..4cd3cb867375 100644 --- a/rlp/encoder_example_test.go +++ b/rlp/encoder_example_test.go @@ -14,11 +14,13 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package rlp +package rlp_test import ( "fmt" "io" + + "github.com/ethereum/go-ethereum/rlp" ) type MyCoolType struct { @@ -28,16 +30,16 @@ type MyCoolType struct { // EncodeRLP writes x as RLP list [a, b] that omits the Name field. func (x *MyCoolType) EncodeRLP(w io.Writer) (err error) { - return Encode(w, []uint{x.a, x.b}) + return rlp.Encode(w, []uint{x.a, x.b}) } func ExampleEncoder() { var t *MyCoolType // t is nil pointer to MyCoolType - bytes, _ := EncodeToBytes(t) + bytes, _ := rlp.EncodeToBytes(t) fmt.Printf("%v → %X\n", t, bytes) t = &MyCoolType{Name: "foobar", a: 5, b: 6} - bytes, _ = EncodeToBytes(t) + bytes, _ = rlp.EncodeToBytes(t) fmt.Printf("%v → %X\n", t, bytes) // Output: diff --git a/rlp/internal/rlpstruct/rlpstruct.go b/rlp/internal/rlpstruct/rlpstruct.go new file mode 100644 index 000000000000..1ebaa960e38c --- /dev/null +++ b/rlp/internal/rlpstruct/rlpstruct.go @@ -0,0 +1,213 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package rlpstruct implements struct processing for RLP encoding/decoding. +// +// In particular, this package handles all rules around field filtering, +// struct tags and nil value determination. +package rlpstruct + +import ( + "fmt" + "reflect" + "strings" +) + +// Field represents a struct field. +type Field struct { + Name string + Index int + Exported bool + Type Type + Tag string +} + +// Type represents the attributes of a Go type. +type Type struct { + Name string + Kind reflect.Kind + IsEncoder bool // whether type implements rlp.Encoder + IsDecoder bool // whether type implements rlp.Decoder + Elem *Type // non-nil for Kind values of Ptr, Slice, Array +} + +// defaultNilValue determines whether a nil pointer to t encodes/decodes +// as an empty string or empty list. +func (t Type) DefaultNilValue() NilKind { + k := t.Kind + if isUint(k) || k == reflect.String || k == reflect.Bool || isByteArray(t) { + return NilKindString + } + return NilKindList +} + +// NilKind is the RLP value encoded in place of nil pointers. +type NilKind uint8 + +const ( + NilKindString NilKind = 0x80 + NilKindList NilKind = 0xC0 +) + +// Tags represents struct tags. +type Tags struct { + // rlp:"nil" controls whether empty input results in a nil pointer. + // nilKind is the kind of empty value allowed for the field. + NilKind NilKind + NilOK bool + + // rlp:"optional" allows for a field to be missing in the input list. + // If this is set, all subsequent fields must also be optional. + Optional bool + + // rlp:"tail" controls whether this field swallows additional list elements. It can + // only be set for the last field, which must be of slice type. + Tail bool + + // rlp:"-" ignores fields. + Ignored bool +} + +// TagError is raised for invalid struct tags. +type TagError struct { + StructType string + + // These are set by this package. + Field string + Tag string + Err string +} + +func (e TagError) Error() string { + field := "field " + e.Field + if e.StructType != "" { + field = e.StructType + "." + e.Field + } + return fmt.Sprintf("rlp: invalid struct tag %q for %s (%s)", e.Tag, field, e.Err) +} + +// ProcessFields filters the given struct fields, returning only fields +// that should be considered for encoding/decoding. +func ProcessFields(allFields []Field) ([]Field, []Tags, error) { + lastPublic := lastPublicField(allFields) + + // Gather all exported fields and their tags. + var fields []Field + var tags []Tags + for _, field := range allFields { + if !field.Exported { + continue + } + ts, err := parseTag(field, lastPublic) + if err != nil { + return nil, nil, err + } + if ts.Ignored { + continue + } + fields = append(fields, field) + tags = append(tags, ts) + } + + // Verify optional field consistency. If any optional field exists, + // all fields after it must also be optional. Note: optional + tail + // is supported. + var anyOptional bool + var firstOptionalName string + for i, ts := range tags { + name := fields[i].Name + if ts.Optional || ts.Tail { + if !anyOptional { + firstOptionalName = name + } + anyOptional = true + } else { + if anyOptional { + msg := fmt.Sprintf("must be optional because preceding field %q is optional", firstOptionalName) + return nil, nil, TagError{Field: name, Err: msg} + } + } + } + return fields, tags, nil +} + +func parseTag(field Field, lastPublic int) (Tags, error) { + name := field.Name + tag := reflect.StructTag(field.Tag) + var ts Tags + for _, t := range strings.Split(tag.Get("rlp"), ",") { + switch t = strings.TrimSpace(t); t { + case "": + // empty tag is allowed for some reason + case "-": + ts.Ignored = true + case "nil", "nilString", "nilList": + ts.NilOK = true + if field.Type.Kind != reflect.Ptr { + return ts, TagError{Field: name, Tag: t, Err: "field is not a pointer"} + } + switch t { + case "nil": + ts.NilKind = field.Type.Elem.DefaultNilValue() + case "nilString": + ts.NilKind = NilKindString + case "nilList": + ts.NilKind = NilKindList + } + case "optional": + ts.Optional = true + if ts.Tail { + return ts, TagError{Field: name, Tag: t, Err: `also has "tail" tag`} + } + case "tail": + ts.Tail = true + if field.Index != lastPublic { + return ts, TagError{Field: name, Tag: t, Err: "must be on last field"} + } + if ts.Optional { + return ts, TagError{Field: name, Tag: t, Err: `also has "optional" tag`} + } + if field.Type.Kind != reflect.Slice { + return ts, TagError{Field: name, Tag: t, Err: "field type is not slice"} + } + default: + return ts, TagError{Field: name, Tag: t, Err: "unknown tag"} + } + } + return ts, nil +} + +func lastPublicField(fields []Field) int { + last := 0 + for _, f := range fields { + if f.Exported { + last = f.Index + } + } + return last +} + +func isUint(k reflect.Kind) bool { + return k >= reflect.Uint && k <= reflect.Uintptr +} + +func isByte(typ Type) bool { + return typ.Kind == reflect.Uint8 && !typ.IsEncoder +} + +func isByteArray(typ Type) bool { + return (typ.Kind == reflect.Slice || typ.Kind == reflect.Array) && isByte(*typ.Elem) +} diff --git a/rlp/rlpgen/gen.go b/rlp/rlpgen/gen.go new file mode 100644 index 000000000000..b36b26947834 --- /dev/null +++ b/rlp/rlpgen/gen.go @@ -0,0 +1,735 @@ +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/types" + "sort" + + "github.com/ethereum/go-ethereum/rlp/internal/rlpstruct" +) + +// buildContext keeps the data needed for make*Op. +type buildContext struct { + topType *types.Named // the type we're creating methods for + + encoderIface *types.Interface + decoderIface *types.Interface + rawValueType *types.Named + + typeToStructCache map[types.Type]*rlpstruct.Type +} + +func newBuildContext(packageRLP *types.Package) *buildContext { + enc := packageRLP.Scope().Lookup("Encoder").Type().Underlying() + dec := packageRLP.Scope().Lookup("Decoder").Type().Underlying() + rawv := packageRLP.Scope().Lookup("RawValue").Type() + return &buildContext{ + typeToStructCache: make(map[types.Type]*rlpstruct.Type), + encoderIface: enc.(*types.Interface), + decoderIface: dec.(*types.Interface), + rawValueType: rawv.(*types.Named), + } +} + +func (bctx *buildContext) isEncoder(typ types.Type) bool { + return types.Implements(typ, bctx.encoderIface) +} + +func (bctx *buildContext) isDecoder(typ types.Type) bool { + return types.Implements(typ, bctx.decoderIface) +} + +// typeToStructType converts typ to rlpstruct.Type. +func (bctx *buildContext) typeToStructType(typ types.Type) *rlpstruct.Type { + if prev := bctx.typeToStructCache[typ]; prev != nil { + return prev // short-circuit for recursive types. + } + + // Resolve named types to their underlying type, but keep the name. + name := types.TypeString(typ, nil) + for { + utype := typ.Underlying() + if utype == typ { + break + } + typ = utype + } + + // Create the type and store it in cache. + t := &rlpstruct.Type{ + Name: name, + Kind: typeReflectKind(typ), + IsEncoder: bctx.isEncoder(typ), + IsDecoder: bctx.isDecoder(typ), + } + bctx.typeToStructCache[typ] = t + + // Assign element type. + switch typ.(type) { + case *types.Array, *types.Slice, *types.Pointer: + etype := typ.(interface{ Elem() types.Type }).Elem() + t.Elem = bctx.typeToStructType(etype) + } + return t +} + +// genContext is passed to the gen* methods of op when generating +// the output code. It tracks packages to be imported by the output +// file and assigns unique names of temporary variables. +type genContext struct { + inPackage *types.Package + imports map[string]struct{} + tempCounter int +} + +func newGenContext(inPackage *types.Package) *genContext { + return &genContext{ + inPackage: inPackage, + imports: make(map[string]struct{}), + } +} + +func (ctx *genContext) temp() string { + v := fmt.Sprintf("_tmp%d", ctx.tempCounter) + ctx.tempCounter++ + return v +} + +func (ctx *genContext) resetTemp() { + ctx.tempCounter = 0 +} + +func (ctx *genContext) addImport(path string) { + if path == ctx.inPackage.Path() { + return // avoid importing the package that we're generating in. + } + // TODO: renaming? + ctx.imports[path] = struct{}{} +} + +// importsList returns all packages that need to be imported. +func (ctx *genContext) importsList() []string { + imp := make([]string, 0, len(ctx.imports)) + for k := range ctx.imports { + imp = append(imp, k) + } + sort.Strings(imp) + return imp +} + +// qualify is the types.Qualifier used for printing types. +func (ctx *genContext) qualify(pkg *types.Package) string { + if pkg.Path() == ctx.inPackage.Path() { + return "" + } + ctx.addImport(pkg.Path()) + // TODO: renaming? + return pkg.Name() +} + +type op interface { + // genWrite creates the encoder. The generated code should write v, + // which is any Go expression, to the rlp.EncoderBuffer 'w'. + genWrite(ctx *genContext, v string) string + + // genDecode creates the decoder. The generated code should read + // a value from the rlp.Stream 'dec' and store it to dst. + genDecode(ctx *genContext) (string, string) +} + +// basicOp handles basic types bool, uint*, string. +type basicOp struct { + typ types.Type + writeMethod string // calle write the value + writeArgType types.Type // parameter type of writeMethod + decMethod string + decResultType types.Type // return type of decMethod + decUseBitSize bool // if true, result bit size is appended to decMethod +} + +func (*buildContext) makeBasicOp(typ *types.Basic) (op, error) { + op := basicOp{typ: typ} + kind := typ.Kind() + switch { + case kind == types.Bool: + op.writeMethod = "WriteBool" + op.writeArgType = types.Typ[types.Bool] + op.decMethod = "Bool" + op.decResultType = types.Typ[types.Bool] + case kind >= types.Uint8 && kind <= types.Uint64: + op.writeMethod = "WriteUint64" + op.writeArgType = types.Typ[types.Uint64] + op.decMethod = "Uint" + op.decResultType = typ + op.decUseBitSize = true + case kind == types.String: + op.writeMethod = "WriteString" + op.writeArgType = types.Typ[types.String] + op.decMethod = "String" + op.decResultType = types.Typ[types.String] + default: + return nil, fmt.Errorf("unhandled basic type: %v", typ) + } + return op, nil +} + +func (*buildContext) makeByteSliceOp(typ *types.Slice) op { + if !isByte(typ.Elem()) { + panic("non-byte slice type in makeByteSliceOp") + } + bslice := types.NewSlice(types.Typ[types.Uint8]) + return basicOp{ + typ: typ, + writeMethod: "WriteBytes", + writeArgType: bslice, + decMethod: "Bytes", + decResultType: bslice, + } +} + +func (bctx *buildContext) makeRawValueOp() op { + bslice := types.NewSlice(types.Typ[types.Uint8]) + return basicOp{ + typ: bctx.rawValueType, + writeMethod: "Write", + writeArgType: bslice, + decMethod: "Raw", + decResultType: bslice, + } +} + +func (op basicOp) writeNeedsConversion() bool { + return !types.AssignableTo(op.typ, op.writeArgType) +} + +func (op basicOp) decodeNeedsConversion() bool { + return !types.AssignableTo(op.decResultType, op.typ) +} + +func (op basicOp) genWrite(ctx *genContext, v string) string { + if op.writeNeedsConversion() { + v = fmt.Sprintf("%s(%s)", op.writeArgType, v) + } + return fmt.Sprintf("w.%s(%s)\n", op.writeMethod, v) +} + +func (op basicOp) genDecode(ctx *genContext) (string, string) { + var ( + resultV = ctx.temp() + result = resultV + method = op.decMethod + ) + if op.decUseBitSize { + // Note: For now, this only works for platform-independent integer + // sizes. makeBasicOp forbids the platform-dependent types. + var sizes types.StdSizes + method = fmt.Sprintf("%s%d", op.decMethod, sizes.Sizeof(op.typ)*8) + } + + // Call the decoder method. + var b bytes.Buffer + fmt.Fprintf(&b, "%s, err := dec.%s()\n", resultV, method) + fmt.Fprintf(&b, "if err != nil { return err }\n") + if op.decodeNeedsConversion() { + conv := ctx.temp() + fmt.Fprintf(&b, "%s := %s(%s)\n", conv, types.TypeString(op.typ, ctx.qualify), resultV) + result = conv + } + return result, b.String() +} + +// byteArrayOp handles [...]byte. +type byteArrayOp struct { + typ types.Type + name types.Type // name != typ for named byte array types (e.g. common.Address) +} + +func (bctx *buildContext) makeByteArrayOp(name *types.Named, typ *types.Array) byteArrayOp { + nt := types.Type(name) + if name == nil { + nt = typ + } + return byteArrayOp{typ, nt} +} + +func (op byteArrayOp) genWrite(ctx *genContext, v string) string { + return fmt.Sprintf("w.WriteBytes(%s[:])\n", v) +} + +func (op byteArrayOp) genDecode(ctx *genContext) (string, string) { + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, types.TypeString(op.name, ctx.qualify)) + fmt.Fprintf(&b, "if err := dec.ReadBytes(%s[:]); err != nil { return err }\n", resultV) + return resultV, b.String() +} + +// bigIntNoPtrOp handles non-pointer big.Int. +// This exists because big.Int has it's own decoder operation on rlp.Stream, +// but the decode method returns *big.Int, so it needs to be dereferenced. +type bigIntOp struct { + pointer bool +} + +func (op bigIntOp) genWrite(ctx *genContext, v string) string { + var b bytes.Buffer + + fmt.Fprintf(&b, "if %s.Sign() == -1 {\n", v) + fmt.Fprintf(&b, " return rlp.ErrNegativeBigInt\n") + fmt.Fprintf(&b, "}\n") + dst := v + if !op.pointer { + dst = "&" + v + } + fmt.Fprintf(&b, "w.WriteBigInt(%s)\n", dst) + + // Wrap with nil check. + if op.pointer { + code := b.String() + b.Reset() + fmt.Fprintf(&b, "if %s == nil {\n", v) + fmt.Fprintf(&b, " w.Write(rlp.EmptyString)") + fmt.Fprintf(&b, "} else {\n") + fmt.Fprint(&b, code) + fmt.Fprintf(&b, "}\n") + } + + return b.String() +} + +func (op bigIntOp) genDecode(ctx *genContext) (string, string) { + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "%s, err := dec.BigInt()\n", resultV) + fmt.Fprintf(&b, "if err != nil { return err }\n") + + result := resultV + if !op.pointer { + result = "(*" + resultV + ")" + } + return result, b.String() +} + +// encoderDecoderOp handles rlp.Encoder and rlp.Decoder. +// In order to be used with this, the type must implement both interfaces. +// This restriction may be lifted in the future by creating separate ops for +// encoding and decoding. +type encoderDecoderOp struct { + typ types.Type +} + +func (op encoderDecoderOp) genWrite(ctx *genContext, v string) string { + return fmt.Sprintf("if err := %s.EncodeRLP(w); err != nil { return err }\n", v) +} + +func (op encoderDecoderOp) genDecode(ctx *genContext) (string, string) { + // DecodeRLP must have pointer receiver, and this is verified in makeOp. + etyp := op.typ.(*types.Pointer).Elem() + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "%s := new(%s)\n", resultV, types.TypeString(etyp, ctx.qualify)) + fmt.Fprintf(&b, "if err := %s.DecodeRLP(dec); err != nil { return err }\n", resultV) + return resultV, b.String() +} + +// ptrOp handles pointer types. +type ptrOp struct { + elemTyp types.Type + elem op + nilOK bool + nilValue rlpstruct.NilKind +} + +func (bctx *buildContext) makePtrOp(elemTyp types.Type, tags rlpstruct.Tags) (op, error) { + elemOp, err := bctx.makeOp(nil, elemTyp, rlpstruct.Tags{}) + if err != nil { + return nil, err + } + op := ptrOp{elemTyp: elemTyp, elem: elemOp} + + // Determine nil value. + if tags.NilOK { + op.nilOK = true + op.nilValue = tags.NilKind + } else { + styp := bctx.typeToStructType(elemTyp) + op.nilValue = styp.DefaultNilValue() + } + return op, nil +} + +func (op ptrOp) genWrite(ctx *genContext, v string) string { + // Note: in writer functions, accesses to v are read-only, i.e. v is any Go + // expression. To make all accesses work through the pointer, we substitute + // v with (*v). This is required for most accesses including `v`, `call(v)`, + // and `v[index]` on slices. + // + // For `v.field` and `v[:]` on arrays, the dereference operation is not required. + var vv string + _, isStruct := op.elem.(structOp) + _, isByteArray := op.elem.(byteArrayOp) + if isStruct || isByteArray { + vv = v + } else { + vv = fmt.Sprintf("(*%s)", v) + } + + var b bytes.Buffer + fmt.Fprintf(&b, "if %s == nil {\n", v) + fmt.Fprintf(&b, " w.Write([]byte{0x%X})\n", op.nilValue) + fmt.Fprintf(&b, "} else {\n") + fmt.Fprintf(&b, " %s", op.elem.genWrite(ctx, vv)) + fmt.Fprintf(&b, "}\n") + return b.String() +} + +func (op ptrOp) genDecode(ctx *genContext) (string, string) { + result, code := op.elem.genDecode(ctx) + if !op.nilOK { + // If nil pointers are not allowed, we can just decode the element. + return "&" + result, code + } + + // nil is allowed, so check the kind and size first. + // If size is zero and kind matches the nilKind of the type, + // the value decodes as a nil pointer. + var ( + resultV = ctx.temp() + kindV = ctx.temp() + sizeV = ctx.temp() + wantKind string + ) + if op.nilValue == rlpstruct.NilKindList { + wantKind = "rlp.List" + } else { + wantKind = "rlp.String" + } + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, types.TypeString(types.NewPointer(op.elemTyp), ctx.qualify)) + fmt.Fprintf(&b, "if %s, %s, err := dec.Kind(); err != nil {\n", kindV, sizeV) + fmt.Fprintf(&b, " return err\n") + fmt.Fprintf(&b, "} else if %s != 0 || %s != %s {\n", sizeV, kindV, wantKind) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, " %s = &%s\n", resultV, result) + fmt.Fprintf(&b, "}\n") + return resultV, b.String() +} + +// structOp handles struct types. +type structOp struct { + named *types.Named + typ *types.Struct + fields []*structField + optionalFields []*structField +} + +type structField struct { + name string + typ types.Type + elem op +} + +func (bctx *buildContext) makeStructOp(named *types.Named, typ *types.Struct) (op, error) { + // Convert fields to []rlpstruct.Field. + var allStructFields []rlpstruct.Field + for i := 0; i < typ.NumFields(); i++ { + f := typ.Field(i) + allStructFields = append(allStructFields, rlpstruct.Field{ + Name: f.Name(), + Exported: f.Exported(), + Index: i, + Tag: typ.Tag(i), + Type: *bctx.typeToStructType(f.Type()), + }) + } + + // Filter/validate fields. + fields, tags, err := rlpstruct.ProcessFields(allStructFields) + if err != nil { + return nil, err + } + + // Create field ops. + var op = structOp{named: named, typ: typ} + for i, field := range fields { + // Advanced struct tags are not supported yet. + tag := tags[i] + if err := checkUnsupportedTags(field.Name, tag); err != nil { + return nil, err + } + typ := typ.Field(field.Index).Type() + elem, err := bctx.makeOp(nil, typ, tags[i]) + if err != nil { + return nil, fmt.Errorf("field %s: %v", field.Name, err) + } + f := &structField{name: field.Name, typ: typ, elem: elem} + if tag.Optional { + op.optionalFields = append(op.optionalFields, f) + } else { + op.fields = append(op.fields, f) + } + } + return op, nil +} + +func checkUnsupportedTags(field string, tag rlpstruct.Tags) error { + if tag.Tail { + return fmt.Errorf(`field %s has unsupported struct tag "tail"`, field) + } + return nil +} + +func (op structOp) genWrite(ctx *genContext, v string) string { + var b bytes.Buffer + var listMarker = ctx.temp() + fmt.Fprintf(&b, "%s := w.List()\n", listMarker) + for _, field := range op.fields { + selector := v + "." + field.name + fmt.Fprint(&b, field.elem.genWrite(ctx, selector)) + } + op.writeOptionalFields(&b, ctx, v) + fmt.Fprintf(&b, "w.ListEnd(%s)\n", listMarker) + return b.String() +} + +func (op structOp) writeOptionalFields(b *bytes.Buffer, ctx *genContext, v string) { + if len(op.optionalFields) == 0 { + return + } + // First check zero-ness of all optional fields. + var zeroV = make([]string, len(op.optionalFields)) + for i, field := range op.optionalFields { + selector := v + "." + field.name + zeroV[i] = ctx.temp() + fmt.Fprintf(b, "%s := %s\n", zeroV[i], nonZeroCheck(selector, field.typ, ctx.qualify)) + } + // Now write the fields. + for i, field := range op.optionalFields { + selector := v + "." + field.name + cond := "" + for j := i; j < len(op.optionalFields); j++ { + if j > i { + cond += " || " + } + cond += zeroV[j] + } + fmt.Fprintf(b, "if %s {\n", cond) + fmt.Fprint(b, field.elem.genWrite(ctx, selector)) + fmt.Fprintf(b, "}\n") + } +} + +func (op structOp) genDecode(ctx *genContext) (string, string) { + // Get the string representation of the type. + // Here, named types are handled separately because the output + // would contain a copy of the struct definition otherwise. + var typeName string + if op.named != nil { + typeName = types.TypeString(op.named, ctx.qualify) + } else { + typeName = types.TypeString(op.typ, ctx.qualify) + } + + // Create struct object. + var resultV = ctx.temp() + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, typeName) + + // Decode fields. + fmt.Fprintf(&b, "{\n") + fmt.Fprintf(&b, "if _, err := dec.List(); err != nil { return err }\n") + for _, field := range op.fields { + result, code := field.elem.genDecode(ctx) + fmt.Fprintf(&b, "// %s:\n", field.name) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, "%s.%s = %s\n", resultV, field.name, result) + } + op.decodeOptionalFields(&b, ctx, resultV) + fmt.Fprintf(&b, "if err := dec.ListEnd(); err != nil { return err }\n") + fmt.Fprintf(&b, "}\n") + return resultV, b.String() +} + +func (op structOp) decodeOptionalFields(b *bytes.Buffer, ctx *genContext, resultV string) { + var suffix bytes.Buffer + for _, field := range op.optionalFields { + result, code := field.elem.genDecode(ctx) + fmt.Fprintf(b, "// %s:\n", field.name) + fmt.Fprintf(b, "if dec.MoreDataInList() {\n") + fmt.Fprint(b, code) + fmt.Fprintf(b, "%s.%s = %s\n", resultV, field.name, result) + fmt.Fprintf(&suffix, "}\n") + } + suffix.WriteTo(b) +} + +// sliceOp handles slice types. +type sliceOp struct { + typ *types.Slice + elemOp op +} + +func (bctx *buildContext) makeSliceOp(typ *types.Slice) (op, error) { + elemOp, err := bctx.makeOp(nil, typ.Elem(), rlpstruct.Tags{}) + if err != nil { + return nil, err + } + return sliceOp{typ: typ, elemOp: elemOp}, nil +} + +func (op sliceOp) genWrite(ctx *genContext, v string) string { + var ( + listMarker = ctx.temp() // holds return value of w.List() + iterElemV = ctx.temp() // iteration variable + elemCode = op.elemOp.genWrite(ctx, iterElemV) + ) + + var b bytes.Buffer + fmt.Fprintf(&b, "%s := w.List()\n", listMarker) + fmt.Fprintf(&b, "for _, %s := range %s {\n", iterElemV, v) + fmt.Fprint(&b, elemCode) + fmt.Fprintf(&b, "}\n") + fmt.Fprintf(&b, "w.ListEnd(%s)\n", listMarker) + return b.String() +} + +func (op sliceOp) genDecode(ctx *genContext) (string, string) { + var sliceV = ctx.temp() // holds the output slice + elemResult, elemCode := op.elemOp.genDecode(ctx) + + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", sliceV, types.TypeString(op.typ, ctx.qualify)) + fmt.Fprintf(&b, "if _, err := dec.List(); err != nil { return err }\n") + fmt.Fprintf(&b, "for dec.MoreDataInList() {\n") + fmt.Fprintf(&b, " %s", elemCode) + fmt.Fprintf(&b, " %s = append(%s, %s)\n", sliceV, sliceV, elemResult) + fmt.Fprintf(&b, "}\n") + fmt.Fprintf(&b, "if err := dec.ListEnd(); err != nil { return err }\n") + return sliceV, b.String() +} + +func (bctx *buildContext) makeOp(name *types.Named, typ types.Type, tags rlpstruct.Tags) (op, error) { + switch typ := typ.(type) { + case *types.Named: + if isBigInt(typ) { + return bigIntOp{}, nil + } + if typ == bctx.rawValueType { + return bctx.makeRawValueOp(), nil + } + if bctx.isDecoder(typ) { + return nil, fmt.Errorf("type %v implements rlp.Decoder with non-pointer receiver", typ) + } + // TODO: same check for encoder? + return bctx.makeOp(typ, typ.Underlying(), tags) + case *types.Pointer: + if isBigInt(typ.Elem()) { + return bigIntOp{pointer: true}, nil + } + // Encoder/Decoder interfaces. + if bctx.isEncoder(typ) { + if bctx.isDecoder(typ) { + return encoderDecoderOp{typ}, nil + } + return nil, fmt.Errorf("type %v implements rlp.Encoder but not rlp.Decoder", typ) + } + if bctx.isDecoder(typ) { + return nil, fmt.Errorf("type %v implements rlp.Decoder but not rlp.Encoder", typ) + } + // Default pointer handling. + return bctx.makePtrOp(typ.Elem(), tags) + case *types.Basic: + return bctx.makeBasicOp(typ) + case *types.Struct: + return bctx.makeStructOp(name, typ) + case *types.Slice: + etyp := typ.Elem() + if isByte(etyp) && !bctx.isEncoder(etyp) { + return bctx.makeByteSliceOp(typ), nil + } + return bctx.makeSliceOp(typ) + case *types.Array: + etyp := typ.Elem() + if isByte(etyp) && !bctx.isEncoder(etyp) { + return bctx.makeByteArrayOp(name, typ), nil + } + return nil, fmt.Errorf("unhandled array type: %v", typ) + default: + return nil, fmt.Errorf("unhandled type: %v", typ) + } +} + +// generateDecoder generates the DecodeRLP method on 'typ'. +func generateDecoder(ctx *genContext, typ string, op op) []byte { + ctx.resetTemp() + ctx.addImport(pathOfPackageRLP) + + result, code := op.genDecode(ctx) + var b bytes.Buffer + fmt.Fprintf(&b, "func (obj *%s) DecodeRLP(dec *rlp.Stream) error {\n", typ) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, " *obj = %s\n", result) + fmt.Fprintf(&b, " return nil\n") + fmt.Fprintf(&b, "}\n") + return b.Bytes() +} + +// generateEncoder generates the EncodeRLP method on 'typ'. +func generateEncoder(ctx *genContext, typ string, op op) []byte { + ctx.resetTemp() + ctx.addImport("io") + ctx.addImport(pathOfPackageRLP) + + var b bytes.Buffer + fmt.Fprintf(&b, "func (obj *%s) EncodeRLP(_w io.Writer) error {\n", typ) + fmt.Fprintf(&b, " w := rlp.NewEncoderBuffer(_w)\n") + fmt.Fprint(&b, op.genWrite(ctx, "obj")) + fmt.Fprintf(&b, " return w.Flush()\n") + fmt.Fprintf(&b, "}\n") + return b.Bytes() +} + +func (bctx *buildContext) generate(typ *types.Named, encoder, decoder bool) ([]byte, error) { + bctx.topType = typ + + pkg := typ.Obj().Pkg() + op, err := bctx.makeOp(nil, typ, rlpstruct.Tags{}) + if err != nil { + return nil, err + } + + var ( + ctx = newGenContext(pkg) + encSource []byte + decSource []byte + ) + if encoder { + encSource = generateEncoder(ctx, typ.Obj().Name(), op) + } + if decoder { + decSource = generateDecoder(ctx, typ.Obj().Name(), op) + } + + var b bytes.Buffer + fmt.Fprintf(&b, "package %s\n\n", pkg.Name()) + for _, imp := range ctx.importsList() { + fmt.Fprintf(&b, "import %q\n", imp) + } + if encoder { + fmt.Fprintln(&b) + b.Write(encSource) + } + if decoder { + fmt.Fprintln(&b) + b.Write(decSource) + } + + source := b.Bytes() + // fmt.Println(string(source)) + return format.Source(source) +} diff --git a/rlp/rlpgen/gen_test.go b/rlp/rlpgen/gen_test.go new file mode 100644 index 000000000000..9940db188da9 --- /dev/null +++ b/rlp/rlpgen/gen_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +// Package RLP is loaded only once and reused for all tests. +var ( + testFset = token.NewFileSet() + testImporter = importer.ForCompiler(testFset, "source", nil).(types.ImporterFrom) + testPackageRLP *types.Package +) + +func init() { + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + testPackageRLP, err = testImporter.ImportFrom(pathOfPackageRLP, cwd, 0) + if err != nil { + panic(fmt.Errorf("can't load package RLP: %v", err)) + } +} + +var tests = []string{"uints", "nil", "rawvalue", "optional", "bigint"} + +func TestOutput(t *testing.T) { + for _, test := range tests { + test := test + t.Run(test, func(t *testing.T) { + inputFile := filepath.Join("testdata", test+".in.txt") + outputFile := filepath.Join("testdata", test+".out.txt") + bctx, typ, err := loadTestSource(inputFile, "Test") + if err != nil { + t.Fatal("error loading test source:", err) + } + output, err := bctx.generate(typ, true, true) + if err != nil { + t.Fatal("error in generate:", err) + } + + // Set this environment variable to regenerate the test outputs. + if os.Getenv("WRITE_TEST_FILES") != "" { + ioutil.WriteFile(outputFile, output, 0644) + } + + // Check if output matches. + wantOutput, err := ioutil.ReadFile(outputFile) + if err != nil { + t.Fatal("error loading expected test output:", err) + } + if !bytes.Equal(output, wantOutput) { + t.Fatal("output mismatch:\n", string(output)) + } + }) + } +} + +func loadTestSource(file string, typeName string) (*buildContext, *types.Named, error) { + // Load the test input. + content, err := ioutil.ReadFile(file) + if err != nil { + return nil, nil, err + } + f, err := parser.ParseFile(testFset, file, content, 0) + if err != nil { + return nil, nil, err + } + conf := types.Config{Importer: testImporter} + pkg, err := conf.Check("test", testFset, []*ast.File{f}, nil) + if err != nil { + return nil, nil, err + } + + // Find the test struct. + bctx := newBuildContext(testPackageRLP) + typ, err := lookupStructType(pkg.Scope(), typeName) + if err != nil { + return nil, nil, fmt.Errorf("can't find type %s: %v", typeName, err) + } + return bctx, typ, nil +} diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go new file mode 100644 index 000000000000..5b240bfd85fb --- /dev/null +++ b/rlp/rlpgen/main.go @@ -0,0 +1,148 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "go/types" + "io/ioutil" + "os" + + "golang.org/x/tools/go/packages" +) + +const pathOfPackageRLP = "github.com/ethereum/go-ethereum/rlp" + +func main() { + var ( + pkgdir = flag.String("dir", ".", "input package") + output = flag.String("out", "-", "output file (default is stdout)") + genEncoder = flag.Bool("encoder", true, "generate EncodeRLP?") + genDecoder = flag.Bool("decoder", false, "generate DecodeRLP?") + typename = flag.String("type", "", "type to generate methods for") + ) + flag.Parse() + + cfg := Config{ + Dir: *pkgdir, + Type: *typename, + GenerateEncoder: *genEncoder, + GenerateDecoder: *genDecoder, + } + code, err := cfg.process() + if err != nil { + fatal(err) + } + if *output == "-" { + os.Stdout.Write(code) + } else if err := ioutil.WriteFile(*output, code, 0644); err != nil { + fatal(err) + } +} + +func fatal(args ...interface{}) { + fmt.Fprintln(os.Stderr, args...) + os.Exit(1) +} + +type Config struct { + Dir string // input package directory + Type string + + GenerateEncoder bool + GenerateDecoder bool +} + +// process generates the Go code. +func (cfg *Config) process() (code []byte, err error) { + // Load packages. + pcfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedTypes | packages.NeedImports | packages.NeedDeps, + Dir: cfg.Dir, + BuildFlags: []string{"-tags", "norlpgen"}, + } + ps, err := packages.Load(pcfg, pathOfPackageRLP, ".") + if err != nil { + return nil, err + } + if len(ps) == 0 { + return nil, fmt.Errorf("no Go package found in %s", cfg.Dir) + } + packages.PrintErrors(ps) + + // Find the packages that were loaded. + var ( + pkg *types.Package + packageRLP *types.Package + ) + for _, p := range ps { + if len(p.Errors) > 0 { + return nil, fmt.Errorf("package %s has errors", p.PkgPath) + } + if p.PkgPath == pathOfPackageRLP { + packageRLP = p.Types + } else { + pkg = p.Types + } + } + bctx := newBuildContext(packageRLP) + + // Find the type and generate. + typ, err := lookupStructType(pkg.Scope(), cfg.Type) + if err != nil { + return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err) + } + code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder) + if err != nil { + return nil, err + } + + // Add build comments. + // This is done here to avoid processing these lines with gofmt. + var header bytes.Buffer + fmt.Fprint(&header, "// Code generated by rlpgen. DO NOT EDIT.\n\n") + fmt.Fprint(&header, "//go:build !norlpgen\n") + fmt.Fprint(&header, "// +build !norlpgen\n\n") + return append(header.Bytes(), code...), nil +} + +func lookupStructType(scope *types.Scope, name string) (*types.Named, error) { + typ, err := lookupType(scope, name) + if err != nil { + return nil, err + } + _, ok := typ.Underlying().(*types.Struct) + if !ok { + return nil, errors.New("not a struct type") + } + return typ, nil +} + +func lookupType(scope *types.Scope, name string) (*types.Named, error) { + obj := scope.Lookup(name) + if obj == nil { + return nil, errors.New("no such identifier") + } + typ, ok := obj.(*types.TypeName) + if !ok { + return nil, errors.New("not a type") + } + return typ.Type().(*types.Named), nil +} diff --git a/rlp/rlpgen/testdata/bigint.in.txt b/rlp/rlpgen/testdata/bigint.in.txt new file mode 100644 index 000000000000..d23d84a28763 --- /dev/null +++ b/rlp/rlpgen/testdata/bigint.in.txt @@ -0,0 +1,10 @@ +// -*- mode: go -*- + +package test + +import "math/big" + +type Test struct { + Int *big.Int + IntNoPtr big.Int +} diff --git a/rlp/rlpgen/testdata/bigint.out.txt b/rlp/rlpgen/testdata/bigint.out.txt new file mode 100644 index 000000000000..f54d1faa15f7 --- /dev/null +++ b/rlp/rlpgen/testdata/bigint.out.txt @@ -0,0 +1,49 @@ +package test + +import "github.com/ethereum/go-ethereum/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + if obj.Int == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Int.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Int) + } + if obj.IntNoPtr.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(&obj.IntNoPtr) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Int: + _tmp1, err := dec.BigInt() + if err != nil { + return err + } + _tmp0.Int = _tmp1 + // IntNoPtr: + _tmp2, err := dec.BigInt() + if err != nil { + return err + } + _tmp0.IntNoPtr = (*_tmp2) + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/nil.in.txt b/rlp/rlpgen/testdata/nil.in.txt new file mode 100644 index 000000000000..a28ff344874d --- /dev/null +++ b/rlp/rlpgen/testdata/nil.in.txt @@ -0,0 +1,30 @@ +// -*- mode: go -*- + +package test + +type Aux struct{ + A uint32 +} + +type Test struct{ + Uint8 *byte `rlp:"nil"` + Uint8List *byte `rlp:"nilList"` + + Uint32 *uint32 `rlp:"nil"` + Uint32List *uint32 `rlp:"nilList"` + + Uint64 *uint64 `rlp:"nil"` + Uint64List *uint64 `rlp:"nilList"` + + String *string `rlp:"nil"` + StringList *string `rlp:"nilList"` + + ByteArray *[3]byte `rlp:"nil"` + ByteArrayList *[3]byte `rlp:"nilList"` + + ByteSlice *[]byte `rlp:"nil"` + ByteSliceList *[]byte `rlp:"nilList"` + + Struct *Aux `rlp:"nil"` + StructString *Aux `rlp:"nilString"` +} diff --git a/rlp/rlpgen/testdata/nil.out.txt b/rlp/rlpgen/testdata/nil.out.txt new file mode 100644 index 000000000000..e0d5dcebad3b --- /dev/null +++ b/rlp/rlpgen/testdata/nil.out.txt @@ -0,0 +1,289 @@ +package test + +import "github.com/ethereum/go-ethereum/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + if obj.Uint8 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64(uint64((*obj.Uint8))) + } + if obj.Uint8List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64(uint64((*obj.Uint8List))) + } + if obj.Uint32 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64(uint64((*obj.Uint32))) + } + if obj.Uint32List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64(uint64((*obj.Uint32List))) + } + if obj.Uint64 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64((*obj.Uint64)) + } + if obj.Uint64List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64((*obj.Uint64List)) + } + if obj.String == nil { + w.Write([]byte{0x80}) + } else { + w.WriteString((*obj.String)) + } + if obj.StringList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteString((*obj.StringList)) + } + if obj.ByteArray == nil { + w.Write([]byte{0x80}) + } else { + w.WriteBytes(obj.ByteArray[:]) + } + if obj.ByteArrayList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteBytes(obj.ByteArrayList[:]) + } + if obj.ByteSlice == nil { + w.Write([]byte{0x80}) + } else { + w.WriteBytes((*obj.ByteSlice)) + } + if obj.ByteSliceList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteBytes((*obj.ByteSliceList)) + } + if obj.Struct == nil { + w.Write([]byte{0xC0}) + } else { + _tmp1 := w.List() + w.WriteUint64(uint64(obj.Struct.A)) + w.ListEnd(_tmp1) + } + if obj.StructString == nil { + w.Write([]byte{0x80}) + } else { + _tmp2 := w.List() + w.WriteUint64(uint64(obj.StructString.A)) + w.ListEnd(_tmp2) + } + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Uint8: + var _tmp2 *byte + if _tmp3, _tmp4, err := dec.Kind(); err != nil { + return err + } else if _tmp4 != 0 || _tmp3 != rlp.String { + _tmp1, err := dec.Uint8() + if err != nil { + return err + } + _tmp2 = &_tmp1 + } + _tmp0.Uint8 = _tmp2 + // Uint8List: + var _tmp6 *byte + if _tmp7, _tmp8, err := dec.Kind(); err != nil { + return err + } else if _tmp8 != 0 || _tmp7 != rlp.List { + _tmp5, err := dec.Uint8() + if err != nil { + return err + } + _tmp6 = &_tmp5 + } + _tmp0.Uint8List = _tmp6 + // Uint32: + var _tmp10 *uint32 + if _tmp11, _tmp12, err := dec.Kind(); err != nil { + return err + } else if _tmp12 != 0 || _tmp11 != rlp.String { + _tmp9, err := dec.Uint32() + if err != nil { + return err + } + _tmp10 = &_tmp9 + } + _tmp0.Uint32 = _tmp10 + // Uint32List: + var _tmp14 *uint32 + if _tmp15, _tmp16, err := dec.Kind(); err != nil { + return err + } else if _tmp16 != 0 || _tmp15 != rlp.List { + _tmp13, err := dec.Uint32() + if err != nil { + return err + } + _tmp14 = &_tmp13 + } + _tmp0.Uint32List = _tmp14 + // Uint64: + var _tmp18 *uint64 + if _tmp19, _tmp20, err := dec.Kind(); err != nil { + return err + } else if _tmp20 != 0 || _tmp19 != rlp.String { + _tmp17, err := dec.Uint64() + if err != nil { + return err + } + _tmp18 = &_tmp17 + } + _tmp0.Uint64 = _tmp18 + // Uint64List: + var _tmp22 *uint64 + if _tmp23, _tmp24, err := dec.Kind(); err != nil { + return err + } else if _tmp24 != 0 || _tmp23 != rlp.List { + _tmp21, err := dec.Uint64() + if err != nil { + return err + } + _tmp22 = &_tmp21 + } + _tmp0.Uint64List = _tmp22 + // String: + var _tmp26 *string + if _tmp27, _tmp28, err := dec.Kind(); err != nil { + return err + } else if _tmp28 != 0 || _tmp27 != rlp.String { + _tmp25, err := dec.String() + if err != nil { + return err + } + _tmp26 = &_tmp25 + } + _tmp0.String = _tmp26 + // StringList: + var _tmp30 *string + if _tmp31, _tmp32, err := dec.Kind(); err != nil { + return err + } else if _tmp32 != 0 || _tmp31 != rlp.List { + _tmp29, err := dec.String() + if err != nil { + return err + } + _tmp30 = &_tmp29 + } + _tmp0.StringList = _tmp30 + // ByteArray: + var _tmp34 *[3]byte + if _tmp35, _tmp36, err := dec.Kind(); err != nil { + return err + } else if _tmp36 != 0 || _tmp35 != rlp.String { + var _tmp33 [3]byte + if err := dec.ReadBytes(_tmp33[:]); err != nil { + return err + } + _tmp34 = &_tmp33 + } + _tmp0.ByteArray = _tmp34 + // ByteArrayList: + var _tmp38 *[3]byte + if _tmp39, _tmp40, err := dec.Kind(); err != nil { + return err + } else if _tmp40 != 0 || _tmp39 != rlp.List { + var _tmp37 [3]byte + if err := dec.ReadBytes(_tmp37[:]); err != nil { + return err + } + _tmp38 = &_tmp37 + } + _tmp0.ByteArrayList = _tmp38 + // ByteSlice: + var _tmp42 *[]byte + if _tmp43, _tmp44, err := dec.Kind(); err != nil { + return err + } else if _tmp44 != 0 || _tmp43 != rlp.String { + _tmp41, err := dec.Bytes() + if err != nil { + return err + } + _tmp42 = &_tmp41 + } + _tmp0.ByteSlice = _tmp42 + // ByteSliceList: + var _tmp46 *[]byte + if _tmp47, _tmp48, err := dec.Kind(); err != nil { + return err + } else if _tmp48 != 0 || _tmp47 != rlp.List { + _tmp45, err := dec.Bytes() + if err != nil { + return err + } + _tmp46 = &_tmp45 + } + _tmp0.ByteSliceList = _tmp46 + // Struct: + var _tmp51 *Aux + if _tmp52, _tmp53, err := dec.Kind(); err != nil { + return err + } else if _tmp53 != 0 || _tmp52 != rlp.List { + var _tmp49 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp50, err := dec.Uint32() + if err != nil { + return err + } + _tmp49.A = _tmp50 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp51 = &_tmp49 + } + _tmp0.Struct = _tmp51 + // StructString: + var _tmp56 *Aux + if _tmp57, _tmp58, err := dec.Kind(); err != nil { + return err + } else if _tmp58 != 0 || _tmp57 != rlp.String { + var _tmp54 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp55, err := dec.Uint32() + if err != nil { + return err + } + _tmp54.A = _tmp55 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp56 = &_tmp54 + } + _tmp0.StructString = _tmp56 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/optional.in.txt b/rlp/rlpgen/testdata/optional.in.txt new file mode 100644 index 000000000000..f1ac9f7899d1 --- /dev/null +++ b/rlp/rlpgen/testdata/optional.in.txt @@ -0,0 +1,17 @@ +// -*- mode: go -*- + +package test + +type Aux struct { + A uint64 +} + +type Test struct { + Uint64 uint64 `rlp:"optional"` + Pointer *uint64 `rlp:"optional"` + String string `rlp:"optional"` + Slice []uint64 `rlp:"optional"` + Array [3]byte `rlp:"optional"` + NamedStruct Aux `rlp:"optional"` + AnonStruct struct{ A string } `rlp:"optional"` +} diff --git a/rlp/rlpgen/testdata/optional.out.txt b/rlp/rlpgen/testdata/optional.out.txt new file mode 100644 index 000000000000..02df8e457f94 --- /dev/null +++ b/rlp/rlpgen/testdata/optional.out.txt @@ -0,0 +1,153 @@ +package test + +import "github.com/ethereum/go-ethereum/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + _tmp1 := obj.Uint64 != 0 + _tmp2 := obj.Pointer != nil + _tmp3 := obj.String != "" + _tmp4 := len(obj.Slice) > 0 + _tmp5 := obj.Array != ([3]byte{}) + _tmp6 := obj.NamedStruct != (Aux{}) + _tmp7 := obj.AnonStruct != (struct{ A string }{}) + if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + w.WriteUint64(obj.Uint64) + } + if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + if obj.Pointer == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64((*obj.Pointer)) + } + } + if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + w.WriteString(obj.String) + } + if _tmp4 || _tmp5 || _tmp6 || _tmp7 { + _tmp8 := w.List() + for _, _tmp9 := range obj.Slice { + w.WriteUint64(_tmp9) + } + w.ListEnd(_tmp8) + } + if _tmp5 || _tmp6 || _tmp7 { + w.WriteBytes(obj.Array[:]) + } + if _tmp6 || _tmp7 { + _tmp10 := w.List() + w.WriteUint64(obj.NamedStruct.A) + w.ListEnd(_tmp10) + } + if _tmp7 { + _tmp11 := w.List() + w.WriteString(obj.AnonStruct.A) + w.ListEnd(_tmp11) + } + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Uint64: + if dec.MoreDataInList() { + _tmp1, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.Uint64 = _tmp1 + // Pointer: + if dec.MoreDataInList() { + _tmp2, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.Pointer = &_tmp2 + // String: + if dec.MoreDataInList() { + _tmp3, err := dec.String() + if err != nil { + return err + } + _tmp0.String = _tmp3 + // Slice: + if dec.MoreDataInList() { + var _tmp4 []uint64 + if _, err := dec.List(); err != nil { + return err + } + for dec.MoreDataInList() { + _tmp5, err := dec.Uint64() + if err != nil { + return err + } + _tmp4 = append(_tmp4, _tmp5) + } + if err := dec.ListEnd(); err != nil { + return err + } + _tmp0.Slice = _tmp4 + // Array: + if dec.MoreDataInList() { + var _tmp6 [3]byte + if err := dec.ReadBytes(_tmp6[:]); err != nil { + return err + } + _tmp0.Array = _tmp6 + // NamedStruct: + if dec.MoreDataInList() { + var _tmp7 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp8, err := dec.Uint64() + if err != nil { + return err + } + _tmp7.A = _tmp8 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp0.NamedStruct = _tmp7 + // AnonStruct: + if dec.MoreDataInList() { + var _tmp9 struct{ A string } + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp10, err := dec.String() + if err != nil { + return err + } + _tmp9.A = _tmp10 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp0.AnonStruct = _tmp9 + } + } + } + } + } + } + } + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/rawvalue.in.txt b/rlp/rlpgen/testdata/rawvalue.in.txt new file mode 100644 index 000000000000..3a657bc907bb --- /dev/null +++ b/rlp/rlpgen/testdata/rawvalue.in.txt @@ -0,0 +1,11 @@ +// -*- mode: go -*- + +package test + +import "github.com/ethereum/go-ethereum/rlp" + +type Test struct { + RawValue rlp.RawValue + PointerToRawValue *rlp.RawValue + SliceOfRawValue []rlp.RawValue +} diff --git a/rlp/rlpgen/testdata/rawvalue.out.txt b/rlp/rlpgen/testdata/rawvalue.out.txt new file mode 100644 index 000000000000..3607c9863676 --- /dev/null +++ b/rlp/rlpgen/testdata/rawvalue.out.txt @@ -0,0 +1,64 @@ +package test + +import "github.com/ethereum/go-ethereum/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.Write(obj.RawValue) + if obj.PointerToRawValue == nil { + w.Write([]byte{0x80}) + } else { + w.Write((*obj.PointerToRawValue)) + } + _tmp1 := w.List() + for _, _tmp2 := range obj.SliceOfRawValue { + w.Write(_tmp2) + } + w.ListEnd(_tmp1) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // RawValue: + _tmp1, err := dec.Raw() + if err != nil { + return err + } + _tmp0.RawValue = _tmp1 + // PointerToRawValue: + _tmp2, err := dec.Raw() + if err != nil { + return err + } + _tmp0.PointerToRawValue = &_tmp2 + // SliceOfRawValue: + var _tmp3 []rlp.RawValue + if _, err := dec.List(); err != nil { + return err + } + for dec.MoreDataInList() { + _tmp4, err := dec.Raw() + if err != nil { + return err + } + _tmp3 = append(_tmp3, _tmp4) + } + if err := dec.ListEnd(); err != nil { + return err + } + _tmp0.SliceOfRawValue = _tmp3 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/uints.in.txt b/rlp/rlpgen/testdata/uints.in.txt new file mode 100644 index 000000000000..8095da997d96 --- /dev/null +++ b/rlp/rlpgen/testdata/uints.in.txt @@ -0,0 +1,10 @@ +// -*- mode: go -*- + +package test + +type Test struct{ + A uint8 + B uint16 + C uint32 + D uint64 +} diff --git a/rlp/rlpgen/testdata/uints.out.txt b/rlp/rlpgen/testdata/uints.out.txt new file mode 100644 index 000000000000..1a354956a409 --- /dev/null +++ b/rlp/rlpgen/testdata/uints.out.txt @@ -0,0 +1,53 @@ +package test + +import "github.com/ethereum/go-ethereum/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteUint64(uint64(obj.A)) + w.WriteUint64(uint64(obj.B)) + w.WriteUint64(uint64(obj.C)) + w.WriteUint64(obj.D) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp1, err := dec.Uint8() + if err != nil { + return err + } + _tmp0.A = _tmp1 + // B: + _tmp2, err := dec.Uint16() + if err != nil { + return err + } + _tmp0.B = _tmp2 + // C: + _tmp3, err := dec.Uint32() + if err != nil { + return err + } + _tmp0.C = _tmp3 + // D: + _tmp4, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.D = _tmp4 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/types.go b/rlp/rlpgen/types.go new file mode 100644 index 000000000000..5926a801eecb --- /dev/null +++ b/rlp/rlpgen/types.go @@ -0,0 +1,98 @@ +package main + +import ( + "fmt" + "go/types" + "reflect" +) + +// typeReflectKind gives the reflect.Kind that represents typ. +func typeReflectKind(typ types.Type) reflect.Kind { + switch typ := typ.(type) { + case *types.Basic: + k := typ.Kind() + if k >= types.Bool && k <= types.Complex128 { + // value order matches for Bool..Complex128 + return reflect.Bool + reflect.Kind(k-types.Bool) + } + if k == types.String { + return reflect.String + } + if k == types.UnsafePointer { + return reflect.UnsafePointer + } + panic(fmt.Errorf("unhandled BasicKind %v", k)) + case *types.Array: + return reflect.Array + case *types.Chan: + return reflect.Chan + case *types.Interface: + return reflect.Interface + case *types.Map: + return reflect.Map + case *types.Pointer: + return reflect.Ptr + case *types.Signature: + return reflect.Func + case *types.Slice: + return reflect.Slice + case *types.Struct: + return reflect.Struct + default: + panic(fmt.Errorf("unhandled type %T", typ)) + } +} + +// nonZeroCheck returns the expression that checks whether 'v' is a non-zero value of type 'vtyp'. +func nonZeroCheck(v string, vtyp types.Type, qualify types.Qualifier) string { + // Resolve type name. + typ := resolveUnderlying(vtyp) + switch typ := typ.(type) { + case *types.Basic: + k := typ.Kind() + switch { + case k == types.Bool: + return v + case k >= types.Uint && k <= types.Complex128: + return fmt.Sprintf("%s != 0", v) + case k == types.String: + return fmt.Sprintf(`%s != ""`, v) + default: + panic(fmt.Errorf("unhandled BasicKind %v", k)) + } + case *types.Array, *types.Struct: + return fmt.Sprintf("%s != (%s{})", v, types.TypeString(vtyp, qualify)) + case *types.Interface, *types.Pointer, *types.Signature: + return fmt.Sprintf("%s != nil", v) + case *types.Slice, *types.Map: + return fmt.Sprintf("len(%s) > 0", v) + default: + panic(fmt.Errorf("unhandled type %T", typ)) + } +} + +// isBigInt checks whether 'typ' is "math/big".Int. +func isBigInt(typ types.Type) bool { + named, ok := typ.(*types.Named) + if !ok { + return false + } + name := named.Obj() + return name.Pkg().Path() == "math/big" && name.Name() == "Int" +} + +// isByte checks whether the underlying type of 'typ' is uint8. +func isByte(typ types.Type) bool { + basic, ok := resolveUnderlying(typ).(*types.Basic) + return ok && basic.Kind() == types.Uint8 +} + +func resolveUnderlying(typ types.Type) types.Type { + for { + t := typ.Underlying() + if t == typ { + return t + } + typ = t + } +} diff --git a/rlp/typecache.go b/rlp/typecache.go index 62553d3b55c1..3e37c9d2fcc7 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -19,9 +19,10 @@ package rlp import ( "fmt" "reflect" - "strings" "sync" "sync/atomic" + + "github.com/ethereum/go-ethereum/rlp/internal/rlpstruct" ) // typeinfo is an entry in the type cache. @@ -32,35 +33,16 @@ type typeinfo struct { writerErr error // error from makeWriter } -// tags represents struct tags. -type tags struct { - // rlp:"nil" controls whether empty input results in a nil pointer. - // nilKind is the kind of empty value allowed for the field. - nilKind Kind - nilOK bool - - // rlp:"optional" allows for a field to be missing in the input list. - // If this is set, all subsequent fields must also be optional. - optional bool - - // rlp:"tail" controls whether this field swallows additional list elements. It can - // only be set for the last field, which must be of slice type. - tail bool - - // rlp:"-" ignores fields. - ignored bool -} - // typekey is the key of a type in typeCache. It includes the struct tags because // they might generate a different decoder. type typekey struct { reflect.Type - tags + rlpstruct.Tags } type decoder func(*Stream, reflect.Value) error -type writer func(reflect.Value, *encbuf) error +type writer func(reflect.Value, *encBuffer) error var theTC = newTypeCache() @@ -95,10 +77,10 @@ func (c *typeCache) info(typ reflect.Type) *typeinfo { } // Not in the cache, need to generate info for this type. - return c.generate(typ, tags{}) + return c.generate(typ, rlpstruct.Tags{}) } -func (c *typeCache) generate(typ reflect.Type, tags tags) *typeinfo { +func (c *typeCache) generate(typ reflect.Type, tags rlpstruct.Tags) *typeinfo { c.mu.Lock() defer c.mu.Unlock() @@ -122,7 +104,7 @@ func (c *typeCache) generate(typ reflect.Type, tags tags) *typeinfo { return info } -func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags tags) *typeinfo { +func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags rlpstruct.Tags) *typeinfo { key := typekey{typ, tags} if info := c.next[key]; info != nil { return info @@ -144,35 +126,40 @@ type field struct { // structFields resolves the typeinfo of all public fields in a struct type. func structFields(typ reflect.Type) (fields []field, err error) { - var ( - lastPublic = lastPublicField(typ) - anyOptional = false - ) + // Convert fields to rlpstruct.Field. + var allStructFields []rlpstruct.Field for i := 0; i < typ.NumField(); i++ { - if f := typ.Field(i); f.PkgPath == "" { // exported - tags, err := parseStructTag(typ, i, lastPublic) - if err != nil { - return nil, err - } - - // Skip rlp:"-" fields. - if tags.ignored { - continue - } - // If any field has the "optional" tag, subsequent fields must also have it. - if tags.optional || tags.tail { - anyOptional = true - } else if anyOptional { - return nil, fmt.Errorf(`rlp: struct field %v.%s needs "optional" tag`, typ, f.Name) - } - info := theTC.infoWhileGenerating(f.Type, tags) - fields = append(fields, field{i, info, tags.optional}) + rf := typ.Field(i) + allStructFields = append(allStructFields, rlpstruct.Field{ + Name: rf.Name, + Index: i, + Exported: rf.PkgPath == "", + Tag: string(rf.Tag), + Type: *rtypeToStructType(rf.Type, nil), + }) + } + + // Filter/validate fields. + structFields, structTags, err := rlpstruct.ProcessFields(allStructFields) + if err != nil { + if tagErr, ok := err.(rlpstruct.TagError); ok { + tagErr.StructType = typ.String() + return nil, tagErr } + return nil, err + } + + // Resolve typeinfo. + for i, sf := range structFields { + typ := typ.Field(sf.Index).Type + tags := structTags[i] + info := theTC.infoWhileGenerating(typ, tags) + fields = append(fields, field{sf.Index, info, tags.Optional}) } return fields, nil } -// anyOptionalFields returns the index of the first field with "optional" tag. +// firstOptionalField returns the index of the first field with "optional" tag. func firstOptionalField(fields []field) int { for i, f := range fields { if f.optional { @@ -192,82 +179,56 @@ func (e structFieldError) Error() string { return fmt.Sprintf("%v (struct field %v.%s)", e.err, e.typ, e.typ.Field(e.field).Name) } -type structTagError struct { - typ reflect.Type - field, tag, err string +func (i *typeinfo) generate(typ reflect.Type, tags rlpstruct.Tags) { + i.decoder, i.decoderErr = makeDecoder(typ, tags) + i.writer, i.writerErr = makeWriter(typ, tags) } -func (e structTagError) Error() string { - return fmt.Sprintf("rlp: invalid struct tag %q for %v.%s (%s)", e.tag, e.typ, e.field, e.err) -} +// rtypeToStructType converts typ to rlpstruct.Type. +func rtypeToStructType(typ reflect.Type, rec map[reflect.Type]*rlpstruct.Type) *rlpstruct.Type { + k := typ.Kind() + if k == reflect.Invalid { + panic("invalid kind") + } -func parseStructTag(typ reflect.Type, fi, lastPublic int) (tags, error) { - f := typ.Field(fi) - var ts tags - for _, t := range strings.Split(f.Tag.Get("rlp"), ",") { - switch t = strings.TrimSpace(t); t { - case "": - case "-": - ts.ignored = true - case "nil", "nilString", "nilList": - ts.nilOK = true - if f.Type.Kind() != reflect.Ptr { - return ts, structTagError{typ, f.Name, t, "field is not a pointer"} - } - switch t { - case "nil": - ts.nilKind = defaultNilKind(f.Type.Elem()) - case "nilString": - ts.nilKind = String - case "nilList": - ts.nilKind = List - } - case "optional": - ts.optional = true - if ts.tail { - return ts, structTagError{typ, f.Name, t, `also has "tail" tag`} - } - case "tail": - ts.tail = true - if fi != lastPublic { - return ts, structTagError{typ, f.Name, t, "must be on last field"} - } - if ts.optional { - return ts, structTagError{typ, f.Name, t, `also has "optional" tag`} - } - if f.Type.Kind() != reflect.Slice { - return ts, structTagError{typ, f.Name, t, "field type is not slice"} - } - default: - return ts, fmt.Errorf("rlp: unknown struct tag %q on %v.%s", t, typ, f.Name) - } + if prev := rec[typ]; prev != nil { + return prev // short-circuit for recursive types + } + if rec == nil { + rec = make(map[reflect.Type]*rlpstruct.Type) } - return ts, nil -} -func lastPublicField(typ reflect.Type) int { - last := 0 - for i := 0; i < typ.NumField(); i++ { - if typ.Field(i).PkgPath == "" { - last = i - } + t := &rlpstruct.Type{ + Name: typ.String(), + Kind: k, + IsEncoder: typ.Implements(encoderInterface), + IsDecoder: typ.Implements(decoderInterface), + } + rec[typ] = t + if k == reflect.Array || k == reflect.Slice || k == reflect.Ptr { + t.Elem = rtypeToStructType(typ.Elem(), rec) } - return last + return t } -func (i *typeinfo) generate(typ reflect.Type, tags tags) { - i.decoder, i.decoderErr = makeDecoder(typ, tags) - i.writer, i.writerErr = makeWriter(typ, tags) -} +// typeNilKind gives the RLP value kind for nil pointers to 'typ'. +func typeNilKind(typ reflect.Type, tags rlpstruct.Tags) Kind { + styp := rtypeToStructType(typ, nil) -// defaultNilKind determines whether a nil pointer to typ encodes/decodes -// as an empty string or empty list. -func defaultNilKind(typ reflect.Type) Kind { - k := typ.Kind() - if isUint(k) || k == reflect.String || k == reflect.Bool || isByteArray(typ) { + var nk rlpstruct.NilKind + if tags.NilOK { + nk = tags.NilKind + } else { + nk = styp.DefaultNilValue() + } + switch nk { + case rlpstruct.NilKindString: return String + case rlpstruct.NilKindList: + return List + default: + panic("invalid nil kind value") } - return List } func isUint(k reflect.Kind) bool { @@ -277,7 +238,3 @@ func isUint(k reflect.Kind) bool { func isByte(typ reflect.Type) bool { return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface) } - -func isByteArray(typ reflect.Type) bool { - return (typ.Kind() == reflect.Slice || typ.Kind() == reflect.Array) && isByte(typ.Elem()) -} diff --git a/rpc/client.go b/rpc/client.go index d55af75545cd..d3ce0297754c 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -186,7 +186,7 @@ func DialContext(ctx context.Context, rawurl string) (*Client, error) { } } -// Client retrieves the client from the context, if any. This can be used to perform +// ClientFromContext retrieves the client from the context, if any. This can be used to perform // 'reverse calls' in a handler method. func ClientFromContext(ctx context.Context) (*Client, bool) { client, ok := ctx.Value(clientContextKey{}).(*Client) @@ -333,7 +333,7 @@ func (c *Client) BatchCall(b []BatchElem) error { return c.BatchCallContext(ctx, b) } -// BatchCall sends all given requests as a single batch and waits for the server +// BatchCallContext sends all given requests as a single batch and waits for the server // to return a response for all of them. The wait duration is bounded by the // context's deadline. // diff --git a/rpc/server.go b/rpc/server.go index e2d5c0383570..babc5688e264 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -26,6 +26,7 @@ import ( ) const MetadataApi = "rpc" +const EngineApi = "engine" // CodecOption specifies which type of messages a codec supports. // diff --git a/rpc/server_test.go b/rpc/server_test.go index c692a071cf1e..e67893710dc2 100644 --- a/rpc/server_test.go +++ b/rpc/server_test.go @@ -134,7 +134,7 @@ func TestServerShortLivedConn(t *testing.T) { if err != nil { t.Fatal("can't dial:", err) } - defer conn.Close() + conn.SetDeadline(deadline) // Write the request, then half-close the connection so the server stops reading. conn.Write([]byte(request)) @@ -142,6 +142,8 @@ func TestServerShortLivedConn(t *testing.T) { // Now try to get the response. buf := make([]byte, 2000) n, err := conn.Read(buf) + conn.Close() + if err != nil { t.Fatal("read error:", err) } diff --git a/rpc/subscription.go b/rpc/subscription.go index 942e764e5d6c..d7ba784fc532 100644 --- a/rpc/subscription.go +++ b/rpc/subscription.go @@ -34,7 +34,7 @@ import ( var ( // ErrNotificationsUnsupported is returned when the connection doesn't support notifications ErrNotificationsUnsupported = errors.New("notifications not supported") - // ErrNotificationNotFound is returned when the notification for the given id is not found + // ErrSubscriptionNotFound is returned when the notification for the given id is not found ErrSubscriptionNotFound = errors.New("subscription not found") ) diff --git a/rpc/types.go b/rpc/types.go index 959e383723bd..46b08caf6800 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -30,10 +30,11 @@ import ( // API describes the set of methods offered over the RPC interface type API struct { - Namespace string // namespace under which the rpc methods of Service are exposed - Version string // api version for DApp's - Service interface{} // receiver instance which holds the methods - Public bool // indication if the methods must be considered safe for public use + Namespace string // namespace under which the rpc methods of Service are exposed + Version string // api version for DApp's + Service interface{} // receiver instance which holds the methods + Public bool // indication if the methods must be considered safe for public use + Authenticated bool // whether the api should only be available behind authentication. } // ServerCodec implements reading, parsing and writing RPC messages for the server side of diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index 8659f798e4a0..f74b7fd08bb4 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -76,7 +76,7 @@ func TestWebsocketOriginCheck(t *testing.T) { // Connections without origin header should work. client, err = DialWebsocket(context.Background(), wsURL, "") if err != nil { - t.Fatal("error for empty origin") + t.Fatalf("error for empty origin: %v", err) } client.Close() } diff --git a/signer/fourbyte/abi.go b/signer/fourbyte/abi.go index d8fbabd3b1b3..352abc59e182 100644 --- a/signer/fourbyte/abi.go +++ b/signer/fourbyte/abi.go @@ -20,7 +20,6 @@ import ( "bytes" "encoding/json" "fmt" - "regexp" "strings" "github.com/ethereum/go-ethereum/accounts/abi" @@ -75,42 +74,15 @@ func verifySelector(selector string, calldata []byte) (*decodedCallData, error) return parseCallData(calldata, string(abidata)) } -// selectorRegexp is used to validate that a 4byte database selector corresponds -// to a valid ABI function declaration. -// -// Note, although uppercase letters are not part of the ABI spec, this regexp -// still accepts it as the general format is valid. It will be rejected later -// by the type checker. -var selectorRegexp = regexp.MustCompile(`^([^\)]+)\(([A-Za-z0-9,\[\]]*)\)`) - // parseSelector converts a method selector into an ABI JSON spec. The returned // data is a valid JSON string which can be consumed by the standard abi package. func parseSelector(unescapedSelector string) ([]byte, error) { - // Define a tiny fake ABI struct for JSON marshalling - type fakeArg struct { - Type string `json:"type"` - } - type fakeABI struct { - Name string `json:"name"` - Type string `json:"type"` - Inputs []fakeArg `json:"inputs"` - } - // Validate the unescapedSelector and extract it's components - groups := selectorRegexp.FindStringSubmatch(unescapedSelector) - if len(groups) != 3 { - return nil, fmt.Errorf("invalid selector %q (%v matches)", unescapedSelector, len(groups)) + selector, err := abi.ParseSelector(unescapedSelector) + if err != nil { + return nil, fmt.Errorf("failed to parse selector: %v", err) } - name := groups[1] - args := groups[2] - // Reassemble the fake ABI and constuct the JSON - arguments := make([]fakeArg, 0) - if len(args) > 0 { - for _, arg := range strings.Split(args, ",") { - arguments = append(arguments, fakeArg{arg}) - } - } - return json.Marshal([]fakeABI{{name, "function", arguments}}) + return json.Marshal([]abi.SelectorMarshaling{selector}) } // parseCallData matches the provided call data against the ABI definition and diff --git a/statediff/README.md b/statediff/README.md index 35d50e39d77a..f262d7a8ecb4 100644 --- a/statediff/README.md +++ b/statediff/README.md @@ -6,11 +6,12 @@ either relaying the state objects to RPC subscribers or writing them directly to It also exposes RPC endpoints for fetching or writing to Postgres the state diff at a specific block height or for a specific block hash, this operates on historical block and state data and so depends on a complete state archive. -Data is emitted in this differential format in order to make it feasible to IPLD-ize and index the *entire* Ethereum state +Data is emitted in this differential format in order to make it feasible to IPLD-ize and index the _entire_ Ethereum state (including intermediate state and storage trie nodes). If this state diff process is ran continuously from genesis, the entire state at any block can be materialized from the cumulative differentials up to that point. ## Statediff object + A state diff `StateObject` is the collection of all the state and storage trie nodes that have been updated in a given block. For convenience, we also associate these nodes with the block number and hash, and optionally the set of code hashes and code for any contracts deployed in this block. @@ -52,6 +53,7 @@ type CodeAndCodeHash struct { Code []byte `json:"code"` } ``` + These objects are packed into a `Payload` structure which can additionally associate the `StateObject` with the block (header, uncles, and transactions), receipts, and total difficulty. This `Payload` encapsulates all of the differential data at a given block, and allows us to index the entire Ethereum data structure @@ -71,38 +73,59 @@ type Payload struct { ``` ## Usage + This state diffing service runs as an auxiliary service concurrent to the regular syncing process of the geth node. ### CLI configuration + This service introduces a CLI flag namespace `statediff` -`--statediff` flag is used to turn on the service -`--statediff.writing` is used to tell the service to write state diff objects it produces from synced ChainEvents directly to a configured Postgres database -`--statediff.workers` is used to set the number of concurrent workers to process state diff objects and write them into the database -`--statediff.db.type` is the type of database we write out to (current options: postgres, dump, file) -`--statediff.dump.dst` is the destination to write to when operating in database dump mode (stdout, stderr, discard) -`--statediff.db.driver` is the specific driver to use for the database (current options for postgres: pgx and sqlx) -`--statediff.db.host` is the hostname/ip to dial to connect to the database -`--statediff.db.port` is the port to dial to connect to the database -`--statediff.db.name` is the name of the database to connect to -`--statediff.db.user` is the user to connect to the database as -`--statediff.db.password` is the password to use to connect to the database -`--statediff.db.conntimeout` is the connection timeout (in seconds) -`--statediff.db.maxconns` is the maximum number of database connections -`--statediff.db.minconns` is the minimum number of database connections -`--statediff.db.maxidleconns` is the maximum number of idle connections -`--statediff.db.maxconnidletime` is the maximum lifetime for an idle connection (in seconds) -`--statediff.db.maxconnlifetime` is the maximum lifetime for a connection (in seconds) -`--statediff.db.nodeid` is the node id to use in the Postgres database -`--statediff.db.clientname` is the client name to use in the Postgres database -`--statediff.file.path` full path (including filename) to write statediff data out to when operating in file mode +`--statediff` flag is used to turn on the service + +`--statediff.writing` is used to tell the service to write state diff objects it produces from synced ChainEvents directly to a configured Postgres database + +`--statediff.workers` is used to set the number of concurrent workers to process state diff objects and write them into the database + +`--statediff.db.type` is the type of database we write out to (current options: postgres, dump, file) + +`--statediff.dump.dst` is the destination to write to when operating in database dump mode (stdout, stderr, discard) + +`--statediff.db.driver` is the specific driver to use for the database (current options for postgres: pgx and sqlx) + +`--statediff.db.host` is the hostname/ip to dial to connect to the database + +`--statediff.db.port` is the port to dial to connect to the database + +`--statediff.db.name` is the name of the database to connect to + +`--statediff.db.user` is the user to connect to the database as + +`--statediff.db.password` is the password to use to connect to the database + +`--statediff.db.conntimeout` is the connection timeout (in seconds) + +`--statediff.db.maxconns` is the maximum number of database connections + +`--statediff.db.minconns` is the minimum number of database connections + +`--statediff.db.maxidleconns` is the maximum number of idle connections + +`--statediff.db.maxconnidletime` is the maximum lifetime for an idle connection (in seconds) + +`--statediff.db.maxconnlifetime` is the maximum lifetime for a connection (in seconds) + +`--statediff.db.nodeid` is the node id to use in the Postgres database + +`--statediff.db.clientname` is the client name to use in the Postgres database + +`--statediff.file.path` full path (including filename) to write statediff data out to when operating in file mode + +`--statediff.file.wapath` full path (including filename) to write statediff watched addresses out to when operating in file mode The service can only operate in full sync mode (`--syncmode=full`), but only the historical RPC endpoints require an archive node (`--gcmode=archive`) e.g. -` -./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db.type=postgres --statediff.db.driver=sqlx --statediff.db.host=localhost --statediff.db.port=5432 --statediff.db.name=vulcanize_test --statediff.db.user=postgres --statediff.db.nodeid=nodeid --statediff.db.clientname=clientname -` +`./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db.type=postgres --statediff.db.driver=sqlx --statediff.db.host=localhost --statediff.db.port=5432 --statediff.db.name=vulcanize_test --statediff.db.user=postgres --statediff.db.nodeid=nodeid --statediff.db.clientname=clientname` When operating in `--statediff.db.type=file` mode, the service will write SQL statements out to the file designated by `--statediff.file.path`. Please note that it writes out SQL statements with all `ON CONFLICT` constraint checks dropped. @@ -112,6 +135,7 @@ de-duplicate using unix tools (`sort statediff.sql | uniq` or `sort -u statediff back afterwards. ### RPC endpoints + The state diffing service exposes both a WS subscription endpoint, and a number of HTTP unary endpoints. Each of these endpoints requires a set of parameters provided by the caller @@ -126,17 +150,16 @@ type Params struct { IncludeTD bool IncludeCode bool WatchedAddresses []common.Address - WatchedStorageSlots []common.Hash } ``` Using these params we can tell the service whether to include state and/or storage intermediate nodes; whether to include the associated block (header, uncles, and transactions); whether to include the associated receipts; whether to include the total difficulty for this block; whether to include the set of code hashes and code for -contracts deployed in this block; whether to limit the diffing process to a list of specific addresses; and/or -whether to limit the diffing process to a list of specific storage slot keys. +contracts deployed in this block; whether to limit the diffing process to a list of specific addresses. #### Subscription endpoint + A websocket supporting RPC endpoint is exposed for subscribing to state diff `StateObjects` that come off the head of the chain while the geth node syncs. ```go @@ -182,7 +205,9 @@ for { ``` #### Unary endpoints + The service also exposes unary RPC endpoints for retrieving the state diff `StateObject` for a specific block height/hash. + ```go // StateDiffAt returns a state diff payload at the specific blockheight StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) @@ -195,12 +220,14 @@ To expose this endpoint the node needs to have the HTTP server turned on (`--htt and the `statediff` namespace exposed (`--http.api=statediff`). ### Direct indexing into Postgres + If `--statediff.writing` is set, the service will convert the state diff `StateObject` data into IPLD objects, persist them directly to Postgres, and generate secondary indexes around the IPLD data. The schema and migrations for this Postgres database are provided in `statediff/db/`. #### Postgres setup + We use [pressly/goose](https://github.com/pressly/goose) as our Postgres migration manager. You can also load the Postgres schema directly into a database using @@ -209,6 +236,7 @@ You can also load the Postgres schema directly into a database using This will only work on a version 12.4 Postgres database. #### Schema overview + Our Postgres schemas are built around a single IPFS backing Postgres IPLD blockstore table (`public.blocks`) that conforms with [go-ds-sql](https://github.com/ipfs/go-ds-sql/blob/master/postgres/postgres.go). All IPLD objects are stored in this table, where `key` is the blockstore-prefixed multihash key for the IPLD object and `data` contains the bytes for the IPLD block (in the case of all Ethereum IPLDs, this is the RLP byte encoding of the Ethereum object). @@ -227,6 +255,7 @@ table contains a `state_id` foreign key which references the `id` for the `state and in turn that `state_cids` entry contains a `header_id` foreign key which references the `id` of the `header_cids` entry that contains the header for the block these state and storage nodes were updated (diffed). ### Optimization + On mainnet this process is extremely IO intensive and requires significant resources to allow it to keep up with the head of the chain. The state diff processing time for a specific block is dependent on the number and complexity of the state changes that occur in a block and the number of updated state nodes that are available in the in-memory cache vs must be retrieved from disc. @@ -236,6 +265,7 @@ This can be done by increasing the overall `--cache` allocation and/or by increa usage with `--cache.trie`. ## Versioning, Branches, Rebasing, and Releasing + Internal tagged releases are maintained for building the latest version of statediffing geth or using it as a go mod dependency. When a new core go-ethereum version is released, statediffing geth is rebased onto and adjusted to work with the new tag. @@ -244,18 +274,20 @@ need to be able to squash our work before performing a rebase. To this end we re the full incremental history. ### Versioning -Versioning for of statediffing geth follows the below format: -`{Root Version}-statediff-{Statediff Version}` +Example: `v1.10.16-statediff-3.0.2` -Where "root version" is the version of the tagged release from the core go-ethereum repository that our release is rebased on top of -and "statediff version" is the version tracking the state of the statediffing service code. - -E.g. the version at the time of writing this is v1.10.3-statediff-0.0.23, v0.0.23 of the statediffing code rebased on top of the v1.10.3 core tag. +- The first section, `v1.10.16`, corresponds to the release of the root branch this version is rebased onto (e.g., [](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16)[https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16)) +- The second section, `3.0.2`, corresponds to the version of our statediffing code. The major version here (3) should always correspond with the major version of the `ipld-eth-db` schema version it works with (e.g., [](https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6)[https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6](https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6)); it is only bumped when we bump the major version of the schema. + - The major version of the schema is only bumped when a breaking change is made to the schema. + - The minor version is bumped when a new feature is added, or a fix is performed that breaks or updates the statediffing API or CLI in some way. + - The patch version is bumped whenever minor fixes/patches/features are done that don’t change/break API/CLI compatibility. +- We are very strict about the first section and the major version of the statediffing code, but some discretion is required when deciding to bump minor versus patch version of the statediffing code. The statediff version is included in the `VersionMeta` in params/version.go ### Branches + We maintain two official kinds of branches: Major Branch: `{Root Version}-statediff` @@ -271,7 +303,9 @@ If a developer is unsure what version their patch should affect, they should rem they can open a PR against the targeted root branch and be directed to the appropriate feature version and branch. ### Rebasing + When a new root tagged release comes out we rebase our statediffing code on top of the new tag using the following process: + 1. Checkout a new major branch for the tag from the current major branch 2. On the new major branch, squash all our commits since the last major rebase 3. On the new major branch, perform the rebase against the new tag diff --git a/statediff/api.go b/statediff/api.go index 5c534cddb589..0a7c5bba8cc7 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -149,3 +149,8 @@ func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber func (api *PublicStateDiffAPI) WriteStateDiffFor(ctx context.Context, blockHash common.Hash, params Params) error { return api.sds.WriteStateDiffFor(blockHash, params) } + +// WatchAddress changes the list of watched addresses to which the direct indexing is restricted according to given operation +func (api *PublicStateDiffAPI) WatchAddress(operation types.OperationType, args []types.WatchAddressArg) error { + return api.sds.WatchAddress(operation, args) +} diff --git a/statediff/builder.go b/statediff/builder.go index 7811c3e829c9..a89ec2773324 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -36,7 +36,7 @@ import ( var ( nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") - emptyNode, _ = rlp.EncodeToBytes([]byte{}) + emptyNode, _ = rlp.EncodeToBytes(&[]byte{}) emptyContractRoot = crypto.Keccak256Hash(emptyNode) nullCodeHash = crypto.Keccak256Hash([]byte{}).Bytes() ) @@ -123,7 +123,7 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]types2.StateNode, [] node.LeafKey = leafKey if !bytes.Equal(account.CodeHash, nullCodeHash) { var storageNodes []types2.StorageNode - err := sdb.buildStorageNodesEventual(account.Root, nil, true, storageNodeAppender(&storageNodes)) + err := sdb.buildStorageNodesEventual(account.Root, true, storageNodeAppender(&storageNodes)) if err != nil { return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err) } @@ -202,7 +202,8 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args types2.StateRo // a map of their leafkey to all the accounts that were touched and exist at A diffAccountsAtA, err := sdb.deletedOrUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, output) + diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafKeys, + params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } @@ -220,12 +221,12 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args types2.StateRo // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two err = sdb.buildAccountUpdates( diffAccountsAtB, diffAccountsAtA, updatedKeys, - params.WatchedStorageSlots, params.IntermediateStorageNodes, output) + params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error building diff for updated accounts: %v", err) } // build the diff nodes for created accounts - err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput) + err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput) if err != nil { return fmt.Errorf("error building diff for created accounts: %v", err) } @@ -247,7 +248,7 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args types2.Stat // and a slice of all the paths for the nodes in both of the above sets diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - params.WatchedAddresses) + params.watchedAddressesLeafKeys) if err != nil { return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) } @@ -256,7 +257,8 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args types2.Stat // a map of their leafkey to all the accounts that were touched and exist at A diffAccountsAtA, err := sdb.deletedOrUpdatedState( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, output) + diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafKeys, + params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } @@ -274,12 +276,12 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args types2.Stat // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two err = sdb.buildAccountUpdates( diffAccountsAtB, diffAccountsAtA, updatedKeys, - params.WatchedStorageSlots, params.IntermediateStorageNodes, output) + params.IntermediateStorageNodes, output) if err != nil { return fmt.Errorf("error building diff for updated accounts: %v", err) } // build the diff nodes for created accounts - err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput) + err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput) if err != nil { return fmt.Errorf("error building diff for created accounts: %v", err) } @@ -289,7 +291,7 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args types2.Stat // createdAndUpdatedState returns // a mapping of their leafkeys to all the accounts that exist in a different state at B than A // and a slice of the paths for all of the nodes included in both -func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddresses []common.Address) (types2.AccountMap, map[string]bool, error) { +func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddressesLeafKeys map[common.Hash]struct{}) (types2.AccountMap, map[string]bool, error) { diffPathsAtB := make(map[string]bool) diffAcountsAtB := make(types2.AccountMap) it, _ := trie.NewDifferenceIterator(a, b) @@ -313,7 +315,7 @@ func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddres valueNodePath := append(node.Path, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - if isWatchedAddress(watchedAddresses, leafKey) { + if isWatchedAddress(watchedAddressesLeafKeys, leafKey) { diffAcountsAtB[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{ NodeType: node.NodeType, Path: node.Path, @@ -386,7 +388,7 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt // deletedOrUpdatedState returns a slice of all the pathes that are emptied at B // and a mapping of their leafkeys to all the accounts that exist in a different state at A than B -func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, output types2.StateNodeSink) (types2.AccountMap, error) { +func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffAccountsAtB types2.AccountMap, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, intermediateStorageNodes bool, output types2.StateNodeSink) (types2.AccountMap, error) { diffAccountAtA := make(types2.AccountMap) it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { @@ -409,24 +411,48 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m valueNodePath := append(node.Path, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - diffAccountAtA[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{ - NodeType: node.NodeType, - Path: node.Path, - NodeValue: node.NodeValue, - LeafKey: leafKey, - Account: &account, - } - // if this node's path did not show up in diffPathsAtB - // that means the node at this path was deleted (or moved) in B - // emit an empty "removed" diff to signify as such - if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { - if err := output(types2.StateNode{ + if isWatchedAddress(watchedAddressesLeafKeys, leafKey) { + diffAccountAtA[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{ + NodeType: node.NodeType, Path: node.Path, - NodeValue: []byte{}, - NodeType: types2.Removed, + NodeValue: node.NodeValue, LeafKey: leafKey, - }); err != nil { - return nil, err + Account: &account, + } + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted (or moved) in B + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + var diff types2.StateNode + // if this node's leaf key also did not show up in diffAccountsAtB + // that means the node was deleted + // in that case, emit an empty "removed" diff state node + // include empty "removed" diff storage nodes for all the storage slots + if _, ok := diffAccountsAtB[common.Bytes2Hex(leafKey)]; !ok { + diff = types2.StateNode{ + NodeType: types2.Removed, + Path: node.Path, + LeafKey: leafKey, + NodeValue: []byte{}, + } + + var storageDiffs []types2.StorageNode + err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) + if err != nil { + return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err) + } + diff.StorageNodes = storageDiffs + } else { + // emit an empty "removed" diff with empty leaf key if the account was moved + diff = types2.StateNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + } + } + + if err := output(diff); err != nil { + return nil, err + } } } case types2.Extension, types2.Branch: @@ -454,8 +480,7 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m // to generate the statediff node objects for all of the accounts that existed at both A and B but in different states // needs to be called before building account creations and deletions as this mutates // those account maps to remove the accounts which were updated -func (sdb *builder) buildAccountUpdates(creations, deletions types2.AccountMap, updatedKeys []string, - watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output types2.StateNodeSink) error { +func (sdb *builder) buildAccountUpdates(creations, deletions types2.AccountMap, updatedKeys []string, intermediateStorageNodes bool, output types2.StateNodeSink) error { var err error for _, key := range updatedKeys { createdAcc := creations[key] @@ -465,7 +490,7 @@ func (sdb *builder) buildAccountUpdates(creations, deletions types2.AccountMap, oldSR := deletedAcc.Account.Root newSR := createdAcc.Account.Root err = sdb.buildStorageNodesIncremental( - oldSR, newSR, watchedStorageKeys, intermediateStorageNodes, + oldSR, newSR, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) if err != nil { return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err) @@ -489,7 +514,7 @@ func (sdb *builder) buildAccountUpdates(creations, deletions types2.AccountMap, // buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A // it also returns the code and codehash for created contract accounts -func (sdb *builder) buildAccountCreations(accounts types2.AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output types2.StateNodeSink, codeOutput types2.CodeSink) error { +func (sdb *builder) buildAccountCreations(accounts types2.AccountMap, intermediateStorageNodes bool, output types2.StateNodeSink, codeOutput types2.CodeSink) error { for _, val := range accounts { diff := types2.StateNode{ NodeType: val.NodeType, @@ -500,7 +525,7 @@ func (sdb *builder) buildAccountCreations(accounts types2.AccountMap, watchedSto if !bytes.Equal(val.Account.CodeHash, nullCodeHash) { // For contract creations, any storage node contained is a diff var storageDiffs []types2.StorageNode - err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) + err := sdb.buildStorageNodesEventual(val.Account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) if err != nil { return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err) } @@ -528,7 +553,7 @@ func (sdb *builder) buildAccountCreations(accounts types2.AccountMap, watchedSto // buildStorageNodesEventual builds the storage diff node objects for a created account // i.e. it returns all the storage nodes at this state, since there is no previous state -func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { +func (sdb *builder) buildStorageNodesEventual(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { return nil } @@ -539,7 +564,7 @@ func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys return err } it := sTrie.NodeIterator(make([]byte, 0)) - err = sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes, output) + err = sdb.buildStorageNodesFromTrie(it, intermediateNodes, output) if err != nil { return err } @@ -547,9 +572,8 @@ func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys } // buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator -// if any storage keys are provided it will only return those leaf nodes // including intermediate nodes can be turned on or off -func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { +func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { for it.Next(true) { // skip value nodes if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { @@ -565,22 +589,82 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStora valueNodePath := append(node.Path, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - if isWatchedStorageKey(watchedStorageKeys, leafKey) { + if err := output(types2.StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + }); err != nil { + return err + } + case types2.Extension, types2.Branch: + if intermediateNodes { if err := output(types2.StorageNode{ NodeType: node.NodeType, Path: node.Path, NodeValue: node.NodeValue, - LeafKey: leafKey, }); err != nil { return err } } + default: + return fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return it.Error() +} + +// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account +func (sdb *builder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { + if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { + return nil + } + log.Debug("Storage Root For Removed Diffs", "root", sr.Hex()) + sTrie, err := sdb.stateCache.OpenTrie(sr) + if err != nil { + log.Info("error in build removed account storage diffs", "error", err) + return err + } + it := sTrie.NodeIterator(make([]byte, 0)) + err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output) + if err != nil { + return err + } + return nil +} + +// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator +// including intermediate nodes can be turned on or off +func (sdb *builder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.stateCache.TrieDB()) + if err != nil { + return err + } + switch node.NodeType { + case types2.Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + LeafKey: leafKey, + }); err != nil { + return err + } case types2.Extension, types2.Branch: if intermediateNodes { if err := output(types2.StorageNode{ - NodeType: node.NodeType, + NodeType: types2.Removed, Path: node.Path, - NodeValue: node.NodeValue, + NodeValue: []byte{}, }); err != nil { return err } @@ -593,7 +677,7 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStora } // buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A -func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { +func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) { return nil } @@ -607,22 +691,23 @@ func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common return err } - diffPathsAtB, err := sdb.createdAndUpdatedStorage( + diffSlotsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStorage( oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - watchedStorageKeys, intermediateNodes, output) + intermediateNodes, output) if err != nil { return err } err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), - diffPathsAtB, watchedStorageKeys, intermediateNodes, output) + diffSlotsAtB, diffPathsAtB, intermediateNodes, output) if err != nil { return err } return nil } -func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output types2.StorageNodeSink) (map[string]bool, error) { +func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) (map[string]bool, map[string]bool, error) { diffPathsAtB := make(map[string]bool) + diffSlotsAtB := make(map[string]bool) it, _ := trie.NewDifferenceIterator(a, b) for it.Next(true) { // skip value nodes @@ -631,7 +716,7 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys } node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.stateCache.TrieDB()) if err != nil { - return nil, err + return nil, nil, err } switch node.NodeType { case types2.Leaf: @@ -639,15 +724,14 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys valueNodePath := append(node.Path, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - if isWatchedStorageKey(watchedKeys, leafKey) { - if err := output(types2.StorageNode{ - NodeType: node.NodeType, - Path: node.Path, - NodeValue: node.NodeValue, - LeafKey: leafKey, - }); err != nil { - return nil, err - } + diffSlotsAtB[common.Bytes2Hex(leafKey)] = true + if err := output(types2.StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + }); err != nil { + return nil, nil, err } case types2.Extension, types2.Branch: if intermediateNodes { @@ -656,18 +740,18 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys Path: node.Path, NodeValue: node.NodeValue, }); err != nil { - return nil, err + return nil, nil, err } } default: - return nil, fmt.Errorf("unexpected node type %s", node.NodeType) + return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType) } diffPathsAtB[common.Bytes2Hex(node.Path)] = true } - return diffPathsAtB, it.Error() + return diffSlotsAtB, diffPathsAtB, it.Error() } -func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { +func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB, diffPathsAtB map[string]bool, intermediateNodes bool, output types2.StorageNodeSink) error { it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { // skip value nodes @@ -678,36 +762,53 @@ func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB if err != nil { return err } - // if this node path showed up in diffPathsAtB - // that means this node was updated at B and we already have the updated diff for it - // otherwise that means this node was deleted in B and we need to add a "removed" diff to represent that event - if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; ok { - continue - } + switch node.NodeType { case types2.Leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(node.Path, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - if isWatchedStorageKey(watchedKeys, leafKey) { - if err := output(types2.StorageNode{ - NodeType: types2.Removed, - Path: node.Path, - NodeValue: []byte{}, - LeafKey: leafKey, - }); err != nil { - return err + + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted (or moved) in B + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + // if this node's leaf key also did not show up in diffSlotsAtB + // that means the node was deleted + // in that case, emit an empty "removed" diff storage node + if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + LeafKey: leafKey, + }); err != nil { + return err + } + } else { + // emit an empty "removed" diff with empty leaf key if the account was moved + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } } } case types2.Extension, types2.Branch: - if intermediateNodes { - if err := output(types2.StorageNode{ - NodeType: types2.Removed, - Path: node.Path, - NodeValue: []byte{}, - }); err != nil { - return err + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted in B + // in that case, emit an empty "removed" diff storage node + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + if intermediateNodes { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } } } default: @@ -718,30 +819,12 @@ func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB } // isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch -func isWatchedAddress(watchedAddresses []common.Address, stateLeafKey []byte) bool { +func isWatchedAddress(watchedAddressesLeafKeys map[common.Hash]struct{}, stateLeafKey []byte) bool { // If we aren't watching any specific addresses, we are watching everything - if len(watchedAddresses) == 0 { + if len(watchedAddressesLeafKeys) == 0 { return true } - for _, addr := range watchedAddresses { - addrHashKey := crypto.Keccak256(addr.Bytes()) - if bytes.Equal(addrHashKey, stateLeafKey) { - return true - } - } - return false -} -// isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch -func isWatchedStorageKey(watchedKeys []common.Hash, storageLeafKey []byte) bool { - // If we aren't watching any specific addresses, we are watching everything - if len(watchedKeys) == 0 { - return true - } - for _, hashKey := range watchedKeys { - if bytes.Equal(hashKey.Bytes(), storageLeafKey) { - return true - } - } - return false + _, ok := watchedAddressesLeafKeys[common.BytesToHash(stateLeafKey)] + return ok } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index d4d67940e32f..3e8e6bdd9c44 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -58,237 +58,233 @@ var ( slot2StorageValue = common.Hex2Bytes("09") slot3StorageValue = common.Hex2Bytes("03") - slot0StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + slot0StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), slot0StorageValue, }) - slot1StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + slot1StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf6"), slot1StorageValue, }) - slot2StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + slot2StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("305787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace"), slot2StorageValue, }) - slot3StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + slot3StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("32575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b"), slot3StorageValue, }) - slot0StorageLeafRootNode, _ = rlp.EncodeToBytes([]interface{}{ - common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), - slot0StorageValue, - }) - contractAccountAtBlock2, _ = rlp.EncodeToBytes(types.StateAccount{ + contractAccountAtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 1, Balance: big.NewInt(0), CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), Root: crypto.Keccak256Hash(block2StorageBranchRootNode), }) - contractAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + contractAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), contractAccountAtBlock2, }) - contractAccountAtBlock3, _ = rlp.EncodeToBytes(types.StateAccount{ + contractAccountAtBlock3, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 1, Balance: big.NewInt(0), CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), Root: crypto.Keccak256Hash(block3StorageBranchRootNode), }) - contractAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + contractAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), contractAccountAtBlock3, }) - contractAccountAtBlock4, _ = rlp.EncodeToBytes(types.StateAccount{ + contractAccountAtBlock4, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 1, Balance: big.NewInt(0), CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), Root: crypto.Keccak256Hash(block4StorageBranchRootNode), }) - contractAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + contractAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), contractAccountAtBlock4, }) - contractAccountAtBlock5, _ = rlp.EncodeToBytes(types.StateAccount{ + contractAccountAtBlock5, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 1, Balance: big.NewInt(0), CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), - Root: crypto.Keccak256Hash(slot0StorageLeafRootNode), + Root: crypto.Keccak256Hash(block5StorageBranchRootNode), }) - contractAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + contractAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), contractAccountAtBlock5, }) - minerAccountAtBlock1, _ = rlp.EncodeToBytes(types.StateAccount{ + minerAccountAtBlock1, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: big.NewInt(2000002625000000000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - minerAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + minerAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), minerAccountAtBlock1, }) - minerAccountAtBlock2, _ = rlp.EncodeToBytes(types.StateAccount{ + minerAccountAtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: big.NewInt(4000111203461610525), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - minerAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + minerAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), minerAccountAtBlock2, }) - account1AtBlock1, _ = rlp.EncodeToBytes(types.StateAccount{ + account1AtBlock1, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: test_helpers.Block1Account1Balance, CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - account1AtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account1AtBlock1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), account1AtBlock1, }) - account1AtBlock2, _ = rlp.EncodeToBytes(types.StateAccount{ + account1AtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 2, Balance: big.NewInt(999555797000009000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - account1AtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account1AtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), account1AtBlock2, }) - account1AtBlock5, _ = rlp.EncodeToBytes(types.StateAccount{ + account1AtBlock5, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 2, - Balance: big.NewInt(2999566008847709960), + Balance: big.NewInt(2999586469962854280), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - account1AtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account1AtBlock5LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), account1AtBlock5, }) - account1AtBlock6, _ = rlp.EncodeToBytes(types.StateAccount{ + account1AtBlock6, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 3, - Balance: big.NewInt(2999537516847709960), + Balance: big.NewInt(2999557977962854280), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - account1AtBlock6LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account1AtBlock6LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), account1AtBlock6, }) - account2AtBlock2, _ = rlp.EncodeToBytes(types.StateAccount{ + account2AtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: big.NewInt(1000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - account2AtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account2AtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), account2AtBlock2, }) - account2AtBlock3, _ = rlp.EncodeToBytes(types.StateAccount{ + account2AtBlock3, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: big.NewInt(2000013574009435976), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - account2AtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account2AtBlock3LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), account2AtBlock3, }) - account2AtBlock4, _ = rlp.EncodeToBytes(types.StateAccount{ + account2AtBlock4, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: big.NewInt(4000048088163070348), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - account2AtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account2AtBlock4LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), account2AtBlock4, }) - account2AtBlock6, _ = rlp.EncodeToBytes(types.StateAccount{ + account2AtBlock6, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, - Balance: big.NewInt(6000063293259748636), + Balance: big.NewInt(6000063258066544204), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - account2AtBlock6LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account2AtBlock6LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), account2AtBlock6, }) - bankAccountAtBlock0, _ = rlp.EncodeToBytes(types.StateAccount{ + bankAccountAtBlock0, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: big.NewInt(test_helpers.TestBankFunds.Int64()), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - bankAccountAtBlock0LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountAtBlock0LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock0, }) block1BankBalance = big.NewInt(test_helpers.TestBankFunds.Int64() - test_helpers.BalanceChange10000 - test_helpers.GasFees) - bankAccountAtBlock1, _ = rlp.EncodeToBytes(types.StateAccount{ + bankAccountAtBlock1, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 1, Balance: block1BankBalance, CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - bankAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock1, }) block2BankBalance = block1BankBalance.Int64() - test_helpers.BalanceChange1Ether - test_helpers.GasFees - bankAccountAtBlock2, _ = rlp.EncodeToBytes(types.StateAccount{ + bankAccountAtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 2, Balance: big.NewInt(block2BankBalance), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - bankAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock2, }) - bankAccountAtBlock3, _ = rlp.EncodeToBytes(types.StateAccount{ + bankAccountAtBlock3, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 3, Balance: big.NewInt(999914255999990000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - bankAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock3, }) - bankAccountAtBlock4, _ = rlp.EncodeToBytes(types.StateAccount{ + bankAccountAtBlock4, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 6, Balance: big.NewInt(999826859999990000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - bankAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock4, }) - bankAccountAtBlock5, _ = rlp.EncodeToBytes(types.StateAccount{ - Nonce: 7, - Balance: big.NewInt(999805027999990000), + bankAccountAtBlock5, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 8, + Balance: big.NewInt(999761283999990000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - bankAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock5, }) - block1BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block1BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ crypto.Keccak256(bankAccountAtBlock1LeafNode), []byte{}, []byte{}, @@ -307,7 +303,7 @@ var ( []byte{}, []byte{}, }) - block2BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block2BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ crypto.Keccak256(bankAccountAtBlock2LeafNode), []byte{}, []byte{}, @@ -326,7 +322,7 @@ var ( []byte{}, []byte{}, }) - block3BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block3BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ crypto.Keccak256(bankAccountAtBlock3LeafNode), []byte{}, []byte{}, @@ -345,7 +341,7 @@ var ( []byte{}, []byte{}, }) - block4BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block4BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ crypto.Keccak256(bankAccountAtBlock4LeafNode), []byte{}, []byte{}, @@ -364,7 +360,7 @@ var ( []byte{}, []byte{}, }) - block5BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block5BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ crypto.Keccak256(bankAccountAtBlock5LeafNode), []byte{}, []byte{}, @@ -383,7 +379,7 @@ var ( []byte{}, []byte{}, }) - block6BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block6BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ crypto.Keccak256(bankAccountAtBlock5LeafNode), []byte{}, []byte{}, @@ -403,7 +399,7 @@ var ( []byte{}, }) - block2StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block2StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ []byte{}, []byte{}, crypto.Keccak256(slot0StorageLeafNode), @@ -422,7 +418,7 @@ var ( []byte{}, []byte{}, }) - block3StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block3StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ []byte{}, []byte{}, crypto.Keccak256(slot0StorageLeafNode), @@ -441,7 +437,7 @@ var ( []byte{}, []byte{}, }) - block4StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block4StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ []byte{}, []byte{}, crypto.Keccak256(slot0StorageLeafNode), @@ -460,6 +456,25 @@ var ( []byte{}, []byte{}, }) + block5StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot3StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) ) func init() { @@ -683,7 +698,7 @@ func TestBuilder(t *testing.T) { if err != nil { t.Error(err) } - receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } @@ -946,7 +961,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { if err != nil { t.Error(err) } - receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } @@ -988,6 +1003,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { params := statediff.Params{ WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr}, } + params.ComputeWatchedAddressesLeafKeys() builder = statediff.NewBuilder(chain.StateCache()) var tests = []struct { @@ -1135,7 +1151,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { if err != nil { t.Error(err) } - receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } @@ -1152,17 +1168,17 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { } } -func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { - blocks, chain := test_helpers.MakeChain(3, test_helpers.Genesis, test_helpers.TestChainGen) +func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { + blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) defer chain.Stop() - block0 = test_helpers.Genesis - block1 = blocks[0] - block2 = blocks[1] block3 = blocks[2] + block4 = blocks[3] + block5 = blocks[4] + block6 = blocks[5] params := statediff.Params{ - WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr}, - WatchedStorageSlots: []common.Hash{slot1StorageKey}, + IntermediateStateNodes: true, + IntermediateStorageNodes: true, } builder = statediff.NewBuilder(chain.StateCache()) @@ -1171,83 +1187,120 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { startingArguments statediff.Args expected *types2.StateObject }{ + // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes { - "testEmptyDiff", - statediff.Args{ - OldStateRoot: block0.Root(), - NewStateRoot: block0.Root(), - BlockNumber: block0.Number(), - BlockHash: block0.Hash(), - }, - &types2.StateObject{ - BlockNumber: block0.Number(), - BlockHash: block0.Hash(), - Nodes: emptyDiffs, - }, - }, - { - "testBlock0", - //10000 transferred from testBankAddress to account1Addr - statediff.Args{ - OldStateRoot: test_helpers.NullHash, - NewStateRoot: block0.Root(), - BlockNumber: block0.Number(), - BlockHash: block0.Hash(), - }, - &types2.StateObject{ - BlockNumber: block0.Number(), - BlockHash: block0.Hash(), - Nodes: emptyDiffs, - }, - }, - { - "testBlock1", - //10000 transferred from testBankAddress to account1Addr + "testBlock4", statediff.Args{ - OldStateRoot: block0.Root(), - NewStateRoot: block1.Root(), - BlockNumber: block1.Number(), - BlockHash: block1.Hash(), + OldStateRoot: block3.Root(), + NewStateRoot: block4.Root(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), }, &types2.StateObject{ - BlockNumber: block1.Number(), - BlockHash: block1.Hash(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), Nodes: []types2.StateNode{ { - Path: []byte{'\x0e'}, + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block4BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, NodeType: types2.Leaf, - LeafKey: test_helpers.Account1LeafKey, - NodeValue: account1AtBlock1LeafNode, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock4LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block4StorageBranchRootNode, + }, + { + Path: []byte{'\x04'}, + NodeType: types2.Leaf, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: slot2StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock4LeafNode, StorageNodes: emptyStorage, }, }, }, }, { - "testBlock2", - //1000 transferred from testBankAddress to account1Addr - //1000 transferred from account1Addr to account2Addr + "testBlock5", statediff.Args{ - OldStateRoot: block1.Root(), - NewStateRoot: block2.Root(), - BlockNumber: block2.Number(), - BlockHash: block2.Hash(), + OldStateRoot: block4.Root(), + NewStateRoot: block5.Root(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), }, &types2.StateObject{ - BlockNumber: block2.Number(), - BlockHash: block2.Hash(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block5BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock5LeafNode, + StorageNodes: emptyStorage, + }, { Path: []byte{'\x06'}, NodeType: types2.Leaf, LeafKey: contractLeafKey, - NodeValue: contractAccountAtBlock2LeafNode, + NodeValue: contractAccountAtBlock5LeafNode, StorageNodes: []types2.StorageNode{ { - Path: []byte{'\x0b'}, + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block5StorageBranchRootNode, + }, + { + Path: []byte{'\x0c'}, NodeType: types2.Leaf, - LeafKey: slot1StorageKey.Bytes(), - NodeValue: slot1StorageLeafNode, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + { + Path: []byte{'\x04'}, + NodeType: types2.Removed, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: []byte{}, }, }, }, @@ -1255,37 +1308,67 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { Path: []byte{'\x0e'}, NodeType: types2.Leaf, LeafKey: test_helpers.Account1LeafKey, - NodeValue: account1AtBlock2LeafNode, + NodeValue: account1AtBlock5LeafNode, StorageNodes: emptyStorage, }, }, - CodeAndCodeHashes: []types2.CodeAndCodeHash{ - { - Hash: test_helpers.CodeHash, - Code: test_helpers.ByteCodeAfterDeployment, - }, - }, }, }, { - "testBlock3", - //the contract's storage is changed - //and the block is mined by account 2 + "testBlock6", statediff.Args{ - OldStateRoot: block2.Root(), - NewStateRoot: block3.Root(), - BlockNumber: block3.Number(), - BlockHash: block3.Hash(), + OldStateRoot: block5.Root(), + NewStateRoot: block6.Root(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), }, &types2.StateObject{ - BlockNumber: block3.Number(), - BlockHash: block3.Hash(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), Nodes: []types2.StateNode{ { - Path: []byte{'\x06'}, + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block6BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Removed, + LeafKey: contractLeafKey, + NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, NodeType: types2.Leaf, - LeafKey: contractLeafKey, - NodeValue: contractAccountAtBlock3LeafNode, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock6LeafNode, StorageNodes: emptyStorage, }, }, @@ -1298,7 +1381,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { if err != nil { t.Error(err) } - receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } @@ -1310,12 +1393,12 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { t.Logf("Test failed: %s", test.name) - t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) } } } -func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { +func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.T) { blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) defer chain.Stop() @@ -1324,8 +1407,8 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { block5 = blocks[4] block6 = blocks[5] params := statediff.Params{ - IntermediateStateNodes: true, - IntermediateStorageNodes: true, + IntermediateStateNodes: false, + IntermediateStorageNodes: false, } builder = statediff.NewBuilder(chain.StateCache()) @@ -1347,12 +1430,6 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { BlockNumber: block4.Number(), BlockHash: block4.Hash(), Nodes: []types2.StateNode{ - { - Path: []byte{}, - NodeType: types2.Branch, - NodeValue: block4BranchRootNode, - StorageNodes: emptyStorage, - }, { Path: []byte{'\x00'}, NodeType: types2.Leaf, @@ -1366,11 +1443,6 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock4LeafNode, StorageNodes: []types2.StorageNode{ - { - Path: []byte{}, - NodeType: types2.Branch, - NodeValue: block4StorageBranchRootNode, - }, { Path: []byte{'\x04'}, NodeType: types2.Leaf, @@ -1413,12 +1485,6 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { BlockNumber: block5.Number(), BlockHash: block5.Hash(), Nodes: []types2.StateNode{ - { - Path: []byte{}, - NodeType: types2.Branch, - NodeValue: block5BranchRootNode, - StorageNodes: emptyStorage, - }, { Path: []byte{'\x00'}, NodeType: types2.Leaf, @@ -1433,16 +1499,10 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { NodeValue: contractAccountAtBlock5LeafNode, StorageNodes: []types2.StorageNode{ { - Path: []byte{}, + Path: []byte{'\x0c'}, NodeType: types2.Leaf, - NodeValue: slot0StorageLeafRootNode, - LeafKey: slot0StorageKey.Bytes(), - }, - { - Path: []byte{'\x02'}, - NodeType: types2.Removed, - LeafKey: slot0StorageKey.Bytes(), - NodeValue: []byte{}, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, }, { Path: []byte{'\x04'}, @@ -1475,17 +1535,24 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { BlockHash: block6.Hash(), Nodes: []types2.StateNode{ { - Path: []byte{}, - NodeType: types2.Branch, - NodeValue: block6BranchRootNode, - StorageNodes: emptyStorage, - }, - { - Path: []byte{'\x06'}, - NodeType: types2.Removed, - LeafKey: contractLeafKey, - NodeValue: []byte{}, - StorageNodes: emptyStorage, + Path: []byte{'\x06'}, + NodeType: types2.Removed, + LeafKey: contractLeafKey, + NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, }, { Path: []byte{'\x0c'}, @@ -1511,14 +1578,16 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { if err != nil { t.Error(err) } - receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) if err != nil { t.Error(err) } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { @@ -1528,7 +1597,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { } } -func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.T) { +func TestBuilderWithRemovedNonWatchedAccount(t *testing.T) { blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) defer chain.Stop() @@ -1537,9 +1606,9 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. block5 = blocks[4] block6 = blocks[5] params := statediff.Params{ - IntermediateStateNodes: false, - IntermediateStorageNodes: false, + WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.Account2Addr}, } + params.ComputeWatchedAddressesLeafKeys() builder = statediff.NewBuilder(chain.StateCache()) var tests = []struct { @@ -1547,7 +1616,6 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. startingArguments statediff.Args expected *types2.StateObject }{ - // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes { "testBlock4", statediff.Args{ @@ -1561,12 +1629,123 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. BlockHash: block4.Hash(), Nodes: []types2.StateNode{ { - Path: []byte{'\x00'}, + Path: []byte{'\x0c'}, NodeType: types2.Leaf, - LeafKey: test_helpers.BankLeafKey, - NodeValue: bankAccountAtBlock4LeafNode, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock5", + statediff.Args{ + OldStateRoot: block4.Root(), + NewStateRoot: block5.Root(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + }, + &types2.StateObject{ + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock6", + statediff.Args{ + OldStateRoot: block5.Root(), + NewStateRoot: block6.Root(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + }, + &types2.StateObject{ + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock6LeafNode, StorageNodes: emptyStorage, }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithRemovedWatchedAccount(t *testing.T) { + blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block3 = blocks[2] + block4 = blocks[3] + block5 = blocks[4] + block6 = blocks[5] + params := statediff.Params{ + WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr}, + } + params.ComputeWatchedAddressesLeafKeys() + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + { + "testBlock4", + statediff.Args{ + OldStateRoot: block3.Root(), + NewStateRoot: block4.Root(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + }, + &types2.StateObject{ + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + Nodes: []types2.StateNode{ { Path: []byte{'\x06'}, NodeType: types2.Leaf, @@ -1593,13 +1772,6 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. }, }, }, - { - Path: []byte{'\x0c'}, - NodeType: types2.Leaf, - LeafKey: test_helpers.Account2LeafKey, - NodeValue: account2AtBlock4LeafNode, - StorageNodes: emptyStorage, - }, }, }, }, @@ -1615,13 +1787,6 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. BlockNumber: block5.Number(), BlockHash: block5.Hash(), Nodes: []types2.StateNode{ - { - Path: []byte{'\x00'}, - NodeType: types2.Leaf, - LeafKey: test_helpers.BankLeafKey, - NodeValue: bankAccountAtBlock5LeafNode, - StorageNodes: emptyStorage, - }, { Path: []byte{'\x06'}, NodeType: types2.Leaf, @@ -1629,16 +1794,10 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. NodeValue: contractAccountAtBlock5LeafNode, StorageNodes: []types2.StorageNode{ { - Path: []byte{}, + Path: []byte{'\x0c'}, NodeType: types2.Leaf, - LeafKey: slot0StorageKey.Bytes(), - NodeValue: slot0StorageLeafRootNode, - }, - { - Path: []byte{'\x02'}, - NodeType: types2.Removed, - LeafKey: slot0StorageKey.Bytes(), - NodeValue: []byte{}, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, }, { Path: []byte{'\x04'}, @@ -1675,13 +1834,20 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. NodeType: types2.Removed, LeafKey: contractLeafKey, NodeValue: []byte{}, - }, - { - Path: []byte{'\x0c'}, - NodeType: types2.Leaf, - LeafKey: test_helpers.Account2LeafKey, - NodeValue: account2AtBlock6LeafNode, - StorageNodes: emptyStorage, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, }, { Path: []byte{'\x0e'}, @@ -1700,7 +1866,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. if err != nil { t.Error(err) } - receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } @@ -1722,44 +1888,44 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. var ( slot00StorageValue = common.Hex2Bytes("9471562b71999873db5b286df957af199ec94617f7") // prefixed TestBankAddress - slot00StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + slot00StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), slot00StorageValue, }) - contractAccountAtBlock01, _ = rlp.EncodeToBytes(types.StateAccount{ + contractAccountAtBlock01, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 1, Balance: big.NewInt(0), CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), Root: crypto.Keccak256Hash(block01StorageBranchRootNode), }) - contractAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + contractAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3cb2583748c26e89ef19c2a8529b05a270f735553b4d44b6f2a1894987a71c8b"), contractAccountAtBlock01, }) - bankAccountAtBlock01, _ = rlp.EncodeToBytes(types.StateAccount{ + bankAccountAtBlock01, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 1, Balance: big.NewInt(3999629697375000000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - bankAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock01, }) - bankAccountAtBlock02, _ = rlp.EncodeToBytes(types.StateAccount{ + bankAccountAtBlock02, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 2, Balance: big.NewInt(5999607323457344852), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - bankAccountAtBlock02LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountAtBlock02LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock02, }) - block01BranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block01BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ crypto.Keccak256Hash(bankAccountAtBlock01LeafNode), crypto.Keccak256Hash(contractAccountAtBlock01LeafNode), []byte{}, @@ -1779,7 +1945,7 @@ var ( []byte{}, }) - block01StorageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{ + block01StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ []byte{}, []byte{}, crypto.Keccak256(slot00StorageLeafNode), @@ -1901,11 +2067,26 @@ func TestBuilderWithMovedAccount(t *testing.T) { NodeType: types2.Removed, LeafKey: contractLeafKey, NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Removed, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + }, + }, }, { Path: []byte{'\x00'}, NodeType: types2.Removed, - LeafKey: test_helpers.BankLeafKey, NodeValue: []byte{}, }, }, @@ -1918,7 +2099,7 @@ func TestBuilderWithMovedAccount(t *testing.T) { if err != nil { t.Error(err) } - receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } @@ -2026,11 +2207,22 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { NodeType: types2.Removed, LeafKey: contractLeafKey, NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + }, + }, }, { Path: []byte{'\x00'}, NodeType: types2.Removed, - LeafKey: test_helpers.BankLeafKey, NodeValue: []byte{}, }, }, @@ -2043,7 +2235,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { if err != nil { t.Error(err) } - receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } @@ -2274,7 +2466,7 @@ func TestBuildStateTrie(t *testing.T) { if err != nil { t.Error(err) } - receivedStateTrieRlp, err := rlp.EncodeToBytes(diff) + receivedStateTrieRlp, err := rlp.EncodeToBytes(&diff) if err != nil { t.Error(err) } diff --git a/statediff/config.go b/statediff/config.go index dc9da579b99d..b4905ab5a75b 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -19,14 +19,19 @@ package statediff import ( "context" "math/big" + "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" ) // Config contains instantiation parameters for the state diffing service type Config struct { + // The configuration used for the stateDiff Indexer IndexerConfig interfaces.Config + // The filepath to write knownGaps insert statements if we can't connect to the DB. + KnownGapsFilePath string // A unique ID used for this service ID string // Name for the client this service is running @@ -35,6 +40,8 @@ type Config struct { EnableWriteLoop bool // Size of the worker pool NumWorkers uint + // Should the statediff service wait until geth has synced to the head of the blockchain? + WaitForSync bool // Context Context context.Context } @@ -48,7 +55,21 @@ type Params struct { IncludeTD bool IncludeCode bool WatchedAddresses []common.Address - WatchedStorageSlots []common.Hash + watchedAddressesLeafKeys map[common.Hash]struct{} +} + +// ComputeWatchedAddressesLeafKeys populates a map with keys (Keccak256Hash) of each of the WatchedAddresses +func (p *Params) ComputeWatchedAddressesLeafKeys() { + p.watchedAddressesLeafKeys = make(map[common.Hash]struct{}, len(p.WatchedAddresses)) + for _, address := range p.WatchedAddresses { + p.watchedAddressesLeafKeys[crypto.Keccak256Hash(address.Bytes())] = struct{}{} + } +} + +// ParamsWithMutex allows to lock the parameters while they are being updated | read from +type ParamsWithMutex struct { + Params + sync.RWMutex } // Args bundles the arguments for the state diff builder diff --git a/statediff/docs/KnownGaps.md b/statediff/docs/KnownGaps.md new file mode 100644 index 000000000000..72e712f472c1 --- /dev/null +++ b/statediff/docs/KnownGaps.md @@ -0,0 +1,17 @@ +# Overview + +This document will provide some insight into the `known_gaps` table, their use cases, and implementation. Please refer to the [following PR](https://github.com/vulcanize/go-ethereum/pull/217) and the [following epic](https://github.com/vulcanize/ops/issues/143) to grasp their inception. + +![known gaps](diagrams/KnownGapsProcess.png) + +# Use Cases + +The known gaps table is updated when the following events occur: + +1. At start up we check the latest block from the `eth.headers_cid` table. We compare the first block that we are processing with the latest block from the DB. If they are not one unit of expectedDifference away from each other, add the gap between the two blocks. +2. If there is any error in processing a block (db connection, deadlock, etc), add that block to the knownErrorBlocks slice, when the next block is successfully written, write this slice into the DB. + +# Glossary + +1. `expectedDifference (number)` - This number indicates what the difference between two blocks should be. If we are capturing all events on a geth node then this number would be `1`. But once we scale nodes, the `expectedDifference` might be `2` or greater. +2. `processingKey (number)` - This number can be used to keep track of different geth nodes and their specific `expectedDifference`. diff --git a/statediff/docs/README.md b/statediff/docs/README.md new file mode 100644 index 000000000000..51b63e20747a --- /dev/null +++ b/statediff/docs/README.md @@ -0,0 +1,3 @@ +# Overview + +This folder keeps tracks of random documents as they relate to the `statediff` service. diff --git a/statediff/docs/database.md b/statediff/docs/database.md new file mode 100644 index 000000000000..847bc8fa2469 --- /dev/null +++ b/statediff/docs/database.md @@ -0,0 +1,21 @@ +# Overview + +This document will go through some notes on the database component of the statediff service. + +# Components + +- Indexer: The indexer creates IPLD and DB models to insert to the Postgres DB. It performs the insert utilizing and atomic function. +- Builder: The builder constructs the statediff object that needs to be inserted. +- Known Gaps: Captures any gaps that might have occured and either writes them to the DB, local sql file, to prometeus, or a local error. + +# Making Code Changes + +## Adding a New Function to the Indexer + +If you want to implement a new feature for adding data to the database. Keep the following in mind: + +1. You need to handle `sql`, `file`, and `dump`. + 1. `sql` - Contains the code needed to write directly to the `sql` db. + 2. `file` - Contains all the code required to write the SQL statements to a file. + 3. `dump` - Contains all the code for outputting events to the console. +2. You will have to add it to the `interfaces.StateDiffIndexer` interface. diff --git a/statediff/docs/diagrams/KnownGapsProcess.png b/statediff/docs/diagrams/KnownGapsProcess.png new file mode 100644 index 000000000000..40ebaa80aad2 Binary files /dev/null and b/statediff/docs/diagrams/KnownGapsProcess.png differ diff --git a/statediff/indexer/constructor.go b/statediff/indexer/constructor.go index 9a66dba8966d..1a4f640019c7 100644 --- a/statediff/indexer/constructor.go +++ b/statediff/indexer/constructor.go @@ -31,22 +31,23 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/shared" ) -// NewStateDiffIndexer creates and returns an implementation of the StateDiffIndexer interface -func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, nodeInfo node.Info, config interfaces.Config) (interfaces.StateDiffIndexer, error) { +// NewStateDiffIndexer creates and returns an implementation of the StateDiffIndexer interface. +func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, nodeInfo node.Info, config interfaces.Config) (sql.Database, interfaces.StateDiffIndexer, error) { switch config.Type() { case shared.FILE: log.Info("Starting statediff service in SQL file writing mode") fc, ok := config.(file.Config) if !ok { - return nil, fmt.Errorf("file config is not the correct type: got %T, expected %T", config, file.Config{}) + return nil, nil, fmt.Errorf("file config is not the correct type: got %T, expected %T", config, file.Config{}) } fc.NodeInfo = nodeInfo - return file.NewStateDiffIndexer(ctx, chainConfig, fc) + ind, err := file.NewStateDiffIndexer(ctx, chainConfig, fc) + return nil, ind, err case shared.POSTGRES: log.Info("Starting statediff service in Postgres writing mode") pgc, ok := config.(postgres.Config) if !ok { - return nil, fmt.Errorf("postgres config is not the correct type: got %T, expected %T", config, postgres.Config{}) + return nil, nil, fmt.Errorf("postgres config is not the correct type: got %T, expected %T", config, postgres.Config{}) } var err error var driver sql.Driver @@ -54,25 +55,27 @@ func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, n case postgres.PGX: driver, err = postgres.NewPGXDriver(ctx, pgc, nodeInfo) if err != nil { - return nil, err + return nil, nil, err } case postgres.SQLX: driver, err = postgres.NewSQLXDriver(ctx, pgc, nodeInfo) if err != nil { - return nil, err + return nil, nil, err } default: - return nil, fmt.Errorf("unrecongized Postgres driver type: %s", pgc.Driver) + return nil, nil, fmt.Errorf("unrecognized Postgres driver type: %s", pgc.Driver) } - return sql.NewStateDiffIndexer(ctx, chainConfig, postgres.NewPostgresDB(driver)) + db := postgres.NewPostgresDB(driver) + ind, err := sql.NewStateDiffIndexer(ctx, chainConfig, db) + return db, ind, err case shared.DUMP: log.Info("Starting statediff service in data dump mode") dumpc, ok := config.(dump.Config) if !ok { - return nil, fmt.Errorf("dump config is not the correct type: got %T, expected %T", config, dump.Config{}) + return nil, nil, fmt.Errorf("dump config is not the correct type: got %T, expected %T", config, dump.Config{}) } - return dump.NewStateDiffIndexer(chainConfig, dumpc), nil + return nil, dump.NewStateDiffIndexer(chainConfig, dumpc), nil default: - return nil, fmt.Errorf("unrecognized database type: %s", config.Type()) + return nil, nil, fmt.Errorf("unrecognized database type: %s", config.Type()) } } diff --git a/statediff/indexer/database/dump/indexer.go b/statediff/indexer/database/dump/indexer.go index e450f941ac2f..42fc8488a6f1 100644 --- a/statediff/indexer/database/dump/indexer.go +++ b/statediff/indexer/database/dump/indexer.go @@ -382,10 +382,11 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return fmt.Errorf("sql batch is expected to be of type %T, got %T", &BatchTx{}, batch) } // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { // short circuit if it is a Removed node // this assumes the db has been initialized and a public.blocks entry for the Removed node is present - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ HeaderID: headerID, Path: stateNode.Path, StateKey: common.BytesToHash(stateNode.LeafKey).String(), @@ -393,25 +394,26 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel) - return err - } - stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node, collect the stateID to reference by FK if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel); err != nil { return err } + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -437,6 +439,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return err } } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed { @@ -496,3 +499,28 @@ func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAnd func (sdi *StateDiffIndexer) Close() error { return sdi.dump.Close() } + +// LoadWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) { + return nil, nil +} + +// InsertWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + return nil +} + +// RemoveWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error { + return nil +} + +// SetWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + return nil +} + +// ClearWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) ClearWatchedAddresses() error { + return nil +} diff --git a/statediff/indexer/database/file/config.go b/statediff/indexer/database/file/config.go index c2c6804c0f97..a075e896b3d7 100644 --- a/statediff/indexer/database/file/config.go +++ b/statediff/indexer/database/file/config.go @@ -23,8 +23,9 @@ import ( // Config holds params for writing sql statements out to a file type Config struct { - FilePath string - NodeInfo node.Info + FilePath string + WatchedAddressesFilePath string + NodeInfo node.Info } // Type satisfies interfaces.Config @@ -34,7 +35,8 @@ func (c Config) Type() shared.DBType { // TestConfig config for unit tests var TestConfig = Config{ - FilePath: "./statediffing_test_file.sql", + FilePath: "./statediffing_test_file.sql", + WatchedAddressesFilePath: "./statediffing_watched_addresses_test_file.sql", NodeInfo: node.Info{ GenesisBlock: "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", NetworkID: "1", diff --git a/statediff/indexer/database/file/indexer.go b/statediff/indexer/database/file/indexer.go index 045ac238adc7..5ddb3f5b94c5 100644 --- a/statediff/indexer/database/file/indexer.go +++ b/statediff/indexer/database/file/indexer.go @@ -17,6 +17,7 @@ package file import ( + "bufio" "context" "errors" "fmt" @@ -29,6 +30,8 @@ import ( "github.com/ipfs/go-cid" node "github.com/ipfs/go-ipld-format" "github.com/multiformats/go-multihash" + pg_query "github.com/pganalyze/pg_query_go/v2" + "github.com/thoas/go-funk" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -45,6 +48,9 @@ import ( ) const defaultFilePath = "./statediff.sql" +const defaultWatchedAddressesFilePath = "./statediff-watched-addresses.sql" + +const watchedAddressesInsert = "INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ('%s', '%d', '%d') ON CONFLICT (address) DO NOTHING;" var _ interfaces.StateDiffIndexer = &StateDiffIndexer{} @@ -60,6 +66,8 @@ type StateDiffIndexer struct { wg *sync.WaitGroup blockNumber string removedCacheFlag *uint32 + + watchedAddressesFilePath string } // NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer @@ -76,15 +84,23 @@ func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, c return nil, fmt.Errorf("unable to create file (%s), err: %v", filePath, err) } log.Info("Writing statediff SQL statements to file", "file", filePath) + + watchedAddressesFilePath := config.WatchedAddressesFilePath + if watchedAddressesFilePath == "" { + watchedAddressesFilePath = defaultWatchedAddressesFilePath + } + log.Info("Writing watched addresses SQL statements to file", "file", watchedAddressesFilePath) + w := NewSQLWriter(file) wg := new(sync.WaitGroup) w.Loop() w.upsertNode(config.NodeInfo) return &StateDiffIndexer{ - fileWriter: w, - chainConfig: chainConfig, - nodeID: config.NodeInfo.ID, - wg: wg, + fileWriter: w, + chainConfig: chainConfig, + nodeID: config.NodeInfo.ID, + wg: wg, + watchedAddressesFilePath: watchedAddressesFilePath, }, nil } @@ -385,12 +401,13 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error { // PushStateNode writes a state diff node object (including any child storage nodes) IPLD insert SQL stmt to a file func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { if atomic.LoadUint32(sdi.removedCacheFlag) == 0 { atomic.StoreUint32(sdi.removedCacheFlag, 1) sdi.fileWriter.upsertIPLDDirect(sdi.blockNumber, shared.RemovedNodeMhKey, []byte{}) } - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ BlockNumber: sdi.blockNumber, HeaderID: headerID, Path: stateNode.Path, @@ -399,24 +416,25 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - sdi.fileWriter.upsertStateCID(stateModel) - return nil - } - stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(sdi.blockNumber, ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - BlockNumber: sdi.blockNumber, - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(sdi.blockNumber, ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + BlockNumber: sdi.blockNumber, + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node sdi.fileWriter.upsertStateCID(stateModel) + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -441,6 +459,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt } sdi.fileWriter.upsertStateAccount(accountModel) } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed { @@ -496,3 +515,165 @@ func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAnd func (sdi *StateDiffIndexer) Close() error { return sdi.fileWriter.Close() } + +// LoadWatchedAddresses loads watched addresses from a file +func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) { + // load sql statements from watched addresses file + stmts, err := loadWatchedAddressesStatements(sdi.watchedAddressesFilePath) + if err != nil { + return nil, err + } + + // extract addresses from the sql statements + watchedAddresses := []common.Address{} + for _, stmt := range stmts { + addressString, err := parseWatchedAddressStatement(stmt) + if err != nil { + return nil, err + } + watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString)) + } + + return watchedAddresses, nil +} + +// InsertWatchedAddresses inserts the given addresses in a file +func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + // load sql statements from watched addresses file + stmts, err := loadWatchedAddressesStatements(sdi.watchedAddressesFilePath) + if err != nil { + return err + } + + // get already watched addresses + var watchedAddresses []string + for _, stmt := range stmts { + addressString, err := parseWatchedAddressStatement(stmt) + if err != nil { + return err + } + + watchedAddresses = append(watchedAddresses, addressString) + } + + // append statements for new addresses to existing statements + for _, arg := range args { + // ignore if already watched + if funk.Contains(watchedAddresses, arg.Address) { + continue + } + + stmt := fmt.Sprintf(watchedAddressesInsert, arg.Address, arg.CreatedAt, currentBlockNumber.Uint64()) + stmts = append(stmts, stmt) + } + + return dumpWatchedAddressesStatements(sdi.watchedAddressesFilePath, stmts) +} + +// RemoveWatchedAddresses removes the given watched addresses from a file +func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error { + // load sql statements from watched addresses file + stmts, err := loadWatchedAddressesStatements(sdi.watchedAddressesFilePath) + if err != nil { + return err + } + + // get rid of statements having addresses to be removed + var filteredStmts []string + for _, stmt := range stmts { + addressString, err := parseWatchedAddressStatement(stmt) + if err != nil { + return err + } + + toRemove := funk.Contains(args, func(arg sdtypes.WatchAddressArg) bool { + return arg.Address == addressString + }) + + if !toRemove { + filteredStmts = append(filteredStmts, stmt) + } + } + + return dumpWatchedAddressesStatements(sdi.watchedAddressesFilePath, filteredStmts) +} + +// SetWatchedAddresses clears and inserts the given addresses in a file +func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + var stmts []string + for _, arg := range args { + stmt := fmt.Sprintf(watchedAddressesInsert, arg.Address, arg.CreatedAt, currentBlockNumber.Uint64()) + stmts = append(stmts, stmt) + } + + return dumpWatchedAddressesStatements(sdi.watchedAddressesFilePath, stmts) +} + +// ClearWatchedAddresses clears all the watched addresses from a file +func (sdi *StateDiffIndexer) ClearWatchedAddresses() error { + return sdi.SetWatchedAddresses([]sdtypes.WatchAddressArg{}, big.NewInt(0)) +} + +// loadWatchedAddressesStatements loads sql statements from the given file in a string slice +func loadWatchedAddressesStatements(filePath string) ([]string, error) { + file, err := os.Open(filePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return []string{}, nil + } + + return nil, fmt.Errorf("error opening watched addresses file: %v", err) + } + defer file.Close() + + stmts := []string{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + stmts = append(stmts, scanner.Text()) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error loading watched addresses: %v", err) + } + + return stmts, nil +} + +// dumpWatchedAddressesStatements dumps sql statements to the given file +func dumpWatchedAddressesStatements(filePath string, stmts []string) error { + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("error creating watched addresses file: %v", err) + } + defer file.Close() + + for _, stmt := range stmts { + _, err := file.Write([]byte(stmt + "\n")) + if err != nil { + return fmt.Errorf("error inserting watched_addresses entry: %v", err) + } + } + + return nil +} + +// parseWatchedAddressStatement parses given sql insert statement to extract the address argument +func parseWatchedAddressStatement(stmt string) (string, error) { + parseResult, err := pg_query.Parse(stmt) + if err != nil { + return "", fmt.Errorf("error parsing sql stmt: %v", err) + } + + // extract address argument from parse output for a SQL statement of form + // "INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) + // VALUES ('0xabc', '123', '130') ON CONFLICT (address) DO NOTHING;" + addressString := parseResult.Stmts[0].Stmt.GetInsertStmt(). + SelectStmt.GetSelectStmt(). + ValuesLists[0].GetList(). + Items[0].GetAConst(). + GetVal(). + GetString_(). + Str + + return addressString, nil +} diff --git a/statediff/indexer/database/file/indexer_legacy_test.go b/statediff/indexer/database/file/indexer_legacy_test.go index ed4bcf000e74..9f85da2e0c04 100644 --- a/statediff/indexer/database/file/indexer_legacy_test.go +++ b/statediff/indexer/database/file/indexer_legacy_test.go @@ -80,7 +80,7 @@ func setupLegacy(t *testing.T) { } } -func dumpData(t *testing.T) { +func dumpFileData(t *testing.T) { sqlFileBytes, err := os.ReadFile(file.TestConfig.FilePath) require.NoError(t, err) @@ -88,10 +88,36 @@ func dumpData(t *testing.T) { require.NoError(t, err) } +func resetAndDumpWatchedAddressesFileData(t *testing.T) { + resetDB(t) + + sqlFileBytes, err := os.ReadFile(file.TestConfig.WatchedAddressesFilePath) + require.NoError(t, err) + + _, err = sqlxdb.Exec(string(sqlFileBytes)) + require.NoError(t, err) +} + +func resetDB(t *testing.T) { + file.TearDownDB(t, sqlxdb) + + connStr := postgres.DefaultConfig.DbConnectionString() + sqlxdb, err = sqlx.Connect("postgres", connStr) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err) + } +} + func tearDown(t *testing.T) { file.TearDownDB(t, sqlxdb) + err := os.Remove(file.TestConfig.FilePath) require.NoError(t, err) + + if err := os.Remove(file.TestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) { + require.NoError(t, err) + } + err = sqlxdb.Close() require.NoError(t, err) } @@ -105,7 +131,7 @@ func expectTrue(t *testing.T, value bool) { func TestFileIndexerLegacy(t *testing.T) { t.Run("Publish and index header IPLDs", func(t *testing.T) { setupLegacy(t) - dumpData(t) + dumpFileData(t) defer tearDown(t) pgStr := `SELECT cid, td, reward, block_hash, coinbase FROM eth.header_cids diff --git a/statediff/indexer/database/file/indexer_test.go b/statediff/indexer/database/file/indexer_test.go index bb03c1ec399c..9cbc78fe660f 100644 --- a/statediff/indexer/database/file/indexer_test.go +++ b/statediff/indexer/database/file/indexer_test.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "math/big" "os" "testing" @@ -28,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" @@ -51,12 +53,15 @@ var ( ind interfaces.StateDiffIndexer ipfsPgGet = `SELECT data FROM public.blocks WHERE key = $1` - tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte - mockBlock *types.Block - headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid - rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid - rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte - state1CID, state2CID, storageCID cid.Cid + tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte + mockBlock *types.Block + headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid + rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid + rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte + state1CID, state2CID, storageCID cid.Cid + contract1Address, contract2Address, contract3Address, contract4Address string + contract1CreatedAt, contract2CreatedAt, contract3CreatedAt, contract4CreatedAt uint64 + lastFilledAt, watchedAt1, watchedAt2, watchedAt3 uint64 ) func init() { @@ -161,15 +166,45 @@ func init() { rctLeaf3 = orderedRctLeafNodes[2] rctLeaf4 = orderedRctLeafNodes[3] rctLeaf5 = orderedRctLeafNodes[4] + + contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F" + contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B" + contract3Address = "0xcfeB164C328CA13EFd3C77E1980d94975aDfedfc" + contract4Address = "0x0Edf0c4f393a628DE4828B228C48175b3EA297fc" + contract1CreatedAt = uint64(1) + contract2CreatedAt = uint64(2) + contract3CreatedAt = uint64(3) + contract4CreatedAt = uint64(4) + + lastFilledAt = uint64(0) + watchedAt1 = uint64(10) + watchedAt2 = uint64(15) + watchedAt3 = uint64(20) } -func setup(t *testing.T) { +func setupIndexer(t *testing.T) { if _, err := os.Stat(file.TestConfig.FilePath); !errors.Is(err, os.ErrNotExist) { err := os.Remove(file.TestConfig.FilePath) require.NoError(t, err) } + + if _, err := os.Stat(file.TestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) { + err := os.Remove(file.TestConfig.WatchedAddressesFilePath) + require.NoError(t, err) + } + ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.TestConfig) require.NoError(t, err) + + connStr := postgres.DefaultConfig.DbConnectionString() + sqlxdb, err = sqlx.Connect("postgres", connStr) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err) + } +} + +func setup(t *testing.T) { + setupIndexer(t) var tx interfaces.Batch tx, err = ind.PushBlock( mockBlock, @@ -192,19 +227,12 @@ func setup(t *testing.T) { } require.Equal(t, mocks.BlockNumber.String(), tx.(*file.BatchTx).BlockNumber) - - connStr := postgres.DefaultConfig.DbConnectionString() - - sqlxdb, err = sqlx.Connect("postgres", connStr) - if err != nil { - t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err) - } } func TestFileIndexer(t *testing.T) { t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { setup(t) - dumpData(t) + dumpFileData(t) defer tearDown(t) pgStr := `SELECT cid, td, reward, block_hash, coinbase FROM eth.header_cids @@ -242,7 +270,7 @@ func TestFileIndexer(t *testing.T) { }) t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { setup(t) - dumpData(t) + dumpFileData(t) defer tearDown(t) // check that txs were properly indexed and published @@ -374,7 +402,7 @@ func TestFileIndexer(t *testing.T) { t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) { setup(t) - dumpData(t) + dumpFileData(t) defer tearDown(t) rcts := make([]string, 0) @@ -430,7 +458,7 @@ func TestFileIndexer(t *testing.T) { t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) { setup(t) - dumpData(t) + dumpFileData(t) defer tearDown(t) // check receipts were properly indexed and published @@ -531,7 +559,7 @@ func TestFileIndexer(t *testing.T) { t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) { setup(t) - dumpData(t) + dumpFileData(t) defer tearDown(t) // check that state nodes were properly indexed and published @@ -603,28 +631,39 @@ func TestFileIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - require.Equal(t, 1, len(stateNodes)) - stateNode := stateNodes[0] - var data []byte - dc, err := cid.Decode(stateNode.CID) - if err != nil { - t.Fatal(err) - } - mhKey := dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() - require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) - err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) + require.Equal(t, 2, len(stateNodes)) + for idx, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) + err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + + if idx == 0 { + require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) + require.Equal(t, common.BytesToHash(mocks.RemovedLeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x02'}, stateNode.Path) + require.Equal(t, []byte{}, data) + } + if idx == 1 { + require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) + require.Equal(t, common.BytesToHash(mocks.Contract2LeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x07'}, stateNode.Path) + require.Equal(t, []byte{}, data) + } } - require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) - require.Equal(t, []byte{'\x02'}, stateNode.Path) - require.Equal(t, []byte{}, data) }) t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { setup(t) - dumpData(t) + dumpFileData(t) defer tearDown(t) // check that storage nodes were properly indexed @@ -673,26 +712,385 @@ func TestFileIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - require.Equal(t, 1, len(storageNodes)) - require.Equal(t, models.StorageNodeWithStateKeyModel{ - BlockNumber: mocks.BlockNumber.String(), - CID: shared.RemovedNodeStorageCID, - NodeType: 3, - StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), - StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), - Path: []byte{'\x03'}, - }, storageNodes[0]) - dc, err = cid.Decode(storageNodes[0].CID) + require.Equal(t, 3, len(storageNodes)) + expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{'\x03'}, + }, + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage2LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0e'}, + }, + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage3LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0f'}, + }, + } + for idx, storageNode := range storageNodes { + require.Equal(t, expectedStorageNodes[idx], storageNode) + dc, err = cid.Decode(storageNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey = dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) + err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) + if err != nil { + t.Fatal(err) + } + require.Equal(t, []byte{}, data) + } + }) +} + +func TestFileWatchAddressMethods(t *testing.T) { + setupIndexer(t) + defer tearDown(t) + + type res struct { + Address string `db:"address"` + CreatedAt uint64 `db:"created_at"` + WatchedAt uint64 `db:"watched_at"` + LastFilledAt uint64 `db:"last_filled_at"` + } + pgStr := "SELECT * FROM eth_meta.watched_addresses" + + t.Run("Load watched addresses (empty table)", func(t *testing.T) { + expectedData := []common.Address{} + + rows, err := ind.LoadWatchedAddresses() + require.NoError(t, err) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Insert watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(watchedAt1))) + require.NoError(t, err) + resetAndDumpWatchedAddressesFileData(t) + + rows := []res{} + err = sqlxdb.Select(&rows, pgStr) if err != nil { t.Fatal(err) } - mhKey = dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() - require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) - err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Insert watched addresses (some already watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(watchedAt2))) + require.NoError(t, err) + resetAndDumpWatchedAddressesFileData(t) + + rows := []res{} + err = sqlxdb.Select(&rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Remove watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + resetAndDumpWatchedAddressesFileData(t) + + rows := []res{} + err = sqlxdb.Select(&rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{} + + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + resetAndDumpWatchedAddressesFileData(t) + + rows := []res{} + err = sqlxdb.Select(&rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Set watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.SetWatchedAddresses(args, big.NewInt(int64(watchedAt2))) + require.NoError(t, err) + resetAndDumpWatchedAddressesFileData(t) + + rows := []res{} + err = sqlxdb.Select(&rows, pgStr) if err != nil { t.Fatal(err) } - require.Equal(t, []byte{}, data) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Set watched addresses (some already watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract4Address, + CreatedAt: contract4CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract4Address, + CreatedAt: contract4CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.SetWatchedAddresses(args, big.NewInt(int64(watchedAt3))) + require.NoError(t, err) + resetAndDumpWatchedAddressesFileData(t) + + rows := []res{} + err = sqlxdb.Select(&rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Load watched addresses", func(t *testing.T) { + expectedData := []common.Address{ + common.HexToAddress(contract4Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + } + + rows, err := ind.LoadWatchedAddresses() + require.NoError(t, err) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Clear watched addresses", func(t *testing.T) { + expectedData := []res{} + + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + resetAndDumpWatchedAddressesFileData(t) + + rows := []res{} + err = sqlxdb.Select(&rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Clear watched addresses (empty table)", func(t *testing.T) { + expectedData := []res{} + + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + resetAndDumpWatchedAddressesFileData(t) + + rows := []res{} + err = sqlxdb.Select(&rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } }) } diff --git a/statediff/indexer/database/file/test_helpers.go b/statediff/indexer/database/file/test_helpers.go index 27d204d55aea..27a1581a48da 100644 --- a/statediff/indexer/database/file/test_helpers.go +++ b/statediff/indexer/database/file/test_helpers.go @@ -57,6 +57,10 @@ func TearDownDB(t *testing.T, db *sqlx.DB) { if err != nil { t.Fatal(err) } + _, err = tx.Exec(`DELETE FROM eth_meta.watched_addresses`) + if err != nil { + t.Fatal(err) + } err = tx.Commit() if err != nil { t.Fatal(err) diff --git a/statediff/indexer/database/sql/indexer.go b/statediff/indexer/database/sql/indexer.go index 15fcd60b2d62..74ce06745f31 100644 --- a/statediff/indexer/database/sql/indexer.go +++ b/statediff/indexer/database/sql/indexer.go @@ -448,9 +448,10 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return fmt.Errorf("sql batch is expected to be of type %T, got %T", &BatchTx{}, batch) } // publish the state node + var stateModel models.StateNodeModel if stateNode.NodeType == sdtypes.Removed { tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{}) - stateModel := models.StateNodeModel{ + stateModel = models.StateNodeModel{ BlockNumber: sdi.blockNumber, HeaderID: headerID, Path: stateNode.Path, @@ -459,25 +460,27 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: shared.RemovedNodeMhKey, NodeType: stateNode.NodeType.Int(), } - return sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel) - } - stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) - if err != nil { - return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) - } - stateModel := models.StateNodeModel{ - BlockNumber: sdi.blockNumber, - HeaderID: headerID, - Path: stateNode.Path, - StateKey: common.BytesToHash(stateNode.LeafKey).String(), - CID: stateCIDStr, - MhKey: stateMhKey, - NodeType: stateNode.NodeType.Int(), + } else { + stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + BlockNumber: sdi.blockNumber, + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } } + // index the state node if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil { return err } + // if we have a leaf, decode and index the account data if stateNode.NodeType == sdtypes.Leaf { var i []interface{} @@ -504,6 +507,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt return err } } + // if there are any storage nodes associated with this node, publish and index them for _, storageNode := range stateNode.StorageNodes { if storageNode.NodeType == sdtypes.Removed { @@ -564,3 +568,120 @@ func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAnd func (sdi *StateDiffIndexer) Close() error { return sdi.dbWriter.Close() } + +// Update the known gaps table with the gap information. + +// LoadWatchedAddresses reads watched addresses from the database +func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) { + addressStrings := make([]string, 0) + pgStr := "SELECT address FROM eth_meta.watched_addresses" + err := sdi.dbWriter.db.Select(sdi.ctx, &addressStrings, pgStr) + if err != nil { + return nil, fmt.Errorf("error loading watched addresses: %v", err) + } + + watchedAddresses := []common.Address{} + for _, addressString := range addressStrings { + watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString)) + } + + return watchedAddresses, nil +} + +// InsertWatchedAddresses inserts the given addresses in the database +func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + tx, err := sdi.dbWriter.db.Begin(sdi.ctx) + if err != nil { + return err + } + defer func() { + if p := recover(); p != nil { + rollback(sdi.ctx, tx) + panic(p) + } else if err != nil { + rollback(sdi.ctx, tx) + } else { + err = tx.Commit(sdi.ctx) + } + }() + + for _, arg := range args { + _, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`, + arg.Address, arg.CreatedAt, currentBlockNumber.Uint64()) + if err != nil { + return fmt.Errorf("error inserting watched_addresses entry: %v", err) + } + } + + return err +} + +// RemoveWatchedAddresses removes the given watched addresses from the database +func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error { + tx, err := sdi.dbWriter.db.Begin(sdi.ctx) + if err != nil { + return err + } + defer func() { + if p := recover(); p != nil { + rollback(sdi.ctx, tx) + panic(p) + } else if err != nil { + rollback(sdi.ctx, tx) + } else { + err = tx.Commit(sdi.ctx) + } + }() + + for _, arg := range args { + _, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses WHERE address = $1`, arg.Address) + if err != nil { + return fmt.Errorf("error removing watched_addresses entry: %v", err) + } + } + + return err +} + +// SetWatchedAddresses clears and inserts the given addresses in the database +func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + tx, err := sdi.dbWriter.db.Begin(sdi.ctx) + if err != nil { + return err + } + defer func() { + if p := recover(); p != nil { + rollback(sdi.ctx, tx) + panic(p) + } else if err != nil { + rollback(sdi.ctx, tx) + } else { + err = tx.Commit(sdi.ctx) + } + }() + + _, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`) + if err != nil { + return fmt.Errorf("error setting watched_addresses table: %v", err) + } + + for _, arg := range args { + _, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`, + arg.Address, arg.CreatedAt, currentBlockNumber.Uint64()) + if err != nil { + return fmt.Errorf("error setting watched_addresses table: %v", err) + } + } + + return err +} + +// ClearWatchedAddresses clears all the watched addresses from the database +func (sdi *StateDiffIndexer) ClearWatchedAddresses() error { + _, err := sdi.dbWriter.db.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`) + if err != nil { + return fmt.Errorf("error clearing watched_addresses table: %v", err) + } + + return nil +} diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go index 394ae07314ee..197fe02b1633 100644 --- a/statediff/indexer/database/sql/indexer_shared_test.go +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -24,12 +24,15 @@ var ( ind interfaces.StateDiffIndexer ipfsPgGet = `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2` - tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte - mockBlock *types.Block - headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid - rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid - rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte - state1CID, state2CID, storageCID cid.Cid + tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte + mockBlock *types.Block + headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid + rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid + rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte + state1CID, state2CID, storageCID cid.Cid + contract1Address, contract2Address, contract3Address, contract4Address string + contract1CreatedAt, contract2CreatedAt, contract3CreatedAt, contract4CreatedAt uint64 + lastFilledAt, watchedAt1, watchedAt2, watchedAt3 uint64 ) func init() { @@ -134,6 +137,20 @@ func init() { rctLeaf3 = orderedRctLeafNodes[2] rctLeaf4 = orderedRctLeafNodes[3] rctLeaf5 = orderedRctLeafNodes[4] + + contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F" + contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B" + contract3Address = "0xcfeB164C328CA13EFd3C77E1980d94975aDfedfc" + contract4Address = "0x0Edf0c4f393a628DE4828B228C48175b3EA297fc" + contract1CreatedAt = uint64(1) + contract2CreatedAt = uint64(2) + contract3CreatedAt = uint64(3) + contract4CreatedAt = uint64(4) + + lastFilledAt = uint64(0) + watchedAt1 = uint64(10) + watchedAt2 = uint64(15) + watchedAt3 = uint64(20) } func expectTrue(t *testing.T, value bool) { diff --git a/statediff/indexer/database/sql/interfaces.go b/statediff/indexer/database/sql/interfaces.go index 445b35d9b631..613a0925194d 100644 --- a/statediff/indexer/database/sql/interfaces.go +++ b/statediff/indexer/database/sql/interfaces.go @@ -54,6 +54,7 @@ type Statements interface { InsertStorageStm() string InsertIPLDStm() string InsertIPLDsStm() string + InsertKnownGapsStm() string } // Tx interface to accommodate different concrete SQL transaction types diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index 197dda04c099..b215187e58c0 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -18,6 +18,7 @@ package sql_test import ( "context" + "math/big" "testing" "github.com/lib/pq" @@ -37,15 +38,20 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) -func setupPGX(t *testing.T) { +func setupPGXIndexer(t *testing.T) { db, err = postgres.SetupPGXDB() if err != nil { t.Fatal(err) } ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db) require.NoError(t, err) +} + +func setupPGX(t *testing.T) { + setupPGXIndexer(t) var tx interfaces.Batch tx, err = ind.PushBlock( mockBlock, @@ -294,12 +300,12 @@ func TestPGXIndexer(t *testing.T) { err = rlp.DecodeBytes(r.Data, &nodeElements) require.NoError(t, err) if len(nodeElements) == 2 { - logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) + logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx]) require.NoError(t, err) // 2nd element of the leaf node contains the encoded log data. require.Equal(t, nodeElements[1].([]byte), logRaw) } else { - logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) + logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx]) require.NoError(t, err) // raw log was IPLDized require.Equal(t, r.Data, logRaw) @@ -483,23 +489,34 @@ func TestPGXIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - require.Equal(t, 1, len(stateNodes)) - stateNode := stateNodes[0] - var data []byte - dc, err := cid.Decode(stateNode.CID) - if err != nil { - t.Fatal(err) - } - mhKey := dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() - require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) + require.Equal(t, 2, len(stateNodes)) + for idx, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + if idx == 0 { + require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) + require.Equal(t, common.BytesToHash(mocks.RemovedLeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x02'}, stateNode.Path) + require.Equal(t, []byte{}, data) + } + if idx == 1 { + require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) + require.Equal(t, common.BytesToHash(mocks.Contract2LeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x07'}, stateNode.Path) + require.Equal(t, []byte{}, data) + } } - require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) - require.Equal(t, []byte{'\x02'}, stateNode.Path) - require.Equal(t, []byte{}, data) }) t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { @@ -552,26 +569,378 @@ func TestPGXIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - require.Equal(t, 1, len(storageNodes)) - require.Equal(t, models.StorageNodeWithStateKeyModel{ - BlockNumber: mocks.BlockNumber.String(), - CID: shared.RemovedNodeStorageCID, - NodeType: 3, - StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), - StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), - Path: []byte{'\x03'}, - }, storageNodes[0]) - dc, err = cid.Decode(storageNodes[0].CID) + require.Equal(t, 3, len(storageNodes)) + expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{'\x03'}, + }, + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage2LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0e'}, + }, + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage3LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0f'}, + }, + } + for idx, storageNode := range storageNodes { + require.Equal(t, expectedStorageNodes[idx], storageNode) + dc, err = cid.Decode(storageNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey = dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, []byte{}, data) + } + }) +} + +func TestPGXWatchAddressMethods(t *testing.T) { + setupPGXIndexer(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + type res struct { + Address string `db:"address"` + CreatedAt uint64 `db:"created_at"` + WatchedAt uint64 `db:"watched_at"` + LastFilledAt uint64 `db:"last_filled_at"` + } + pgStr := "SELECT * FROM eth_meta.watched_addresses" + + t.Run("Load watched addresses (empty table)", func(t *testing.T) { + expectedData := []common.Address{} + + rows, err := ind.LoadWatchedAddresses() + require.NoError(t, err) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Insert watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(watchedAt1))) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) if err != nil { t.Fatal(err) } - mhKey = dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() - require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Insert watched addresses (some already watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(watchedAt2))) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) if err != nil { t.Fatal(err) } - require.Equal(t, []byte{}, data) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Remove watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{} + + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Set watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.SetWatchedAddresses(args, big.NewInt(int64(watchedAt2))) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Set watched addresses (some already watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract4Address, + CreatedAt: contract4CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract4Address, + CreatedAt: contract4CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.SetWatchedAddresses(args, big.NewInt(int64(watchedAt3))) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Load watched addresses", func(t *testing.T) { + expectedData := []common.Address{ + common.HexToAddress(contract4Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + } + + rows, err := ind.LoadWatchedAddresses() + require.NoError(t, err) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Clear watched addresses", func(t *testing.T) { + expectedData := []res{} + + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Clear watched addresses (empty table)", func(t *testing.T) { + expectedData := []res{} + + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } }) } diff --git a/statediff/indexer/database/sql/postgres/database.go b/statediff/indexer/database/sql/postgres/database.go index b0aa49515d4d..f3f03b473530 100644 --- a/statediff/indexer/database/sql/postgres/database.go +++ b/statediff/indexer/database/sql/postgres/database.go @@ -36,6 +36,7 @@ type DB struct { } // InsertHeaderStm satisfies the sql.Statements interface +// Stm == Statement func (db *DB) InsertHeaderStm() string { return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) @@ -99,3 +100,10 @@ func (db *DB) InsertIPLDStm() string { func (db *DB) InsertIPLDsStm() string { return `INSERT INTO public.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT (block_number, key) DO NOTHING` } + +// InsertKnownGapsStm satisfies the sql.Statements interface +func (db *DB) InsertKnownGapsStm() string { + return `INSERT INTO eth_meta.known_gaps (starting_block_number, ending_block_number, checked_out, processing_key) VALUES ($1, $2, $3, $4) + ON CONFLICT (starting_block_number) DO UPDATE SET (ending_block_number, processing_key) = ($2, $4) + WHERE eth_meta.known_gaps.ending_block_number <= $2` +} diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index d64cb90b7275..0db98797cf10 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -18,6 +18,7 @@ package sql_test import ( "context" + "math/big" "testing" "github.com/ipfs/go-cid" @@ -36,15 +37,20 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) -func setupSQLX(t *testing.T) { +func setupSQLXIndexer(t *testing.T) { db, err = postgres.SetupSQLXDB() if err != nil { t.Fatal(err) } ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db) require.NoError(t, err) +} + +func setupSQLX(t *testing.T) { + setupSQLXIndexer(t) var tx interfaces.Batch tx, err = ind.PushBlock( mockBlock, @@ -278,12 +284,12 @@ func TestSQLXIndexer(t *testing.T) { err = rlp.DecodeBytes(r.Data, &nodeElements) require.NoError(t, err) if len(nodeElements) == 2 { - logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) + logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx]) require.NoError(t, err) // 2nd element of the leaf node contains the encoded log data. require.Equal(t, nodeElements[1].([]byte), logRaw) } else { - logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) + logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx]) require.NoError(t, err) // raw log was IPLDized require.Equal(t, r.Data, logRaw) @@ -466,23 +472,34 @@ func TestSQLXIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - require.Equal(t, 1, len(stateNodes)) - stateNode := stateNodes[0] - var data []byte - dc, err := cid.Decode(stateNode.CID) - if err != nil { - t.Fatal(err) - } - mhKey := dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() - require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) + require.Equal(t, 2, len(stateNodes)) + for idx, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + if idx == 0 { + require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) + require.Equal(t, common.BytesToHash(mocks.RemovedLeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x02'}, stateNode.Path) + require.Equal(t, []byte{}, data) + } + if idx == 1 { + require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) + require.Equal(t, common.BytesToHash(mocks.Contract2LeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x07'}, stateNode.Path) + require.Equal(t, []byte{}, data) + } } - require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) - require.Equal(t, []byte{'\x02'}, stateNode.Path) - require.Equal(t, []byte{}, data) }) t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { @@ -535,26 +552,378 @@ func TestSQLXIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - require.Equal(t, 1, len(storageNodes)) - require.Equal(t, models.StorageNodeWithStateKeyModel{ - BlockNumber: mocks.BlockNumber.String(), - CID: shared.RemovedNodeStorageCID, - NodeType: 3, - StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), - StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), - Path: []byte{'\x03'}, - }, storageNodes[0]) - dc, err = cid.Decode(storageNodes[0].CID) + require.Equal(t, 3, len(storageNodes)) + expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{'\x03'}, + }, + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage2LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0e'}, + }, + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage3LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0f'}, + }, + } + for idx, storageNode := range storageNodes { + require.Equal(t, expectedStorageNodes[idx], storageNode) + dc, err = cid.Decode(storageNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey = dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, []byte{}, data) + } + }) +} + +func TestSQLXWatchAddressMethods(t *testing.T) { + setupSQLXIndexer(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + type res struct { + Address string `db:"address"` + CreatedAt uint64 `db:"created_at"` + WatchedAt uint64 `db:"watched_at"` + LastFilledAt uint64 `db:"last_filled_at"` + } + pgStr := "SELECT * FROM eth_meta.watched_addresses" + + t.Run("Load watched addresses (empty table)", func(t *testing.T) { + expectedData := []common.Address{} + + rows, err := ind.LoadWatchedAddresses() + require.NoError(t, err) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Insert watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(watchedAt1))) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) if err != nil { t.Fatal(err) } - mhKey = dshelp.MultihashToDsKey(dc.Hash()) - prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() - require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Insert watched addresses (some already watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(watchedAt2))) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) if err != nil { t.Fatal(err) } - require.Equal(t, []byte{}, data) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Remove watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt1, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + expectedData := []res{} + + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Set watched addresses", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt2, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.SetWatchedAddresses(args, big.NewInt(int64(watchedAt2))) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Set watched addresses (some already watched)", func(t *testing.T) { + args := []sdtypes.WatchAddressArg{ + { + Address: contract4Address, + CreatedAt: contract4CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + expectedData := []res{ + { + Address: contract4Address, + CreatedAt: contract4CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + WatchedAt: watchedAt3, + LastFilledAt: lastFilledAt, + }, + } + + err = ind.SetWatchedAddresses(args, big.NewInt(int64(watchedAt3))) + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Load watched addresses", func(t *testing.T) { + expectedData := []common.Address{ + common.HexToAddress(contract4Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + } + + rows, err := ind.LoadWatchedAddresses() + require.NoError(t, err) + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Clear watched addresses", func(t *testing.T) { + expectedData := []res{} + + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } + }) + + t.Run("Clear watched addresses (empty table)", func(t *testing.T) { + expectedData := []res{} + + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + rows := []res{} + err = db.Select(context.Background(), &rows, pgStr) + if err != nil { + t.Fatal(err) + } + + expectTrue(t, len(rows) == len(expectedData)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } }) } diff --git a/statediff/indexer/database/sql/test_helpers.go b/statediff/indexer/database/sql/test_helpers.go index b1032f8ffc94..398258a0ee7c 100644 --- a/statediff/indexer/database/sql/test_helpers.go +++ b/statediff/indexer/database/sql/test_helpers.go @@ -73,6 +73,10 @@ func TearDownDB(t *testing.T, db Database) { if err != nil { t.Fatal(err) } + _, err = tx.Exec(ctx, `DELETE FROM eth_meta.watched_addresses`) + if err != nil { + t.Fatal(err) + } err = tx.Commit(ctx) if err != nil { t.Fatal(err) diff --git a/statediff/indexer/interfaces/interfaces.go b/statediff/indexer/interfaces/interfaces.go index 8f951230d7e7..6910e3f4962a 100644 --- a/statediff/indexer/interfaces/interfaces.go +++ b/statediff/indexer/interfaces/interfaces.go @@ -21,6 +21,7 @@ import ( "math/big" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/statediff/indexer/shared" sdtypes "github.com/ethereum/go-ethereum/statediff/types" @@ -32,6 +33,14 @@ type StateDiffIndexer interface { PushStateNode(tx Batch, stateNode sdtypes.StateNode, headerID string) error PushCodeAndCodeHash(tx Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error ReportDBMetrics(delay time.Duration, quit <-chan bool) + + // Methods used by WatchAddress API/functionality + LoadWatchedAddresses() ([]common.Address, error) + InsertWatchedAddresses(addresses []sdtypes.WatchAddressArg, currentBlock *big.Int) error + RemoveWatchedAddresses(addresses []sdtypes.WatchAddressArg) error + SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error + ClearWatchedAddresses() error + io.Closer } diff --git a/statediff/indexer/ipld/eth_parser.go b/statediff/indexer/ipld/eth_parser.go index 4e08f2d24708..03061f828740 100644 --- a/statediff/indexer/ipld/eth_parser.go +++ b/statediff/indexer/ipld/eth_parser.go @@ -102,7 +102,7 @@ func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) { return nil, nil, nil, err } - headerRawData := getRLP(obj.Result.Header) + headerRawData := getRLP(&obj.Result.Header) c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256) if err != nil { return nil, nil, nil, err diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index dc7e7638f415..3a82266565b8 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -57,6 +57,7 @@ var ( Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) + ContractAddress2 = crypto.CreateAddress(SenderAddr, MockTransactions[3].Nonce()) MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} mockTopic11 = common.HexToHash("0x04") mockTopic12 = common.HexToHash("0x06") @@ -124,7 +125,7 @@ var ( MockStorageLeafKey = crypto.Keccak256Hash(mockStorageLocation[:]).Bytes() StorageValue = common.Hex2Bytes("01") StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") - StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ StoragePartialPath, StorageValue, }) @@ -133,31 +134,37 @@ var ( ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea") ContractLeafKey = test_helpers.AddressToLeafKey(ContractAddress) - ContractAccount, _ = rlp.EncodeToBytes(types.StateAccount{ + ContractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: nonce1, Balance: big.NewInt(0), CodeHash: ContractCodeHash.Bytes(), Root: common.HexToHash(ContractRoot), }) ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45") - ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + ContractLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ ContractPartialPath, ContractAccount, }) + Contract2LeafKey = test_helpers.AddressToLeafKey(ContractAddress2) + storage2Location = common.HexToHash("2") + Storage2LeafKey = crypto.Keccak256Hash(storage2Location[:]).Bytes() + storage3Location = common.HexToHash("3") + Storage3LeafKey = crypto.Keccak256Hash(storage3Location[:]).Bytes() + nonce0 = uint64(0) AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") AccountLeafKey = test_helpers.Account2LeafKey RemovedLeafKey = test_helpers.Account1LeafKey - Account, _ = rlp.EncodeToBytes(types.StateAccount{ + Account, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: nonce0, Balance: big.NewInt(1000), CodeHash: AccountCodeHash.Bytes(), Root: common.HexToHash(AccountRoot), }) AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45") - AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + AccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ AccountPartialPath, Account, }) @@ -196,6 +203,26 @@ var ( LeafKey: RemovedLeafKey, NodeValue: []byte{}, }, + { + Path: []byte{'\x07'}, + NodeType: sdtypes.Removed, + LeafKey: Contract2LeafKey, + NodeValue: []byte{}, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Removed, + LeafKey: Storage2LeafKey, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0f'}, + NodeType: sdtypes.Removed, + LeafKey: Storage3LeafKey, + NodeValue: []byte{}, + }, + }, + }, } ) diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go index 4e1cfa88883c..cbf17d977b0b 100644 --- a/statediff/indexer/models/models.go +++ b/statediff/indexer/models/models.go @@ -157,3 +157,11 @@ type LogsModel struct { Topic2 string `db:"topic2"` Topic3 string `db:"topic3"` } + +// KnownGaps is the data structure for eth_meta.known_gaps +type KnownGapsModel struct { + StartingBlockNumber string `db:"starting_block_number"` + EndingBlockNumber string `db:"ending_block_number"` + CheckedOut bool `db:"checked_out"` + ProcessingKey int64 `db:"processing_key"` +} diff --git a/statediff/known_gaps.go b/statediff/known_gaps.go new file mode 100644 index 000000000000..682903ed437e --- /dev/null +++ b/statediff/known_gaps.go @@ -0,0 +1,278 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "context" + "fmt" + "io/ioutil" + "math/big" + "os" + "strings" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/models" +) + +var ( + knownGapsInsert = "INSERT INTO eth_meta.known_gaps (starting_block_number, ending_block_number, checked_out, processing_key) " + + "VALUES ('%s', '%s', %t, %d) " + + "ON CONFLICT (starting_block_number) DO UPDATE SET (ending_block_number, processing_key) = ('%s', %d) " + + "WHERE eth_meta.known_gaps.ending_block_number <= '%s';\n" + dbQueryString = "SELECT MAX(block_number) FROM eth.header_cids" + defaultWriteFilePath = "./known_gaps.sql" +) + +type KnownGapsState struct { + // Should we check for gaps by looking at the DB and comparing the latest block with head + checkForGaps bool + // Arbitrary processingKey that can be used down the line to differentiate different geth nodes. + processingKey int64 + // This number indicates the expected difference between blocks. + // Currently, this is 1 since the geth node processes each block. But down the road this can be used in + // Tandom with the processingKey to differentiate block processing logic. + expectedDifference *big.Int + // Indicates if Geth is in an error state + // This is used to indicate the right time to upserts + errorState bool + // This array keeps track of errorBlocks as they occur. + // When the errorState is false again, we can process these blocks. + // Do we need a list, can we have /KnownStartErrorBlock and knownEndErrorBlock ints instead? + knownErrorBlocks []*big.Int + // The filepath to write SQL statements if we can't connect to the DB. + writeFilePath string + // DB object to use for reading and writing to the DB + db sql.Database + //Do we have entries in the local sql file that need to be written to the DB + sqlFileWaitingForWrite bool + // Metrics object used to track metrics. + statediffMetrics statediffMetricsHandles +} + +// Create a new KnownGapsState struct, currently unused. +func NewKnownGapsState(checkForGaps bool, processingKey int64, expectedDifference *big.Int, + errorState bool, writeFilePath string, db sql.Database, statediffMetrics statediffMetricsHandles) *KnownGapsState { + + return &KnownGapsState{ + checkForGaps: checkForGaps, + processingKey: processingKey, + expectedDifference: expectedDifference, + errorState: errorState, + writeFilePath: writeFilePath, + db: db, + statediffMetrics: statediffMetrics, + } + +} + +func minMax(array []*big.Int) (*big.Int, *big.Int) { + var max *big.Int = array[0] + var min *big.Int = array[0] + for _, value := range array { + if max.Cmp(value) == -1 { + max = value + } + if min.Cmp(value) == 1 { + min = value + } + } + return min, max +} + +// This function actually performs the write of the known gaps. It will try to do the following, it only goes to the next step if a failure occurs. +// 1. Write to the DB directly. +// 2. Write to sql file locally. +// 3. Write to prometheus directly. +// 4. Logs and error. +func (kg *KnownGapsState) pushKnownGaps(startingBlockNumber *big.Int, endingBlockNumber *big.Int, checkedOut bool, processingKey int64) error { + if startingBlockNumber.Cmp(endingBlockNumber) != -1 { + return fmt.Errorf("Starting Block %d, is greater than ending block %d", startingBlockNumber, endingBlockNumber) + } + knownGap := models.KnownGapsModel{ + StartingBlockNumber: startingBlockNumber.String(), + EndingBlockNumber: endingBlockNumber.String(), + CheckedOut: checkedOut, + ProcessingKey: processingKey, + } + + log.Info("Updating Metrics for the start and end block") + kg.statediffMetrics.knownGapStart.Update(startingBlockNumber.Int64()) + kg.statediffMetrics.knownGapEnd.Update(endingBlockNumber.Int64()) + + var writeErr error + log.Info("Writing known gaps to the DB") + if kg.db != nil { + dbErr := kg.upsertKnownGaps(knownGap) + if dbErr != nil { + log.Warn("Error writing knownGaps to DB, writing them to file instead") + writeErr = kg.upsertKnownGapsFile(knownGap) + } + } else { + writeErr = kg.upsertKnownGapsFile(knownGap) + } + if writeErr != nil { + log.Error("Unsuccessful when writing to a file", "Error", writeErr) + log.Error("Updating Metrics for the start and end error block") + log.Error("Unable to write the following Gaps to DB or File", "startBlock", startingBlockNumber, "endBlock", endingBlockNumber) + kg.statediffMetrics.knownGapErrorStart.Update(startingBlockNumber.Int64()) + kg.statediffMetrics.knownGapErrorEnd.Update(endingBlockNumber.Int64()) + } + return nil +} + +// This is a simple wrapper function to write gaps from a knownErrorBlocks array. +func (kg *KnownGapsState) captureErrorBlocks(knownErrorBlocks []*big.Int) { + startErrorBlock, endErrorBlock := minMax(knownErrorBlocks) + + log.Warn("The following Gaps were found", "knownErrorBlocks", knownErrorBlocks) + log.Warn("Updating known Gaps table", "startErrorBlock", startErrorBlock, "endErrorBlock", endErrorBlock, "processingKey", kg.processingKey) + kg.pushKnownGaps(startErrorBlock, endErrorBlock, false, kg.processingKey) + +} + +// Users provide the latestBlockInDb and the latestBlockOnChain +// as well as the expected difference. This function does some simple math. +// The expected difference for the time being is going to be 1, but as we run +// More geth nodes, the expected difference might fluctuate. +func isGap(latestBlockInDb *big.Int, latestBlockOnChain *big.Int, expectedDifference *big.Int) bool { + latestBlock := big.NewInt(0) + if latestBlock.Sub(latestBlockOnChain, expectedDifference).Cmp(latestBlockInDb) != 0 { + log.Warn("We found a gap", "latestBlockInDb", latestBlockInDb, "latestBlockOnChain", latestBlockOnChain, "expectedDifference", expectedDifference) + return true + } + return false + +} + +// This function will check for Gaps and update the DB if gaps are found. +// The processingKey will currently be set to 0, but as we start to leverage horizontal scaling +// It might be a useful parameter to update depending on the geth node. +// TODO: +// REmove the return value +// Write to file if err in writing to DB +func (kg *KnownGapsState) findAndUpdateGaps(latestBlockOnChain *big.Int, expectedDifference *big.Int, processingKey int64) error { + // Make this global + latestBlockInDb, err := kg.queryDbToBigInt(dbQueryString) + if err != nil { + return err + } + + gapExists := isGap(latestBlockInDb, latestBlockOnChain, expectedDifference) + if gapExists { + startBlock := big.NewInt(0) + endBlock := big.NewInt(0) + startBlock.Add(latestBlockInDb, expectedDifference) + endBlock.Sub(latestBlockOnChain, expectedDifference) + + log.Warn("Found Gaps starting at", "startBlock", startBlock, "endingBlock", endBlock) + err := kg.pushKnownGaps(startBlock, endBlock, false, processingKey) + if err != nil { + log.Error("We were unable to write the following gap to the DB", "start Block", startBlock, "endBlock", endBlock, "error", err) + return err + } + } + + return nil +} + +// Upserts known gaps to the DB. +func (kg *KnownGapsState) upsertKnownGaps(knownGaps models.KnownGapsModel) error { + _, err := kg.db.Exec(context.Background(), kg.db.InsertKnownGapsStm(), + knownGaps.StartingBlockNumber, knownGaps.EndingBlockNumber, knownGaps.CheckedOut, knownGaps.ProcessingKey) + if err != nil { + return fmt.Errorf("error upserting known_gaps entry: %v", err) + } + log.Info("Successfully Wrote gaps to the DB", "startBlock", knownGaps.StartingBlockNumber, "endBlock", knownGaps.EndingBlockNumber) + return nil +} + +// Write upsert statement into a local file. +func (kg *KnownGapsState) upsertKnownGapsFile(knownGaps models.KnownGapsModel) error { + insertStmt := []byte(fmt.Sprintf(knownGapsInsert, knownGaps.StartingBlockNumber, knownGaps.EndingBlockNumber, knownGaps.CheckedOut, knownGaps.ProcessingKey, + knownGaps.EndingBlockNumber, knownGaps.ProcessingKey, knownGaps.EndingBlockNumber)) + log.Info("Trying to write file") + if kg.writeFilePath == "" { + kg.writeFilePath = defaultWriteFilePath + } + f, err := os.OpenFile(kg.writeFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Info("Unable to open a file for writing") + return err + } + defer f.Close() + + if _, err = f.Write(insertStmt); err != nil { + log.Info("Unable to open write insert statement to file") + return err + } + log.Info("Wrote the gaps to a local SQL file") + kg.sqlFileWaitingForWrite = true + return nil +} + +func (kg *KnownGapsState) writeSqlFileStmtToDb() error { + log.Info("Writing the local SQL file for KnownGaps to the DB") + file, err := ioutil.ReadFile(kg.writeFilePath) + + if err != nil { + log.Error("Unable to open local SQL File for writing") + return err + } + + requests := strings.Split(string(file), ";") + + for _, request := range requests { + _, err := kg.db.Exec(context.Background(), request) + if err != nil { + log.Error("Unable to run insert statement from file to the DB") + return err + } + } + if err := os.Truncate(kg.writeFilePath, 0); err != nil { + log.Info("Failed to empty knownGaps file after inserting statements to the DB", "error", err) + } + kg.sqlFileWaitingForWrite = false + return nil +} + +// This is a simple wrapper function which will run QueryRow on the DB +func (kg *KnownGapsState) queryDb(queryString string) (string, error) { + var ret string + err := kg.db.QueryRow(context.Background(), queryString).Scan(&ret) + if err != nil { + log.Error(fmt.Sprint("Can't properly query the DB for query: ", queryString)) + return "", err + } + return ret, nil +} + +// This function is a simple wrapper which will call QueryDb but the return value will be +// a big int instead of a string +func (kg *KnownGapsState) queryDbToBigInt(queryString string) (*big.Int, error) { + ret := new(big.Int) + res, err := kg.queryDb(queryString) + if err != nil { + return ret, err + } + ret, ok := ret.SetString(res, 10) + if !ok { + log.Error(fmt.Sprint("Can't turn the res ", res, "into a bigInt")) + return ret, fmt.Errorf("Can't turn %s into a bigInt", res) + } + return ret, nil +} diff --git a/statediff/known_gaps_test.go b/statediff/known_gaps_test.go new file mode 100644 index 000000000000..bd1c72a2a381 --- /dev/null +++ b/statediff/known_gaps_test.go @@ -0,0 +1,212 @@ +package statediff + +import ( + "context" + "fmt" + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/stretchr/testify/require" +) + +var ( + knownGapsFilePath = "./known_gaps.sql" +) + +type gapValues struct { + lastProcessedBlock int64 + currentBlock int64 + knownErrorBlocksStart int64 + knownErrorBlocksEnd int64 + expectedDif int64 + processingKey int64 +} + +// Add clean db +// Test for failures when they are expected, when we go from smaller block to larger block +// We should no longer see the smaller block in DB +func TestKnownGaps(t *testing.T) { + + tests := []gapValues{ + // Known Gaps + {knownErrorBlocksStart: 115, knownErrorBlocksEnd: 120, expectedDif: 1, processingKey: 1}, + /// Same tests as above with a new expected DIF + {knownErrorBlocksStart: 1150, knownErrorBlocksEnd: 1200, expectedDif: 2, processingKey: 2}, + // Test update when block number is larger!! + {knownErrorBlocksStart: 1150, knownErrorBlocksEnd: 1204, expectedDif: 2, processingKey: 2}, + // Update when processing key is different! + {knownErrorBlocksStart: 1150, knownErrorBlocksEnd: 1204, expectedDif: 2, processingKey: 10}, + } + + testWriteToDb(t, tests, true) + testWriteToFile(t, tests, true) + testFindAndUpdateGaps(t, true) +} + +// test writing blocks to the DB +func testWriteToDb(t *testing.T, tests []gapValues, wipeDbBeforeStart bool) { + t.Log("Starting Write to DB test") + db := setupDb(t) + + // Clear Table first, this is needed because we updated an entry to have a larger endblock number + // so we can't find the original start and endblock pair. + if wipeDbBeforeStart { + t.Log("Cleaning up eth_meta.known_gaps table") + db.Exec(context.Background(), "DELETE FROM eth_meta.known_gaps") + } + + for _, tc := range tests { + // Create an array with knownGaps based on user inputs + knownGaps := KnownGapsState{ + processingKey: tc.processingKey, + expectedDifference: big.NewInt(tc.expectedDif), + db: db, + statediffMetrics: RegisterStatediffMetrics(metrics.DefaultRegistry), + } + service := &Service{ + KnownGaps: knownGaps, + } + knownErrorBlocks := (make([]*big.Int, 0)) + knownErrorBlocks = createKnownErrorBlocks(knownErrorBlocks, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) + service.KnownGaps.knownErrorBlocks = knownErrorBlocks + // Upsert + testCaptureErrorBlocks(t, service) + // Validate that the upsert was done correctly. + validateUpsert(t, service, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) + } + tearDown(t, db) + +} + +// test writing blocks to file and then inserting them to DB +func testWriteToFile(t *testing.T, tests []gapValues, wipeDbBeforeStart bool) { + t.Log("Starting write to file test") + db := setupDb(t) + // Clear Table first, this is needed because we updated an entry to have a larger endblock number + // so we can't find the original start and endblock pair. + if wipeDbBeforeStart { + t.Log("Cleaning up eth_meta.known_gaps table") + db.Exec(context.Background(), "DELETE FROM eth_meta.known_gaps") + } + if _, err := os.Stat(knownGapsFilePath); err == nil { + err := os.Remove(knownGapsFilePath) + if err != nil { + t.Fatal("Can't delete local file") + } + } + tearDown(t, db) + for _, tc := range tests { + knownGaps := KnownGapsState{ + processingKey: tc.processingKey, + expectedDifference: big.NewInt(tc.expectedDif), + writeFilePath: knownGapsFilePath, + statediffMetrics: RegisterStatediffMetrics(metrics.DefaultRegistry), + db: nil, // Only set to nil to be verbose that we can't use it + } + service := &Service{ + KnownGaps: knownGaps, + } + knownErrorBlocks := (make([]*big.Int, 0)) + knownErrorBlocks = createKnownErrorBlocks(knownErrorBlocks, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) + service.KnownGaps.knownErrorBlocks = knownErrorBlocks + + testCaptureErrorBlocks(t, service) + newDb := setupDb(t) + service.KnownGaps.db = newDb + if service.KnownGaps.sqlFileWaitingForWrite { + writeErr := service.KnownGaps.writeSqlFileStmtToDb() + require.NoError(t, writeErr) + } + + // Validate that the upsert was done correctly. + validateUpsert(t, service, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) + tearDown(t, newDb) + } +} + +// Find a gap, if no gaps exist, it will create an arbitrary one +func testFindAndUpdateGaps(t *testing.T, wipeDbBeforeStart bool) { + db := setupDb(t) + + if wipeDbBeforeStart { + db.Exec(context.Background(), "DELETE FROM eth_meta.known_gaps") + } + knownGaps := KnownGapsState{ + processingKey: 1, + expectedDifference: big.NewInt(1), + db: db, + statediffMetrics: RegisterStatediffMetrics(metrics.DefaultRegistry), + } + service := &Service{ + KnownGaps: knownGaps, + } + + latestBlockInDb, err := service.KnownGaps.queryDbToBigInt("SELECT MAX(block_number) FROM eth.header_cids") + if err != nil { + t.Skip("Can't find a block in the eth.header_cids table.. Please put one there") + } + + // Add the gapDifference for testing purposes + gapDifference := big.NewInt(10) // Set a difference between latestBlock in DB and on Chain + expectedDifference := big.NewInt(1) // Set what the expected difference between latestBlock in DB and on Chain should be + + latestBlockOnChain := big.NewInt(0) + latestBlockOnChain.Add(latestBlockInDb, gapDifference) + + t.Log("The latest block on the chain is: ", latestBlockOnChain) + t.Log("The latest block on the DB is: ", latestBlockInDb) + + gapUpsertErr := service.KnownGaps.findAndUpdateGaps(latestBlockOnChain, expectedDifference, 0) + require.NoError(t, gapUpsertErr) + + startBlock := big.NewInt(0) + endBlock := big.NewInt(0) + + startBlock.Add(latestBlockInDb, expectedDifference) + endBlock.Sub(latestBlockOnChain, expectedDifference) + validateUpsert(t, service, startBlock.Int64(), endBlock.Int64()) + +} + +// test capturing missed blocks +func testCaptureErrorBlocks(t *testing.T, service *Service) { + service.KnownGaps.captureErrorBlocks(service.KnownGaps.knownErrorBlocks) +} + +// Helper function to create an array of gaps given a start and end block +func createKnownErrorBlocks(knownErrorBlocks []*big.Int, knownErrorBlocksStart int64, knownErrorBlocksEnd int64) []*big.Int { + for i := knownErrorBlocksStart; i <= knownErrorBlocksEnd; i++ { + knownErrorBlocks = append(knownErrorBlocks, big.NewInt(i)) + } + return knownErrorBlocks +} + +// Make sure the upsert was performed correctly +func validateUpsert(t *testing.T, service *Service, startingBlock int64, endingBlock int64) { + t.Logf("Starting to query blocks: %d - %d", startingBlock, endingBlock) + queryString := fmt.Sprintf("SELECT starting_block_number from eth_meta.known_gaps WHERE starting_block_number = %d AND ending_block_number = %d", startingBlock, endingBlock) + + _, queryErr := service.KnownGaps.queryDb(queryString) // Figure out the string. + t.Logf("Updated Known Gaps table starting from, %d, and ending at, %d", startingBlock, endingBlock) + require.NoError(t, queryErr) +} + +// Create a DB object to use +func setupDb(t *testing.T) sql.Database { + db, err := postgres.SetupSQLXDB() + if err != nil { + t.Error("Can't create a DB connection....") + t.Fatal(err) + } + return db +} + +// Teardown the DB +func tearDown(t *testing.T, db sql.Database) { + t.Log("Starting tearDown") + db.Close() +} diff --git a/statediff/mainnet_tests/builder_test.go b/statediff/mainnet_tests/builder_test.go index d838302e0631..c487304f9e6e 100644 --- a/statediff/mainnet_tests/builder_test.go +++ b/statediff/mainnet_tests/builder_test.go @@ -50,18 +50,18 @@ var ( emptyStorage = make([]sdtypes.StorageNode, 0) // block 1 data - block1CoinbaseAccount, _ = rlp.EncodeToBytes(types.StateAccount{ + block1CoinbaseAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: big.NewInt(5000000000000000000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - block1CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + block1CoinbaseLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("38251692195afc818c92b485fcb8a4691af89cbe5a2ab557b83a4261be2a9a"), block1CoinbaseAccount, }) block1CoinbaseLeafNodeHash = crypto.Keccak256(block1CoinbaseLeafNode) - block1x040bBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block1x040bBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("cc947d5ebb80600bad471f12c6ad5e4981e3525ecf8a2d982cc032536ae8b66d"), common.Hex2Bytes("e80e52462e635a834e90e86ccf7673a6430384aac17004d626f4db831f0624bc"), common.Hex2Bytes("59a8f11f60cb0a8488831f242da02944a26fd269d0608a44b8b873ded9e59e1b"), @@ -81,7 +81,7 @@ var ( []byte{}, }) block1x040bBranchNodeHash = crypto.Keccak256(block1x040bBranchNode) - block1x04BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block1x04BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("a9317a59365ca09cefcd384018696590afffc432e35a97e8f85aa48907bf3247"), common.Hex2Bytes("e0bc229254ce7a6a736c3953e570ab18b4a7f5f2a9aa3c3057b5f17d250a1cad"), common.Hex2Bytes("a2484ec8884dbe0cf24ece99d67df0d1fe78992d67cc777636a817cb2ef205aa"), @@ -101,7 +101,7 @@ var ( []byte{}, }) block1x04BranchNodeHash = crypto.Keccak256(block1x04BranchNode) - block1RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block1RootBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("90dcaf88c40c7bbc95a912cbdde67c175767b31173df9ee4b0d733bfdd511c43"), common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), @@ -122,30 +122,30 @@ var ( }) // block 2 data - block2CoinbaseAccount, _ = rlp.EncodeToBytes(types.StateAccount{ + block2CoinbaseAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: big.NewInt(5000000000000000000), CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - block2CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + block2CoinbaseLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("20679cbcf198c1741a6f4e4473845659a30caa8b26f8d37a0be2e2bc0d8892"), block2CoinbaseAccount, }) block2CoinbaseLeafNodeHash = crypto.Keccak256(block2CoinbaseLeafNode) block2MovedPremineBalance, _ = new(big.Int).SetString("4000000000000000000000", 10) - block2MovedPremineAccount, _ = rlp.EncodeToBytes(types.StateAccount{ + block2MovedPremineAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: block2MovedPremineBalance, CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - block2MovedPremineLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + block2MovedPremineLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("20f2e24db7943eab4415f99e109698863b0fecca1cf9ffc500f38cefbbe29e"), block2MovedPremineAccount, }) block2MovedPremineLeafNodeHash = crypto.Keccak256(block2MovedPremineLeafNode) - block2x00080dBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block2x00080dBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ block2MovedPremineLeafNodeHash, []byte{}, []byte{}, @@ -165,7 +165,7 @@ var ( []byte{}, }) block2x00080dBranchNodeHash = crypto.Keccak256(block2x00080dBranchNode) - block2x0008BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block2x0008BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("def97a26f824fc3911cf7f8c41dfc9bc93cc36ae2248de22ecae01d6950b2dc9"), common.Hex2Bytes("234a575e2c5badab8de0f6515b6723195323a0562fbe1316255888637043f1c1"), common.Hex2Bytes("29659740af1c23306ee8f8294c71a5632ace8c80b1eb61cfdf7022f47ff52305"), @@ -185,7 +185,7 @@ var ( []byte{}, }) block2x0008BranchNodeHash = crypto.Keccak256(block2x0008BranchNode) - block2x00BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block2x00BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("e45a9e85cab1b6eb18b30df2c6acc448bbac6a30d81646823b31223e16e5063e"), common.Hex2Bytes("33bd7171d556b981f6849064eb09412b24fedc0812127db936067043f53db1b9"), common.Hex2Bytes("ca56945f074da4f15587404593faf3a50d17ea0e21a418ad6ec99bdf4bf3f914"), @@ -205,7 +205,7 @@ var ( []byte{}, }) block2x00BranchNodeHash = crypto.Keccak256(block2x00BranchNode) - block2RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block2RootBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ block2x00BranchNodeHash, common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), @@ -228,45 +228,45 @@ var ( // block3 data // path 060e0f blcok3CoinbaseBalance, _ = new(big.Int).SetString("5156250000000000000", 10) - block3CoinbaseAccount, _ = rlp.EncodeToBytes(types.StateAccount{ + block3CoinbaseAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: blcok3CoinbaseBalance, CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - block3CoinbaseLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + block3CoinbaseLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3a174f00e64521a535f35e67c1aa241951c791639b2f3d060f49c5d9fa8b9e"), block3CoinbaseAccount, }) block3CoinbaseLeafNodeHash = crypto.Keccak256(block3CoinbaseLeafNode) // path 0c0e050703 block3MovedPremineBalance1, _ = new(big.Int).SetString("3750000000000000000", 10) - block3MovedPremineAccount1, _ = rlp.EncodeToBytes(types.StateAccount{ + block3MovedPremineAccount1, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: block3MovedPremineBalance1, CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - block3MovedPremineLeafNode1, _ = rlp.EncodeToBytes([]interface{}{ + block3MovedPremineLeafNode1, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190"), // ce573ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190 block3MovedPremineAccount1, }) block3MovedPremineLeafNodeHash1 = crypto.Keccak256(block3MovedPremineLeafNode1) // path 0c0e050708 block3MovedPremineBalance2, _ = new(big.Int).SetString("1999944000000000000000", 10) - block3MovedPremineAccount2, _ = rlp.EncodeToBytes(types.StateAccount{ + block3MovedPremineAccount2, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, Balance: block3MovedPremineBalance2, CodeHash: test_helpers.NullCodeHash.Bytes(), Root: test_helpers.EmptyContractRoot, }) - block3MovedPremineLeafNode2, _ = rlp.EncodeToBytes([]interface{}{ + block3MovedPremineLeafNode2, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("33bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012"), // ce5783bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012 block3MovedPremineAccount2, }) block3MovedPremineLeafNodeHash2 = crypto.Keccak256(block3MovedPremineLeafNode2) - block3x0c0e0507BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block3x0c0e0507BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ []byte{}, []byte{}, []byte{}, @@ -287,7 +287,7 @@ var ( }) block3x0c0e0507BranchNodeHash = crypto.Keccak256(block3x0c0e0507BranchNode) - block3x0c0e05BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block3x0c0e05BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("452e3beb503b1d87ae7c672b98a8e3fd043a671405502562ae1043dc97151a50"), []byte{}, common.Hex2Bytes("2f5bb16f77086f67ce8c4258cb9061cb299e597b2ad4ad6d7ccc474d6d88e85e"), @@ -308,7 +308,7 @@ var ( }) block3x0c0e05BranchNodeHash = crypto.Keccak256(block3x0c0e05BranchNode) - block3x060eBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block3x060eBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("94d77c7c30b88829c9989948b206cda5e532b38b49534261c517aebf4a3e6fdb"), common.Hex2Bytes("a5cf57a50da8204964e834a12a53f9bed7afc9b700a4a81b440122d60c7603a7"), []byte{}, @@ -329,7 +329,7 @@ var ( }) block3x060eBranchNodeHash = crypto.Keccak256(block3x060eBranchNode) - block3x0c0eBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block3x0c0eBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("70647f11b2b995d718f9e8aceb44c8839e0055641930d216fa6090280a9d63d5"), common.Hex2Bytes("fdfb17cd2fba2a14219981cb7886a1977cd85dbef5c767c562f4a5f547febff0"), common.Hex2Bytes("ff87313253ec6f860142b7bf62efb4cb07ea668c57aa90cbe9ef22b72fee15c7"), @@ -350,7 +350,7 @@ var ( }) block3x0c0eBranchNodeHash = crypto.Keccak256(block3x0c0eBranchNode) - block3x06BranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block3x06BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("68f7ff8c074d6e4cccd55b5b1c2116a6dd7047d4332090e6db8839362991b0ae"), common.Hex2Bytes("c446eb4377c750701374c56e50759e6ba68b7adf4d543e718c8b28a99ae3b6ad"), common.Hex2Bytes("ef2c49ec64cb65eae0d99684e74c8af2bd0206c9a0214d9d3eddf0881dd8412a"), @@ -371,7 +371,7 @@ var ( }) block3x06BranchNodeHash = crypto.Keccak256(block3x06BranchNode) - block3x0cBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block3x0cBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("dae48f5b47930c28bb116fbd55e52cd47242c71bf55373b55eb2805ee2e4a929"), common.Hex2Bytes("0f1f37f337ec800e2e5974e2e7355f10f1a4832b39b846d916c3597a460e0676"), common.Hex2Bytes("da8f627bb8fbeead17b318e0a8e4f528db310f591bb6ab2deda4a9f7ca902ab5"), @@ -392,7 +392,7 @@ var ( }) block3x0cBranchNodeHash = crypto.Keccak256(block3x0cBranchNode) - block3RootBranchNode, _ = rlp.EncodeToBytes([]interface{}{ + block3RootBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("f646da473c426e79f1c796b00d4873f47de1dbe1c9d19d63993a05eeb8b4041d"), common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), @@ -675,7 +675,7 @@ func TestBuilderOnMainnetBlocks(t *testing.T) { if err != nil { t.Error(err) } - expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + expectedStateDiffRlp, err := rlp.EncodeToBytes(&test.expected) if err != nil { t.Error(err) } diff --git a/statediff/metrics.go b/statediff/metrics.go index afc80e40e5bb..e67499b94328 100644 --- a/statediff/metrics.go +++ b/statediff/metrics.go @@ -50,6 +50,14 @@ type statediffMetricsHandles struct { // Current length of chainEvent channels serviceLoopChannelLen metrics.Gauge writeLoopChannelLen metrics.Gauge + // The start block of the known gap + knownGapStart metrics.Gauge + // The end block of the known gap + knownGapEnd metrics.Gauge + // A known gaps start block which had an error being written to the DB + knownGapErrorStart metrics.Gauge + // A known gaps end block which had an error being written to the DB + knownGapErrorEnd metrics.Gauge } func RegisterStatediffMetrics(reg metrics.Registry) statediffMetricsHandles { @@ -59,6 +67,10 @@ func RegisterStatediffMetrics(reg metrics.Registry) statediffMetricsHandles { lastStatediffHeight: metrics.NewGauge(), serviceLoopChannelLen: metrics.NewGauge(), writeLoopChannelLen: metrics.NewGauge(), + knownGapStart: metrics.NewGauge(), + knownGapEnd: metrics.NewGauge(), + knownGapErrorStart: metrics.NewGauge(), + knownGapErrorEnd: metrics.NewGauge(), } subsys := "service" reg.Register(metricName(subsys, "last_sync_height"), ctx.lastSyncHeight) @@ -66,5 +78,9 @@ func RegisterStatediffMetrics(reg metrics.Registry) statediffMetricsHandles { reg.Register(metricName(subsys, "last_statediff_height"), ctx.lastStatediffHeight) reg.Register(metricName(subsys, "service_loop_channel_len"), ctx.serviceLoopChannelLen) reg.Register(metricName(subsys, "write_loop_channel_len"), ctx.writeLoopChannelLen) + reg.Register(metricName(subsys, "known_gaps_start"), ctx.knownGapStart) + reg.Register(metricName(subsys, "known_gaps_end"), ctx.knownGapEnd) + reg.Register(metricName(subsys, "known_gaps_error_start"), ctx.knownGapErrorStart) + reg.Register(metricName(subsys, "known_gaps_error_end"), ctx.knownGapErrorEnd) return ctx } diff --git a/statediff/service.go b/statediff/service.go index bf45b22fe428..4bdb9bf50d29 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -18,6 +18,7 @@ package statediff import ( "bytes" + "fmt" "math/big" "strconv" "strings" @@ -33,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" @@ -40,32 +42,40 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ind "github.com/ethereum/go-ethereum/statediff/indexer" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" types2 "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" + "github.com/thoas/go-funk" ) const ( - chainEventChanSize = 20000 - genesisBlockNumber = 0 - defaultRetryLimit = 3 // default retry limit once deadlock is detected. - deadlockDetected = "deadlock detected" // 40P01 https://www.postgresql.org/docs/current/errcodes-appendix.html + chainEventChanSize = 20000 + genesisBlockNumber = 0 + defaultRetryLimit = 3 // default retry limit once deadlock is detected. + deadlockDetected = "deadlock detected" // 40P01 https://www.postgresql.org/docs/current/errcodes-appendix.html + typeAssertionFailed = "type assertion failed" + unexpectedOperation = "unexpected operation" ) -var writeLoopParams = Params{ - IntermediateStateNodes: true, - IntermediateStorageNodes: true, - IncludeBlock: true, - IncludeReceipts: true, - IncludeTD: true, - IncludeCode: true, +var writeLoopParams = ParamsWithMutex{ + Params: Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + IncludeBlock: true, + IncludeReceipts: true, + IncludeTD: true, + IncludeCode: true, + }, } var statediffMetrics = RegisterStatediffMetrics(metrics.DefaultRegistry) type blockChain interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + CurrentBlock() *types.Block GetBlockByHash(hash common.Hash) *types.Block GetBlockByNumber(number uint64) *types.Block GetReceiptsByHash(hash common.Hash) types.Receipts @@ -100,6 +110,8 @@ type IService interface { WriteStateDiffFor(blockHash common.Hash, params Params) error // WriteLoop event loop for progressively processing and writing diffs directly to DB WriteLoop(chainEventCh chan core.ChainEvent) + // Method to change the addresses being watched in write loop params + WatchAddress(operation types2.OperationType, args []types2.WatchAddressArg) error } // Service is the underlying struct for the state diffing service @@ -118,11 +130,17 @@ type Service struct { SubscriptionTypes map[common.Hash]Params // Cache the last block so that we can avoid having to lookup the next block's parent BlockCache BlockCache + // The publicBackendAPI which provides useful information about the current state + BackendAPI ethapi.Backend + // Should the statediff service wait for geth to sync to head? + WaitForSync bool + // Used to signal if we should check for KnownGaps + KnownGaps KnownGapsState // Whether or not we have any subscribers; only if we do, do we processes state diffs subscribers int32 // Interface for publishing statediffs as PG-IPLD objects indexer interfaces.StateDiffIndexer - // Whether to enable writing state diffs directly to track blochain head + // Whether to enable writing state diffs directly to track blockchain head. enableWriteLoop bool // Size of the worker pool numWorkers uint @@ -146,9 +164,11 @@ func NewBlockCache(max uint) BlockCache { // New creates a new statediff.Service // func New(stack *node.Node, ethServ *eth.Ethereum, dbParams *DBParams, enableWriteLoop bool) error { -func New(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params Config) error { +func New(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params Config, backend ethapi.Backend) error { blockChain := ethServ.BlockChain() var indexer interfaces.StateDiffIndexer + var db sql.Database + var err error quitCh := make(chan bool) if params.IndexerConfig != nil { info := nodeinfo.Info{ @@ -159,16 +179,34 @@ func New(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params ClientName: params.ClientName, } var err error - indexer, err = ind.NewStateDiffIndexer(params.Context, blockChain.Config(), info, params.IndexerConfig) + db, indexer, err = ind.NewStateDiffIndexer(params.Context, blockChain.Config(), info, params.IndexerConfig) if err != nil { return err } indexer.ReportDBMetrics(10*time.Second, quitCh) } + workers := params.NumWorkers if workers == 0 { workers = 1 } + // If we ever have multiple processingKeys we can update them here + // along with the expectedDifference + knownGaps := &KnownGapsState{ + processingKey: 0, + expectedDifference: big.NewInt(1), + errorState: false, + writeFilePath: params.KnownGapsFilePath, + db: db, + statediffMetrics: statediffMetrics, + sqlFileWaitingForWrite: false, + } + if params.IndexerConfig.Type() == shared.POSTGRES { + knownGaps.checkForGaps = true + } else { + log.Info("We are not going to check for gaps on start up since we are not connected to Postgres!") + knownGaps.checkForGaps = false + } sds := &Service{ Mutex: sync.Mutex{}, BlockChain: blockChain, @@ -177,6 +215,9 @@ func New(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params Subscriptions: make(map[common.Hash]map[rpc.ID]Subscription), SubscriptionTypes: make(map[common.Hash]Params), BlockCache: NewBlockCache(workers), + BackendAPI: backend, + WaitForSync: params.WaitForSync, + KnownGaps: *knownGaps, indexer: indexer, enableWriteLoop: params.EnableWriteLoop, numWorkers: workers, @@ -184,6 +225,12 @@ func New(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params } stack.RegisterLifecycle(sds) stack.RegisterAPIs(sds.APIs()) + + err = loadWatchedAddresses(indexer) + if err != nil { + return err + } + return nil } @@ -273,7 +320,9 @@ func (sds *Service) WriteLoop(chainEventCh chan core.ChainEvent) { func (sds *Service) writeGenesisStateDiff(currBlock *types.Block, workerId uint) { // For genesis block we need to return the entire state trie hence we diff it with an empty trie. log.Info("Writing state diff", "block height", genesisBlockNumber, "worker", workerId) - err := sds.writeStateDiffWithRetry(currBlock, common.Hash{}, writeLoopParams) + writeLoopParams.RLock() + err := sds.writeStateDiffWithRetry(currBlock, common.Hash{}, writeLoopParams.Params) + writeLoopParams.RUnlock() if err != nil { log.Error("statediff.Service.WriteLoop: processing error", "block height", genesisBlockNumber, "error", err.Error(), "worker", workerId) @@ -301,12 +350,44 @@ func (sds *Service) writeLoopWorker(params workerParams) { sds.writeGenesisStateDiff(parentBlock, params.id) } + // If for any reason we need to check for gaps, + // Check and update the gaps table. + if sds.KnownGaps.checkForGaps && !sds.KnownGaps.errorState { + log.Info("Checking for Gaps at", "current block", currentBlock.Number()) + go sds.KnownGaps.findAndUpdateGaps(currentBlock.Number(), sds.KnownGaps.expectedDifference, sds.KnownGaps.processingKey) + sds.KnownGaps.checkForGaps = false + } + log.Info("Writing state diff", "block height", currentBlock.Number().Uint64(), "worker", params.id) - err := sds.writeStateDiffWithRetry(currentBlock, parentBlock.Root(), writeLoopParams) + writeLoopParams.RLock() + err := sds.writeStateDiffWithRetry(currentBlock, parentBlock.Root(), writeLoopParams.Params) + writeLoopParams.RUnlock() if err != nil { log.Error("statediff.Service.WriteLoop: processing error", "block height", currentBlock.Number().Uint64(), "error", err.Error(), "worker", params.id) + sds.KnownGaps.errorState = true + log.Warn("Updating the following block to knownErrorBlocks to be inserted into knownGaps table", "blockNumber", currentBlock.Number()) + sds.KnownGaps.knownErrorBlocks = append(sds.KnownGaps.knownErrorBlocks, currentBlock.Number()) + // Write object to startdiff continue } + sds.KnownGaps.errorState = false + if sds.KnownGaps.knownErrorBlocks != nil { + // We must pass in parameters by VALUE not reference. + // If we pass them in my reference, the references can change before the computation is complete! + staticKnownErrorBlocks := make([]*big.Int, len(sds.KnownGaps.knownErrorBlocks)) + copy(staticKnownErrorBlocks, sds.KnownGaps.knownErrorBlocks) + sds.KnownGaps.knownErrorBlocks = nil + go sds.KnownGaps.captureErrorBlocks(staticKnownErrorBlocks) + } + + if sds.KnownGaps.sqlFileWaitingForWrite { + log.Info("There are entries in the SQL file for knownGaps that should be written") + err := sds.KnownGaps.writeSqlFileStmtToDb() + if err != nil { + log.Error("Unable to write KnownGap sql file to DB") + } + } + // TODO: how to handle with concurrent workers statediffMetrics.lastStatediffHeight.Update(int64(currentBlock.Number().Uint64())) case <-sds.QuitChan: @@ -395,6 +476,10 @@ func (sds *Service) streamStateDiff(currentBlock *types.Block, parentRoot common func (sds *Service) StateDiffAt(blockNumber uint64, params Params) (*Payload, error) { currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) log.Info("sending state diff", "block height", blockNumber) + + // compute leaf keys of watched addresses in the params + params.ComputeWatchedAddressesLeafKeys() + if blockNumber == 0 { return sds.processStateDiff(currentBlock, common.Hash{}, params) } @@ -407,6 +492,10 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params Params) (*Payload, er func (sds *Service) StateDiffFor(blockHash common.Hash, params Params) (*Payload, error) { currentBlock := sds.BlockChain.GetBlockByHash(blockHash) log.Info("sending state diff", "block hash", blockHash) + + // compute leaf keys of watched addresses in the params + params.ComputeWatchedAddressesLeafKeys() + if currentBlock.NumberU64() == 0 { return sds.processStateDiff(currentBlock, common.Hash{}, params) } @@ -427,7 +516,7 @@ func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot commo if err != nil { return nil, err } - stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + stateDiffRlp, err := rlp.EncodeToBytes(&stateDiff) if err != nil { return nil, err } @@ -465,6 +554,10 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params Pa func (sds *Service) StateTrieAt(blockNumber uint64, params Params) (*Payload, error) { currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) log.Info("sending state trie", "block height", blockNumber) + + // compute leaf keys of watched addresses in the params + params.ComputeWatchedAddressesLeafKeys() + return sds.processStateTrie(currentBlock, params) } @@ -473,7 +566,7 @@ func (sds *Service) processStateTrie(block *types.Block, params Params) (*Payloa if err != nil { return nil, err } - stateTrieRlp, err := rlp.EncodeToBytes(stateNodes) + stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes) if err != nil { return nil, err } @@ -487,8 +580,12 @@ func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- boo if atomic.CompareAndSwapInt32(&sds.subscribers, 0, 1) { log.Info("State diffing subscription received; beginning statediff processing") } + + // compute leaf keys of watched addresses in the params + params.ComputeWatchedAddressesLeafKeys() + // Subscription type is defined as the hash of the rlp-serialized subscription params - by, err := rlp.EncodeToBytes(params) + by, err := rlp.EncodeToBytes(¶ms) if err != nil { log.Error("State diffing params need to be rlp-serializable") return @@ -528,15 +625,62 @@ func (sds *Service) Unsubscribe(id rpc.ID) error { return nil } +// This function will check the status of geth syncing. +// It will return false if geth has finished syncing. +// It will return a true Geth is still syncing. +func (sds *Service) GetSyncStatus(pubEthAPI *ethapi.PublicEthereumAPI) (bool, error) { + syncStatus, err := pubEthAPI.Syncing() + if err != nil { + return true, err + } + + if syncStatus != false { + return true, err + } + return false, err +} + +// This function calls GetSyncStatus to check if we have caught up to head. +// It will keep looking and checking if we have caught up to head. +// It will only complete if we catch up to head, otherwise it will keep looping forever. +func (sds *Service) WaitingForSync() error { + log.Info("We are going to wait for geth to sync to head!") + + // Has the geth node synced to head? + Synced := false + pubEthAPI := ethapi.NewPublicEthereumAPI(sds.BackendAPI) + for !Synced { + syncStatus, err := sds.GetSyncStatus(pubEthAPI) + if err != nil { + return err + } + if !syncStatus { + log.Info("Geth has caught up to the head of the chain") + Synced = true + } else { + time.Sleep(1 * time.Second) + } + } + return nil +} + // Start is used to begin the service func (sds *Service) Start() error { log.Info("Starting statediff service") + if sds.WaitForSync { + log.Info("Statediff service will wait until geth has caught up to the head of the chain.") + err := sds.WaitingForSync() + if err != nil { + return err + } + log.Info("Continuing with startdiff start process") + } chainEventCh := make(chan core.ChainEvent, chainEventChanSize) go sds.Loop(chainEventCh) if sds.enableWriteLoop { - log.Info("Starting statediff DB write loop", "params", writeLoopParams) + log.Info("Starting statediff DB write loop", "params", writeLoopParams.Params) chainEventCh := make(chan core.ChainEvent, chainEventChanSize) go sds.WriteLoop(chainEventCh) } @@ -633,6 +777,9 @@ func (sds *Service) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- typ // This operation cannot be performed back past the point of db pruning; it requires an archival node // for historical data func (sds *Service) WriteStateDiffAt(blockNumber uint64, params Params) error { + // compute leaf keys of watched addresses in the params + params.ComputeWatchedAddressesLeafKeys() + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) parentRoot := common.Hash{} if blockNumber != 0 { @@ -646,6 +793,9 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params Params) error { // This operation cannot be performed back past the point of db pruning; it requires an archival node // for historical data func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params Params) error { + // compute leaf keys of watched addresses in the params + params.ComputeWatchedAddressesLeafKeys() + currentBlock := sds.BlockChain.GetBlockByHash(blockHash) parentRoot := common.Hash{} if currentBlock.NumberU64() != 0 { @@ -713,3 +863,130 @@ func (sds *Service) writeStateDiffWithRetry(block *types.Block, parentRoot commo } return err } + +// Performs one of following operations on the watched addresses in writeLoopParams and the db: +// add | remove | set | clear +func (sds *Service) WatchAddress(operation types2.OperationType, args []types2.WatchAddressArg) error { + // lock writeLoopParams for a write + writeLoopParams.Lock() + defer writeLoopParams.Unlock() + + // get the current block number + currentBlockNumber := sds.BlockChain.CurrentBlock().Number() + + switch operation { + case types2.Add: + // filter out args having an already watched address with a warning + filteredArgs, ok := funk.Filter(args, func(arg types2.WatchAddressArg) bool { + if funk.Contains(writeLoopParams.WatchedAddresses, common.HexToAddress(arg.Address)) { + log.Warn("Address already being watched", "address", arg.Address) + return false + } + return true + }).([]types2.WatchAddressArg) + if !ok { + return fmt.Errorf("add: filtered args %s", typeAssertionFailed) + } + + // get addresses from the filtered args + filteredAddresses, err := MapWatchAddressArgsToAddresses(filteredArgs) + if err != nil { + return fmt.Errorf("add: filtered addresses %s", err.Error()) + } + + // update the db + err = sds.indexer.InsertWatchedAddresses(filteredArgs, currentBlockNumber) + if err != nil { + return err + } + + // update in-memory params + writeLoopParams.WatchedAddresses = append(writeLoopParams.WatchedAddresses, filteredAddresses...) + funk.ForEach(filteredAddresses, func(address common.Address) { + writeLoopParams.watchedAddressesLeafKeys[crypto.Keccak256Hash(address.Bytes())] = struct{}{} + }) + case types2.Remove: + // get addresses from args + argAddresses, err := MapWatchAddressArgsToAddresses(args) + if err != nil { + return fmt.Errorf("remove: mapped addresses %s", err.Error()) + } + + // remove the provided addresses from currently watched addresses + addresses, ok := funk.Subtract(writeLoopParams.WatchedAddresses, argAddresses).([]common.Address) + if !ok { + return fmt.Errorf("remove: filtered addresses %s", typeAssertionFailed) + } + + // update the db + err = sds.indexer.RemoveWatchedAddresses(args) + if err != nil { + return err + } + + // update in-memory params + writeLoopParams.WatchedAddresses = addresses + funk.ForEach(argAddresses, func(address common.Address) { + delete(writeLoopParams.watchedAddressesLeafKeys, crypto.Keccak256Hash(address.Bytes())) + }) + case types2.Set: + // get addresses from args + argAddresses, err := MapWatchAddressArgsToAddresses(args) + if err != nil { + return fmt.Errorf("set: mapped addresses %s", err.Error()) + } + + // update the db + err = sds.indexer.SetWatchedAddresses(args, currentBlockNumber) + if err != nil { + return err + } + + // update in-memory params + writeLoopParams.WatchedAddresses = argAddresses + writeLoopParams.ComputeWatchedAddressesLeafKeys() + case types2.Clear: + // update the db + err := sds.indexer.ClearWatchedAddresses() + if err != nil { + return err + } + + // update in-memory params + writeLoopParams.WatchedAddresses = []common.Address{} + writeLoopParams.ComputeWatchedAddressesLeafKeys() + + default: + return fmt.Errorf("%s %s", unexpectedOperation, operation) + } + + return nil +} + +// loadWatchedAddresses loads watched addresses to in-memory write loop params +func loadWatchedAddresses(indexer interfaces.StateDiffIndexer) error { + watchedAddresses, err := indexer.LoadWatchedAddresses() + if err != nil { + return err + } + + writeLoopParams.Lock() + defer writeLoopParams.Unlock() + + writeLoopParams.WatchedAddresses = watchedAddresses + writeLoopParams.ComputeWatchedAddressesLeafKeys() + + return nil +} + +// MapWatchAddressArgsToAddresses maps []WatchAddressArg to corresponding []common.Address +func MapWatchAddressArgsToAddresses(args []types2.WatchAddressArg) ([]common.Address, error) { + addresses, ok := funk.Map(args, func(arg types2.WatchAddressArg) common.Address { + return common.HexToAddress(arg.Address) + }).([]common.Address) + if !ok { + return nil, fmt.Errorf(typeAssertionFailed) + } + + return addresses, nil +} diff --git a/statediff/service_test.go b/statediff/service_test.go index a17f892173aa..f9a08bbf1beb 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -23,18 +23,18 @@ import ( "reflect" "sync" "testing" - - types2 "github.com/ethereum/go-ethereum/statediff/types" - - "github.com/ethereum/go-ethereum/trie" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" statediff "github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/statediff/test_helpers/mocks" + types2 "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" ) func TestServiceLoop(t *testing.T) { @@ -130,11 +130,11 @@ func testErrorInChainEventLoop(t *testing.T) { t.Logf("Actual number of payloads does not equal expected.\nactual: %+v\nexpected: 3", len(payloads)) } - testReceipts1Rlp, err := rlp.EncodeToBytes(testReceipts1) + testReceipts1Rlp, err := rlp.EncodeToBytes(&testReceipts1) if err != nil { t.Error(err) } - testReceipts2Rlp, err := rlp.EncodeToBytes(testReceipts2) + testReceipts2Rlp, err := rlp.EncodeToBytes(&testReceipts2) if err != nil { t.Error(err) } @@ -146,6 +146,7 @@ func testErrorInChainEventLoop(t *testing.T) { } } + defaultParams.ComputeWatchedAddressesLeafKeys() if !reflect.DeepEqual(builder.Params, defaultParams) { t.Error("Test failure:", t.Name()) t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) @@ -197,6 +198,8 @@ func testErrorInBlockLoop(t *testing.T) { } }() service.Loop(eventsChannel) + + defaultParams.ComputeWatchedAddressesLeafKeys() if !reflect.DeepEqual(builder.Params, defaultParams) { t.Error("Test failure:", t.Name()) t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) @@ -224,11 +227,11 @@ func testErrorInStateDiffAt(t *testing.T) { BlockNumber: testBlock1.Number(), BlockHash: testBlock1.Hash(), } - expectedStateDiffRlp, err := rlp.EncodeToBytes(mockStateDiff) + expectedStateDiffRlp, err := rlp.EncodeToBytes(&mockStateDiff) if err != nil { t.Error(err) } - expectedReceiptsRlp, err := rlp.EncodeToBytes(testReceipts1) + expectedReceiptsRlp, err := rlp.EncodeToBytes(&testReceipts1) if err != nil { t.Error(err) } @@ -241,7 +244,7 @@ func testErrorInStateDiffAt(t *testing.T) { ReceiptsRlp: expectedReceiptsRlp, BlockRlp: expectedBlockRlp, } - expectedStateDiffPayloadRlp, err := rlp.EncodeToBytes(expectedStateDiffPayload) + expectedStateDiffPayloadRlp, err := rlp.EncodeToBytes(&expectedStateDiffPayload) if err != nil { t.Error(err) } @@ -270,6 +273,8 @@ func testErrorInStateDiffAt(t *testing.T) { if err != nil { t.Error(err) } + + defaultParams.ComputeWatchedAddressesLeafKeys() if !reflect.DeepEqual(builder.Params, defaultParams) { t.Error("Test failure:", t.Name()) t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) @@ -291,3 +296,145 @@ func testErrorInStateDiffAt(t *testing.T) { t.Logf("Actual state diff payload does not equal expected.\nactual:%+v\nexpected: %+v", expectedStateDiffPayload, stateDiffPayload) } } + +func TestWaitForSync(t *testing.T) { + testWaitForSync(t) + testGetSyncStatus(t) +} + +// This function will create a backend and service object which includes a generic Backend +func createServiceWithMockBackend(curBlock uint64, highestBlock uint64) (*mocks.Backend, *statediff.Service) { + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + backend := mocks.Backend{ + StartingBlock: 1, + CurrBlock: curBlock, + HighestBlock: highestBlock, + SyncedAccounts: 5, + SyncedAccountBytes: 5, + SyncedBytecodes: 5, + SyncedBytecodeBytes: 5, + SyncedStorage: 5, + SyncedStorageBytes: 5, + HealedTrienodes: 5, + HealedTrienodeBytes: 5, + HealedBytecodes: 5, + HealedBytecodeBytes: 5, + HealingTrienodes: 5, + HealingBytecode: 5, + } + + service := &statediff.Service{ + Mutex: sync.Mutex{}, + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + BlockCache: statediff.NewBlockCache(1), + BackendAPI: &backend, + WaitForSync: true, + } + return &backend, service +} + +// This function will test to make sure that the state diff waits +// until the blockchain has caught up to head! +func testWaitForSync(t *testing.T) { + t.Log("Starting Sync") + _, service := createServiceWithMockBackend(10, 10) + err := service.WaitingForSync() + if err != nil { + t.Fatal("Sync Failed") + } + t.Log("Sync Complete") +} + +// This test will run the WaitForSync() at the start of the execusion +// It will then incrementally increase the currentBlock to match the highestBlock +// At each interval it will run the GetSyncStatus to ensure that the return value is not false. +// It will also check to make sure that the WaitForSync() function has not completed! +func testGetSyncStatus(t *testing.T) { + t.Log("Starting Get Sync Status Test") + var highestBlock uint64 = 5 + // Create a backend and a service + // the backend is lagging behind the sync. + backend, service := createServiceWithMockBackend(0, highestBlock) + + checkSyncComplete := make(chan int, 1) + + go func() { + // Start the sync function which will wait for the sync + // Once the sync is complete add a value to the checkSyncComplet channel + t.Log("Starting Sync") + err := service.WaitingForSync() + if err != nil { + t.Error("Sync Failed") + checkSyncComplete <- 1 + } + t.Log("We have finally synced!") + checkSyncComplete <- 0 + }() + + tables := []struct { + currentBlock uint64 + highestBlock uint64 + }{ + {1, highestBlock}, + {2, highestBlock}, + {3, highestBlock}, + {4, highestBlock}, + {5, highestBlock}, + } + + time.Sleep(2 * time.Second) + for _, table := range tables { + // Iterate over each block + // Once the highest block reaches the current block the sync should complete + + // Update the backend current block value + t.Log("Updating Current Block to: ", table.currentBlock) + backend.CurrBlock = table.currentBlock + pubEthAPI := ethapi.NewPublicEthereumAPI(service.BackendAPI) + syncStatus, err := service.GetSyncStatus(pubEthAPI) + + if err != nil { + t.Fatal("Sync Failed") + } + + time.Sleep(2 * time.Second) + + // Make sure if syncStatus is false that WaitForSync has completed! + if !syncStatus && len(checkSyncComplete) == 0 { + t.Error("Sync is complete but WaitForSync is not") + } + + if syncStatus && len(checkSyncComplete) == 1 { + t.Error("Sync is not complete but WaitForSync is") + } + + // Make sure sync hasn't completed and that the checkSyncComplete channel is empty + if syncStatus && len(checkSyncComplete) == 0 { + continue + } + + // This code will only be run if the sync is complete and the WaitForSync function is complete + + // If syncstatus is complete, make sure that the blocks match + if !syncStatus && table.currentBlock != table.highestBlock { + t.Errorf("syncStatus indicated sync was complete even when current block, %d, and highest block %d aren't equal", + table.currentBlock, table.highestBlock) + } + + // Make sure that WaitForSync completed once the current block caught up to head! + checkSyncCompleteVal := <-checkSyncComplete + if checkSyncCompleteVal != 0 { + t.Errorf("syncStatus indicated sync was complete but the checkSyncComplete has a value of %d", + checkSyncCompleteVal) + } else { + t.Log("Test Passed!") + } + + } + +} diff --git a/statediff/test_helpers/helpers.go b/statediff/test_helpers/helpers.go index 8373f75378cf..39d8081bbdcb 100644 --- a/statediff/test_helpers/helpers.go +++ b/statediff/test_helpers/helpers.go @@ -106,13 +106,17 @@ func TestChainGen(i int, block *core.BlockGen) { block.AddTx(tx3) case 4: // Block 5 has one tx from bankAccount to the contract, that transfers no value - // It sets the remaining storage value to zero + // It sets the one storage value to zero and the other to new value. // Block 5 is mined by Account1Addr block.SetCoinbase(Account1Addr) - data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000") + data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000") + data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") nonce := block.TxNonce(TestBankAddress) - tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data), signer, TestBankKey) - block.AddTx(tx) + tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data1), signer, TestBankKey) + nonce++ + tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data2), signer, TestBankKey) + block.AddTx(tx1) + block.AddTx(tx2) case 5: // Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value // Block 6 is mined by Account2Addr diff --git a/statediff/test_helpers/mocks/backend.go b/statediff/test_helpers/mocks/backend.go new file mode 100644 index 000000000000..24b328882b1d --- /dev/null +++ b/statediff/test_helpers/mocks/backend.go @@ -0,0 +1,253 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "context" + "math/big" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +// Builder is a mock state diff builder +type Backend struct { + StartingBlock uint64 + CurrBlock uint64 + HighestBlock uint64 + SyncedAccounts uint64 + SyncedAccountBytes uint64 + SyncedBytecodes uint64 + SyncedBytecodeBytes uint64 + SyncedStorage uint64 + SyncedStorageBytes uint64 + HealedTrienodes uint64 + HealedTrienodeBytes uint64 + HealedBytecodes uint64 + HealedBytecodeBytes uint64 + HealingTrienodes uint64 + HealingBytecode uint64 +} + +// General Ethereum API +func (backend *Backend) SyncProgress() ethereum.SyncProgress { + l := ethereum.SyncProgress{ + StartingBlock: backend.StartingBlock, + CurrentBlock: backend.CurrBlock, + HighestBlock: backend.HighestBlock, + SyncedAccounts: backend.SyncedAccounts, + SyncedAccountBytes: backend.SyncedAccountBytes, + SyncedBytecodes: backend.SyncedBytecodes, + SyncedBytecodeBytes: backend.SyncedBytecodeBytes, + SyncedStorage: backend.SyncedStorage, + SyncedStorageBytes: backend.SyncedStorageBytes, + HealedTrienodes: backend.HealedTrienodes, + HealedTrienodeBytes: backend.HealedTrienodeBytes, + HealedBytecodes: backend.HealedBytecodes, + HealedBytecodeBytes: backend.HealedBytecodeBytes, + HealingTrienodes: backend.HealingTrienodes, + HealingBytecode: backend.HealingBytecode, + } + return l +} + +func (backend *Backend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) ChainDb() ethdb.Database { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) AccountManager() *accounts.Manager { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) ExtRPCEnabled() bool { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) RPCGasCap() uint64 { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) RPCEVMTimeout() time.Duration { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) RPCTxFeeCap() float64 { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) UnprotectedAllowed() bool { + panic("not implemented") // TODO: Implement +} + +// Blockchain API +func (backend *Backend) SetHead(number uint64) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) CurrentHeader() *types.Header { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) CurrentBlock() *types.Block { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetTd(ctx context.Context, hash common.Hash) *big.Int { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +// Transaction pool API +func (backend *Backend) SendTx(ctx context.Context, signedTx *types.Transaction) error { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetPoolTransactions() (types.Transactions, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetPoolTransaction(txHash common.Hash) *types.Transaction { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) Stats() (pending int, queued int) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeNewTxsEvent(_ chan<- core.NewTxsEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +// Filter API +func (backend *Backend) BloomStatus() (uint64, uint64) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) ChainConfig() *params.ChainConfig { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) Engine() consensus.Engine { + panic("not implemented") // TODO: Implement +} diff --git a/statediff/test_helpers/mocks/blockchain.go b/statediff/test_helpers/mocks/blockchain.go index b4b1f36942a4..f2a77af64ba5 100644 --- a/statediff/test_helpers/mocks/blockchain.go +++ b/statediff/test_helpers/mocks/blockchain.go @@ -39,6 +39,7 @@ type BlockChain struct { Receipts map[common.Hash]types.Receipts TDByHash map[common.Hash]*big.Int TDByNum map[uint64]*big.Int + currentBlock *types.Block } // SetBlocksForHashes mock method @@ -128,6 +129,16 @@ func (bc *BlockChain) GetTd(hash common.Hash, blockNum uint64) *big.Int { return nil } +// SetCurrentBlock test method +func (bc *BlockChain) SetCurrentBlock(block *types.Block) { + bc.currentBlock = block +} + +// CurrentBlock mock method +func (bc *BlockChain) CurrentBlock() *types.Block { + return bc.currentBlock +} + func (bc *BlockChain) SetTd(hash common.Hash, blockNum uint64, td *big.Int) { if bc.TDByHash == nil { bc.TDByHash = make(map[common.Hash]*big.Int) diff --git a/statediff/test_helpers/mocks/indexer.go b/statediff/test_helpers/mocks/indexer.go new file mode 100644 index 000000000000..92005a8b4c4d --- /dev/null +++ b/statediff/test_helpers/mocks/indexer.go @@ -0,0 +1,70 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var _ interfaces.StateDiffIndexer = &StateDiffIndexer{} + +// StateDiffIndexer is a mock state diff indexer +type StateDiffIndexer struct{} + +func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) { + return nil, nil +} + +func (sdi *StateDiffIndexer) PushStateNode(tx interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { + return nil +} + +func (sdi *StateDiffIndexer) PushCodeAndCodeHash(tx interfaces.Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error { + return nil +} + +func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) {} + +func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) { + return nil, nil +} + +func (sdi *StateDiffIndexer) InsertWatchedAddresses(addresses []sdtypes.WatchAddressArg, currentBlock *big.Int) error { + return nil +} + +func (sdi *StateDiffIndexer) RemoveWatchedAddresses(addresses []sdtypes.WatchAddressArg) error { + return nil +} + +func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + return nil +} + +func (sdi *StateDiffIndexer) ClearWatchedAddresses() error { + return nil +} + +func (sdi *StateDiffIndexer) Close() error { + return nil +} diff --git a/statediff/test_helpers/mocks/service.go b/statediff/test_helpers/mocks/service.go index f10017df43f3..13722d14833d 100644 --- a/statediff/test_helpers/mocks/service.go +++ b/statediff/test_helpers/mocks/service.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" + "github.com/thoas/go-funk" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -32,9 +33,15 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) +var ( + typeAssertionFailed = "type assertion failed" + unexpectedOperation = "unexpected operation" +) + // MockStateDiffService is a mock state diff service type MockStateDiffService struct { sync.Mutex @@ -47,6 +54,8 @@ type MockStateDiffService struct { QuitChan chan bool Subscriptions map[common.Hash]map[rpc.ID]statediff.Subscription SubscriptionTypes map[common.Hash]statediff.Params + Indexer interfaces.StateDiffIndexer + writeLoopParams statediff.ParamsWithMutex } // Protocols mock method @@ -147,7 +156,7 @@ func (sds *MockStateDiffService) processStateDiff(currentBlock *types.Block, par if err != nil { return nil, err } - stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + stateDiffRlp, err := rlp.EncodeToBytes(&stateDiff) if err != nil { return nil, err } @@ -228,7 +237,7 @@ func (sds *MockStateDiffService) stateTrieAt(block *types.Block, params statedif if err != nil { return nil, err } - stateTrieRlp, err := rlp.EncodeToBytes(stateNodes) + stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes) if err != nil { return nil, err } @@ -238,7 +247,7 @@ func (sds *MockStateDiffService) stateTrieAt(block *types.Block, params statedif // Subscribe is used by the API to subscribe to the service loop func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool, params statediff.Params) { // Subscription type is defined as the hash of the rlp-serialized subscription params - by, err := rlp.EncodeToBytes(params) + by, err := rlp.EncodeToBytes(¶ms) if err != nil { return } @@ -332,3 +341,98 @@ func sendNonBlockingQuit(id rpc.ID, sub statediff.Subscription) { log.Info("unable to close subscription %s; channel has no receiver", id) } } + +// Performs one of following operations on the watched addresses in writeLoopParams and the db: +// add | remove | set | clear +func (sds *MockStateDiffService) WatchAddress(operation sdtypes.OperationType, args []sdtypes.WatchAddressArg) error { + // lock writeLoopParams for a write + sds.writeLoopParams.Lock() + defer sds.writeLoopParams.Unlock() + + // get the current block number + currentBlockNumber := sds.BlockChain.CurrentBlock().Number() + + switch operation { + case sdtypes.Add: + // filter out args having an already watched address with a warning + filteredArgs, ok := funk.Filter(args, func(arg sdtypes.WatchAddressArg) bool { + if funk.Contains(sds.writeLoopParams.WatchedAddresses, common.HexToAddress(arg.Address)) { + log.Warn("Address already being watched", "address", arg.Address) + return false + } + return true + }).([]sdtypes.WatchAddressArg) + if !ok { + return fmt.Errorf("add: filtered args %s", typeAssertionFailed) + } + + // get addresses from the filtered args + filteredAddresses, err := statediff.MapWatchAddressArgsToAddresses(filteredArgs) + if err != nil { + return fmt.Errorf("add: filtered addresses %s", err.Error()) + } + + // update the db + err = sds.Indexer.InsertWatchedAddresses(filteredArgs, currentBlockNumber) + if err != nil { + return err + } + + // update in-memory params + sds.writeLoopParams.WatchedAddresses = append(sds.writeLoopParams.WatchedAddresses, filteredAddresses...) + sds.writeLoopParams.ComputeWatchedAddressesLeafKeys() + case sdtypes.Remove: + // get addresses from args + argAddresses, err := statediff.MapWatchAddressArgsToAddresses(args) + if err != nil { + return fmt.Errorf("remove: mapped addresses %s", err.Error()) + } + + // remove the provided addresses from currently watched addresses + addresses, ok := funk.Subtract(sds.writeLoopParams.WatchedAddresses, argAddresses).([]common.Address) + if !ok { + return fmt.Errorf("remove: filtered addresses %s", typeAssertionFailed) + } + + // update the db + err = sds.Indexer.RemoveWatchedAddresses(args) + if err != nil { + return err + } + + // update in-memory params + sds.writeLoopParams.WatchedAddresses = addresses + sds.writeLoopParams.ComputeWatchedAddressesLeafKeys() + case sdtypes.Set: + // get addresses from args + argAddresses, err := statediff.MapWatchAddressArgsToAddresses(args) + if err != nil { + return fmt.Errorf("set: mapped addresses %s", err.Error()) + } + + // update the db + err = sds.Indexer.SetWatchedAddresses(args, currentBlockNumber) + if err != nil { + return err + } + + // update in-memory params + sds.writeLoopParams.WatchedAddresses = argAddresses + sds.writeLoopParams.ComputeWatchedAddressesLeafKeys() + case sdtypes.Clear: + // update the db + err := sds.Indexer.ClearWatchedAddresses() + if err != nil { + return err + } + + // update in-memory params + sds.writeLoopParams.WatchedAddresses = []common.Address{} + sds.writeLoopParams.ComputeWatchedAddressesLeafKeys() + + default: + return fmt.Errorf("%s %s", unexpectedOperation, operation) + } + + return nil +} diff --git a/statediff/test_helpers/mocks/service_test.go b/statediff/test_helpers/mocks/service_test.go index dde784316386..4911163beaa7 100644 --- a/statediff/test_helpers/mocks/service_test.go +++ b/statediff/test_helpers/mocks/service_test.go @@ -21,6 +21,7 @@ import ( "fmt" "math/big" "os" + "reflect" "sort" "sync" "testing" @@ -39,38 +40,38 @@ var ( emptyStorage = make([]sdtypes.StorageNode, 0) block0, block1 *types.Block minerLeafKey = test_helpers.AddressToLeafKey(common.HexToAddress("0x0")) - account1, _ = rlp.EncodeToBytes(types.StateAccount{ + account1, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: uint64(0), Balance: big.NewInt(10000), CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), }) - account1LeafNode, _ = rlp.EncodeToBytes([]interface{}{ + account1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), account1, }) - minerAccount, _ = rlp.EncodeToBytes(types.StateAccount{ + minerAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: uint64(0), Balance: big.NewInt(2000002625000000000), CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), }) - minerAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + minerAccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), minerAccount, }) - bankAccount, _ = rlp.EncodeToBytes(types.StateAccount{ + bankAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: uint64(1), Balance: big.NewInt(1999978999999990000), CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), }) - bankAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + bankAccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccount, }) mockTotalDifficulty = big.NewInt(1337) - params = statediff.Params{ + parameters = statediff.Params{ IntermediateStateNodes: false, IncludeTD: true, IncludeBlock: true, @@ -88,6 +89,7 @@ func init() { func TestAPI(t *testing.T) { testSubscriptionAPI(t) testHTTPAPI(t) + testWatchAddressAPI(t) } func testSubscriptionAPI(t *testing.T) { @@ -100,7 +102,7 @@ func testSubscriptionAPI(t *testing.T) { BlockNumber: block1.Number(), BlockHash: block1.Hash(), } - expectedReceiptBytes, _ := rlp.EncodeToBytes(types.Receipts{mockReceipt}) + expectedReceiptBytes, _ := rlp.EncodeToBytes(&types.Receipts{mockReceipt}) expectedStateDiff := sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), @@ -128,7 +130,7 @@ func testSubscriptionAPI(t *testing.T) { }, }, } - expectedStateDiffBytes, _ := rlp.EncodeToBytes(expectedStateDiff) + expectedStateDiffBytes, _ := rlp.EncodeToBytes(&expectedStateDiff) blockChan := make(chan *types.Block) parentBlockChain := make(chan *types.Block) @@ -176,7 +178,7 @@ func testSubscriptionAPI(t *testing.T) { } }() time.Sleep(1) - mockService.Subscribe(id, payloadChan, quitChan, params) + mockService.Subscribe(id, payloadChan, quitChan, parameters) blockChan <- block1 parentBlockChain <- block0 wg.Wait() @@ -192,7 +194,7 @@ func testHTTPAPI(t *testing.T) { BlockNumber: block1.Number(), BlockHash: block1.Hash(), } - expectedReceiptBytes, _ := rlp.EncodeToBytes(types.Receipts{mockReceipt}) + expectedReceiptBytes, _ := rlp.EncodeToBytes(&types.Receipts{mockReceipt}) expectedStateDiff := sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), @@ -220,7 +222,7 @@ func testHTTPAPI(t *testing.T) { }, }, } - expectedStateDiffBytes, _ := rlp.EncodeToBytes(expectedStateDiff) + expectedStateDiffBytes, _ := rlp.EncodeToBytes(&expectedStateDiff) mockBlockChain := &BlockChain{} mockBlockChain.SetBlocksForHashes(map[common.Hash]*types.Block{ block0.Hash(): block0, @@ -234,7 +236,7 @@ func testHTTPAPI(t *testing.T) { Builder: statediff.NewBuilder(chain.StateCache()), BlockChain: mockBlockChain, } - payload, err := mockService.StateDiffAt(block1.Number().Uint64(), params) + payload, err := mockService.StateDiffAt(block1.Number().Uint64(), parameters) if err != nil { t.Error(err) } @@ -253,3 +255,286 @@ func testHTTPAPI(t *testing.T) { t.Errorf("paylaod does not have the expected total difficulty\r\nactual td: %d\r\nexpected td: %d", payload.TotalDifficulty.Int64(), mockTotalDifficulty.Int64()) } } + +func testWatchAddressAPI(t *testing.T) { + blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) + defer chain.Stop() + block6 := blocks[5] + + mockBlockChain := &BlockChain{} + mockBlockChain.SetCurrentBlock(block6) + mockIndexer := StateDiffIndexer{} + mockService := MockStateDiffService{ + BlockChain: mockBlockChain, + Indexer: &mockIndexer, + } + + // test data + var ( + contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F" + contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B" + contract3Address = "0xcfeB164C328CA13EFd3C77E1980d94975aDfedfc" + contract4Address = "0x0Edf0c4f393a628DE4828B228C48175b3EA297fc" + contract1CreatedAt = uint64(1) + contract2CreatedAt = uint64(2) + contract3CreatedAt = uint64(3) + contract4CreatedAt = uint64(4) + + args1 = []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + startingParams1 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + expectedParams1 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract1Address), + common.HexToAddress(contract2Address), + }, + } + + args2 = []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + startingParams2 = expectedParams1 + expectedParams2 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract1Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + }, + } + + args3 = []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + startingParams3 = expectedParams2 + expectedParams3 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract1Address), + }, + } + + args4 = []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + startingParams4 = expectedParams3 + expectedParams4 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + + args5 = []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + startingParams5 = expectedParams4 + expectedParams5 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract1Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + }, + } + + args6 = []sdtypes.WatchAddressArg{ + { + Address: contract4Address, + CreatedAt: contract4CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + startingParams6 = expectedParams5 + expectedParams6 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract4Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + }, + } + + args7 = []sdtypes.WatchAddressArg{} + startingParams7 = expectedParams6 + expectedParams7 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + + args8 = []sdtypes.WatchAddressArg{} + startingParams8 = expectedParams6 + expectedParams8 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + + args9 = []sdtypes.WatchAddressArg{} + startingParams9 = expectedParams8 + expectedParams9 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + ) + + tests := []struct { + name string + operation sdtypes.OperationType + args []sdtypes.WatchAddressArg + startingParams statediff.Params + expectedParams statediff.Params + expectedErr error + }{ + { + "testAddAddresses", + sdtypes.Add, + args1, + startingParams1, + expectedParams1, + nil, + }, + { + "testAddAddressesSomeWatched", + sdtypes.Add, + args2, + startingParams2, + expectedParams2, + nil, + }, + { + "testRemoveAddresses", + sdtypes.Remove, + args3, + startingParams3, + expectedParams3, + nil, + }, + { + "testRemoveAddressesSomeWatched", + sdtypes.Remove, + args4, + startingParams4, + expectedParams4, + nil, + }, + { + "testSetAddresses", + sdtypes.Set, + args5, + startingParams5, + expectedParams5, + nil, + }, + { + "testSetAddressesSomeWatched", + sdtypes.Set, + args6, + startingParams6, + expectedParams6, + nil, + }, + { + "testSetAddressesEmtpyArgs", + sdtypes.Set, + args7, + startingParams7, + expectedParams7, + nil, + }, + { + "testClearAddresses", + sdtypes.Clear, + args8, + startingParams8, + expectedParams8, + nil, + }, + { + "testClearAddressesEmpty", + sdtypes.Clear, + args9, + startingParams9, + expectedParams9, + nil, + }, + + // invalid args + { + "testInvalidOperation", + "WrongOp", + args9, + startingParams9, + statediff.Params{}, + fmt.Errorf("%s WrongOp", unexpectedOperation), + }, + } + + for _, test := range tests { + // set indexing params + mockService.writeLoopParams = statediff.ParamsWithMutex{ + Params: test.startingParams, + } + mockService.writeLoopParams.ComputeWatchedAddressesLeafKeys() + + // make the API call to change watched addresses + err := mockService.WatchAddress(test.operation, test.args) + if test.expectedErr != nil { + if err.Error() != test.expectedErr.Error() { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual err: %+v\nexpected err: %+v", err, test.expectedErr) + } + + continue + } + if err != nil { + t.Error(err) + } + + // check updated indexing params + test.expectedParams.ComputeWatchedAddressesLeafKeys() + updatedParams := mockService.writeLoopParams.Params + if !reflect.DeepEqual(updatedParams, test.expectedParams) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual params: %+v\nexpected params: %+v", updatedParams, test.expectedParams) + } + } +} diff --git a/statediff/test_helpers/test_data.go b/statediff/test_helpers/test_data.go index e5b021364d02..eb250c8cb09d 100644 --- a/statediff/test_helpers/test_data.go +++ b/statediff/test_helpers/test_data.go @@ -69,6 +69,6 @@ var ( CodeHash = common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127") ContractAddr common.Address - EmptyRootNode, _ = rlp.EncodeToBytes([]byte{}) + EmptyRootNode, _ = rlp.EncodeToBytes(&[]byte{}) EmptyContractRoot = crypto.Keccak256Hash(EmptyRootNode) ) diff --git a/statediff/types/types.go b/statediff/types/types.go index 36008a784063..0a29adaf892b 100644 --- a/statediff/types/types.go +++ b/statediff/types/types.go @@ -101,3 +101,20 @@ type CodeAndCodeHash struct { type StateNodeSink func(StateNode) error type StorageNodeSink func(StorageNode) error type CodeSink func(CodeAndCodeHash) error + +// OperationType for type of WatchAddress operation +type OperationType string + +const ( + Add OperationType = "add" + Remove OperationType = "remove" + Set OperationType = "set" + Clear OperationType = "clear" +) + +// WatchAddressArg is a arg type for WatchAddress API +type WatchAddressArg struct { + // Address represents common.Address + Address string + CreatedAt uint64 +} diff --git a/tests/fuzzers/bn256/bn256_fuzz.go b/tests/fuzzers/bn256/bn256_fuzz.go index 030ac19b3f52..11fd9e18df00 100644 --- a/tests/fuzzers/bn256/bn256_fuzz.go +++ b/tests/fuzzers/bn256/bn256_fuzz.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. +//go:build gofuzz // +build gofuzz package bn256 diff --git a/tests/fuzzers/secp256k1/secp_fuzzer.go b/tests/fuzzers/secp256k1/secp_fuzzer.go index 53845b643345..47083d5fe3a9 100644 --- a/tests/fuzzers/secp256k1/secp_fuzzer.go +++ b/tests/fuzzers/secp256k1/secp_fuzzer.go @@ -21,7 +21,7 @@ package secp256k1 import ( "fmt" - "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/btcec/v2" "github.com/ethereum/go-ethereum/crypto/secp256k1" fuzz "github.com/google/gofuzz" ) diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index e73ef4851a9c..9ed8bcbc51d5 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -66,6 +66,8 @@ func (s *spongeDb) Has(key []byte) (bool, error) { panic("implement func (s *spongeDb) Get(key []byte) ([]byte, error) { return nil, errors.New("no such elem") } func (s *spongeDb) Delete(key []byte) error { panic("implement me") } func (s *spongeDb) NewBatch() ethdb.Batch { return &spongeBatch{s} } +func (s *spongeDb) NewBatchWithSize(size int) ethdb.Batch { return &spongeBatch{s} } +func (s *spongeDb) NewSnapshot() (ethdb.Snapshot, error) { panic("implement me") } func (s *spongeDb) Stat(property string) (string, error) { panic("implement me") } func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Close() error { return nil } diff --git a/tests/fuzzers/vflux/clientpool-fuzzer.go b/tests/fuzzers/vflux/clientpool-fuzzer.go index 0414c001ec4e..b3b523cc8243 100644 --- a/tests/fuzzers/vflux/clientpool-fuzzer.go +++ b/tests/fuzzers/vflux/clientpool-fuzzer.go @@ -267,9 +267,7 @@ func FuzzClientPool(input []byte) int { bias = f.randomDelay() requested = f.randomBool() ) - if _, err := pool.SetCapacity(f.peers[index].node, reqCap, bias, requested); err == vfs.ErrCantFindMaximum { - panic(nil) - } + pool.SetCapacity(f.peers[index].node, reqCap, bias, requested) doLog("Set capacity", "id", f.peers[index].node.ID(), "reqcap", reqCap, "bias", bias, "requested", requested) case 7: index := f.randomByte() diff --git a/trie/committer.go b/trie/committer.go index 0721990a2179..db753e2fa0c4 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -44,7 +44,6 @@ type leaf struct { // By 'some level' of parallelism, it's still the case that all leaves will be // processed sequentially - onleaf will never be called in parallel or out of order. type committer struct { - tmp sliceBuffer sha crypto.KeccakState onleaf LeafCallback @@ -55,7 +54,6 @@ type committer struct { var committerPool = sync.Pool{ New: func() interface{} { return &committer{ - tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. sha: sha3.NewLegacyKeccak256().(crypto.KeccakState), } }, diff --git a/trie/database.go b/trie/database.go index 58ca4e6f3caa..d71abeee476a 100644 --- a/trie/database.go +++ b/trie/database.go @@ -113,16 +113,9 @@ func (n rawFullNode) cache() (hashNode, bool) { panic("this should never end u func (n rawFullNode) fstring(ind string) string { panic("this should never end up in a live trie") } func (n rawFullNode) EncodeRLP(w io.Writer) error { - var nodes [17]node - - for i, child := range n { - if child != nil { - nodes[i] = child - } else { - nodes[i] = nilValueNode - } - } - return rlp.Encode(w, nodes) + eb := rlp.NewEncoderBuffer(w) + n.encode(eb) + return eb.Flush() } // rawShortNode represents only the useful data content of a short node, with the @@ -164,11 +157,7 @@ func (n *cachedNode) rlp() []byte { if node, ok := n.node.(rawNode); ok { return node } - blob, err := rlp.EncodeToBytes(n.node) - if err != nil { - panic(err) - } - return blob + return nodeToBytes(n.node) } // obj returns the decoded and expanded trie node, either directly from the cache, diff --git a/trie/hasher.go b/trie/hasher.go index 3a62a2f1199c..7f0748c13df3 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -24,22 +24,12 @@ import ( "golang.org/x/crypto/sha3" ) -type sliceBuffer []byte - -func (b *sliceBuffer) Write(data []byte) (n int, err error) { - *b = append(*b, data...) - return len(data), nil -} - -func (b *sliceBuffer) Reset() { - *b = (*b)[:0] -} - // hasher is a type used for the trie Hash operation. A hasher has some // internal preallocated temp space type hasher struct { sha crypto.KeccakState - tmp sliceBuffer + tmp []byte + encbuf rlp.EncoderBuffer parallel bool // Whether to use paralallel threads when hashing } @@ -47,8 +37,9 @@ type hasher struct { var hasherPool = sync.Pool{ New: func() interface{} { return &hasher{ - tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. - sha: sha3.NewLegacyKeccak256().(crypto.KeccakState), + tmp: make([]byte, 0, 550), // cap is as large as a full fullNode. + sha: sha3.NewLegacyKeccak256().(crypto.KeccakState), + encbuf: rlp.NewEncoderBuffer(nil), } }, } @@ -153,30 +144,41 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached // into compact form for RLP encoding. // If the rlp data is smaller than 32 bytes, `nil` is returned. func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { - h.tmp.Reset() - if err := rlp.Encode(&h.tmp, n); err != nil { - panic("encode error: " + err.Error()) - } + n.encode(h.encbuf) + enc := h.encodedBytes() - if len(h.tmp) < 32 && !force { + if len(enc) < 32 && !force { return n // Nodes smaller than 32 bytes are stored inside their parent } - return h.hashData(h.tmp) + return h.hashData(enc) } // shortnodeToHash is used to creates a hashNode from a set of hashNodes, (which // may contain nil values) func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { - h.tmp.Reset() - // Generate the RLP encoding of the node - if err := n.EncodeRLP(&h.tmp); err != nil { - panic("encode error: " + err.Error()) - } + n.encode(h.encbuf) + enc := h.encodedBytes() - if len(h.tmp) < 32 && !force { + if len(enc) < 32 && !force { return n // Nodes smaller than 32 bytes are stored inside their parent } - return h.hashData(h.tmp) + return h.hashData(enc) +} + +// encodedBytes returns the result of the last encoding operation on h.encbuf. +// This also resets the encoder buffer. +// +// All node encoding must be done like this: +// +// node.encode(h.encbuf) +// enc := h.encodedBytes() +// +// This convention exists because node.encode can only be inlined/escape-analyzed when +// called on a concrete receiver type. +func (h *hasher) encodedBytes() []byte { + h.tmp = h.encbuf.AppendToBytes(h.tmp[:0]) + h.encbuf.Reset(nil) + return h.tmp } // hashData hashes the provided data diff --git a/trie/iterator.go b/trie/iterator.go index 5451cb71c60a..977bc55854e5 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/rlp" ) // Iterator is a key-value trie iterator that traverses a Trie. @@ -86,6 +85,10 @@ type NodeIterator interface { // For leaf nodes, the last element of the path is the 'terminator symbol' 0x10. Path() []byte + // NodeBlob returns the rlp-encoded value of the current iterated node. + // If the node is an embedded node in its parent, nil is returned then. + NodeBlob() []byte + // Leaf returns true iff the current node is a leaf node. Leaf() bool @@ -151,8 +154,11 @@ func (e seekError) Error() string { } func newNodeIterator(trie *Trie, start []byte) NodeIterator { - if trie.Hash() == emptyState { - return new(nodeIterator) + if trie.Hash() == emptyRoot { + return &nodeIterator{ + trie: trie, + err: errIteratorEnd, + } } it := &nodeIterator{trie: trie} it.err = it.seek(start) @@ -210,8 +216,7 @@ func (it *nodeIterator) LeafProof() [][]byte { // Gather nodes that end up as hash nodes (or the root) node, hashed := hasher.proofHash(item.node) if _, ok := hashed.(hashNode); ok || i == 0 { - enc, _ := rlp.EncodeToBytes(node) - proofs = append(proofs, enc) + proofs = append(proofs, nodeToBytes(node)) } } return proofs @@ -224,6 +229,18 @@ func (it *nodeIterator) Path() []byte { return it.path } +func (it *nodeIterator) NodeBlob() []byte { + if it.Hash() == (common.Hash{}) { + return nil // skip the non-standalone node + } + blob, err := it.resolveBlob(it.Hash().Bytes(), it.Path()) + if err != nil { + it.err = err + return nil + } + return blob +} + func (it *nodeIterator) Error() error { if it.err == errIteratorEnd { return nil @@ -362,6 +379,15 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { return resolved, err } +func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { + if it.resolver != nil { + if blob, err := it.resolver.Get(hash); err == nil && len(blob) > 0 { + return blob, nil + } + } + return it.trie.resolveBlob(hash, path) +} + func (st *nodeIteratorState) resolve(it *nodeIterator, path []byte) error { if hash, ok := st.node.(hashNode); ok { resolved, err := it.resolveHash(hash, path) @@ -402,7 +428,7 @@ func findChild(n *fullNode, index int, path []byte, ancestor common.Hash) (node, func (it *nodeIterator) nextChild(parent *nodeIteratorState, ancestor common.Hash) (*nodeIteratorState, []byte, bool) { switch node := parent.node.(type) { case *fullNode: - //Full node, move to the first non-nil child. + // Full node, move to the first non-nil child. if child, state, path, index := findChild(node, parent.index+1, it.path, ancestor); child != nil { parent.index = index - 1 return state, path, true @@ -480,8 +506,9 @@ func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path [] } func (it *nodeIterator) pop() { - parent := it.stack[len(it.stack)-1] - it.path = it.path[:parent.pathlen] + last := it.stack[len(it.stack)-1] + it.path = it.path[:last.pathlen] + it.stack[len(it.stack)-1] = nil it.stack = it.stack[:len(it.stack)-1] } @@ -549,6 +576,10 @@ func (it *differenceIterator) Path() []byte { return it.b.Path() } +func (it *differenceIterator) NodeBlob() []byte { + return it.b.NodeBlob() +} + func (it *differenceIterator) AddResolver(resolver ethdb.KeyValueReader) { panic("not implemented") } @@ -660,6 +691,10 @@ func (it *unionIterator) Path() []byte { return (*it.items)[0].Path() } +func (it *unionIterator) NodeBlob() []byte { + return (*it.items)[0].NodeBlob() +} + func (it *unionIterator) AddResolver(resolver ethdb.KeyValueReader) { panic("not implemented") } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 95cafdd3bdce..9a46e9b99548 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -29,6 +29,19 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) +func TestEmptyIterator(t *testing.T) { + trie := newEmpty() + iter := trie.NodeIterator(nil) + + seen := make(map[string]struct{}) + for iter.Next(true) { + seen[string(iter.Path())] = struct{}{} + } + if len(seen) != 0 { + t.Fatal("Unexpected trie node iterated") + } +} + func TestIterator(t *testing.T) { trie := newEmpty() vals := []struct{ k, v string }{ @@ -470,10 +483,18 @@ func (l *loggingDb) NewBatch() ethdb.Batch { return l.backend.NewBatch() } +func (l *loggingDb) NewBatchWithSize(size int) ethdb.Batch { + return l.backend.NewBatchWithSize(size) +} + func (l *loggingDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator { - fmt.Printf("NewIterator\n") return l.backend.NewIterator(prefix, start) } + +func (l *loggingDb) NewSnapshot() (ethdb.Snapshot, error) { + return l.backend.NewSnapshot() +} + func (l *loggingDb) Stat(property string) (string, error) { return l.backend.Stat(property) } @@ -521,3 +542,54 @@ func TestNodeIteratorLargeTrie(t *testing.T) { t.Fatalf("Too many lookups during seek, have %d want %d", have, want) } } + +func TestIteratorNodeBlob(t *testing.T) { + var ( + db = memorydb.New() + triedb = NewDatabase(db) + trie, _ = New(common.Hash{}, triedb) + ) + vals := []struct{ k, v string }{ + {"do", "verb"}, + {"ether", "wookiedoo"}, + {"horse", "stallion"}, + {"shaman", "horse"}, + {"doge", "coin"}, + {"dog", "puppy"}, + {"somethingveryoddindeedthis is", "myothernodedata"}, + } + all := make(map[string]string) + for _, val := range vals { + all[val.k] = val.v + trie.Update([]byte(val.k), []byte(val.v)) + } + trie.Commit(nil) + triedb.Cap(0) + + found := make(map[common.Hash][]byte) + it := trie.NodeIterator(nil) + for it.Next(true) { + if it.Hash() == (common.Hash{}) { + continue + } + found[it.Hash()] = it.NodeBlob() + } + + dbIter := db.NewIterator(nil, nil) + defer dbIter.Release() + + var count int + for dbIter.Next() { + got, present := found[common.BytesToHash(dbIter.Key())] + if !present { + t.Fatalf("Miss trie node %v", dbIter.Key()) + } + if !bytes.Equal(got, dbIter.Value()) { + t.Fatalf("Unexpected trie node want %v got %v", dbIter.Value(), got) + } + count += 1 + } + if count != len(found) { + t.Fatal("Find extra trie node via iterator") + } +} diff --git a/trie/node.go b/trie/node.go index f4055e779a1b..bf3f024bb8a7 100644 --- a/trie/node.go +++ b/trie/node.go @@ -28,8 +28,9 @@ import ( var indices = []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "[17]"} type node interface { - fstring(string) string cache() (hashNode, bool) + encode(w rlp.EncoderBuffer) + fstring(string) string } type ( @@ -52,16 +53,9 @@ var nilValueNode = valueNode(nil) // EncodeRLP encodes a full node into the consensus RLP format. func (n *fullNode) EncodeRLP(w io.Writer) error { - var nodes [17]node - - for i, child := range &n.Children { - if child != nil { - nodes[i] = child - } else { - nodes[i] = nilValueNode - } - } - return rlp.Encode(w, nodes) + eb := rlp.NewEncoderBuffer(w) + n.encode(eb) + return eb.Flush() } func (n *fullNode) copy() *fullNode { copy := *n; return © } diff --git a/trie/node_enc.go b/trie/node_enc.go new file mode 100644 index 000000000000..cade35b707c2 --- /dev/null +++ b/trie/node_enc.go @@ -0,0 +1,87 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "github.com/ethereum/go-ethereum/rlp" +) + +func nodeToBytes(n node) []byte { + w := rlp.NewEncoderBuffer(nil) + n.encode(w) + result := w.ToBytes() + w.Flush() + return result +} + +func (n *fullNode) encode(w rlp.EncoderBuffer) { + offset := w.List() + for _, c := range n.Children { + if c != nil { + c.encode(w) + } else { + w.Write(rlp.EmptyString) + } + } + w.ListEnd(offset) +} + +func (n *shortNode) encode(w rlp.EncoderBuffer) { + offset := w.List() + w.WriteBytes(n.Key) + if n.Val != nil { + n.Val.encode(w) + } else { + w.Write(rlp.EmptyString) + } + w.ListEnd(offset) +} + +func (n hashNode) encode(w rlp.EncoderBuffer) { + w.WriteBytes(n) +} + +func (n valueNode) encode(w rlp.EncoderBuffer) { + w.WriteBytes(n) +} + +func (n rawFullNode) encode(w rlp.EncoderBuffer) { + offset := w.List() + for _, c := range n { + if c != nil { + c.encode(w) + } else { + w.Write(rlp.EmptyString) + } + } + w.ListEnd(offset) +} + +func (n *rawShortNode) encode(w rlp.EncoderBuffer) { + offset := w.List() + w.WriteBytes(n.Key) + if n.Val != nil { + n.Val.encode(w) + } else { + w.Write(rlp.EmptyString) + } + w.ListEnd(offset) +} + +func (n rawNode) encode(w rlp.EncoderBuffer) { + w.Write(n) +} diff --git a/trie/proof.go b/trie/proof.go index 9be3b62216a8..88ca80b0e706 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) // Prove constructs a merkle proof for key. The result contains all encoded nodes @@ -79,7 +78,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e if hash, ok := hn.(hashNode); ok || i == 0 { // If the node's database encoding is a hash (or is the // root node), it becomes a proof element. - enc, _ := rlp.EncodeToBytes(n) + enc := nodeToBytes(n) if !ok { hash = hasher.hashData(enc) } diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 76258c31123c..b38bb01b0fb3 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) var ErrCommitDisabled = errors.New("no database for committing") @@ -224,6 +223,7 @@ func (st *StackTrie) insert(key, value []byte) { switch st.nodeType { case branchNode: /* Branch */ idx := int(key[0]) + // Unresolve elder siblings for i := idx - 1; i >= 0; i-- { if st.children[i] != nil { @@ -233,12 +233,14 @@ func (st *StackTrie) insert(key, value []byte) { break } } + // Add new child if st.children[idx] == nil { st.children[idx] = newLeaf(key[1:], value, st.db) } else { st.children[idx].insert(key[1:], value) } + case extNode: /* Ext */ // Compare both key chunks and see where they differ diffidx := st.getDiffIndex(key) @@ -326,10 +328,9 @@ func (st *StackTrie) insert(key, value []byte) { p = st.children[0] } - // Create the two child leaves: the one containing the - // original value and the one containing the new value - // The child leave will be hashed directly in order to - // free up some memory. + // Create the two child leaves: one containing the original + // value and another containing the new value. The child leaf + // is hashed directly in order to free up some memory. origIdx := st.key[diffidx] p.children[origIdx] = newLeaf(st.key[diffidx+1:], st.val, st.db) p.children[origIdx].hash() @@ -341,19 +342,22 @@ func (st *StackTrie) insert(key, value []byte) { // over to the children. st.key = st.key[:diffidx] st.val = nil + case emptyNode: /* Empty */ st.nodeType = leafNode st.key = key st.val = value + case hashedNode: panic("trying to insert into hash") + default: panic("invalid type") } } -// hash() hashes the node 'st' and converts it into 'hashedNode', if possible. -// Possible outcomes: +// hash converts st into a 'hashedNode', if possible. Possible outcomes: +// // 1. The rlp-encoded value was >= 32 bytes: // - Then the 32-byte `hash` will be accessible in `st.val`. // - And the 'st.type' will be 'hashedNode' @@ -361,119 +365,116 @@ func (st *StackTrie) insert(key, value []byte) { // - Then the <32 byte rlp-encoded value will be accessible in 'st.val'. // - And the 'st.type' will be 'hashedNode' AGAIN // -// This method will also: -// set 'st.type' to hashedNode -// clear 'st.key' +// This method also sets 'st.type' to hashedNode, and clears 'st.key'. func (st *StackTrie) hash() { - /* Shortcut if node is already hashed */ - if st.nodeType == hashedNode { - return - } - // The 'hasher' is taken from a pool, but we don't actually - // claim an instance until all children are done with their hashing, - // and we actually need one - var h *hasher + h := newHasher(false) + defer returnHasherToPool(h) + + st.hashRec(h) +} + +func (st *StackTrie) hashRec(hasher *hasher) { + // The switch below sets this to the RLP-encoding of this node. + var encodedNode []byte switch st.nodeType { + case hashedNode: + return + + case emptyNode: + st.val = emptyRoot.Bytes() + st.key = st.key[:0] + st.nodeType = hashedNode + return + case branchNode: - var nodes [17]node + var nodes rawFullNode for i, child := range st.children { if child == nil { nodes[i] = nilValueNode continue } - child.hash() + + child.hashRec(hasher) if len(child.val) < 32 { nodes[i] = rawNode(child.val) } else { nodes[i] = hashNode(child.val) } - st.children[i] = nil // Reclaim mem from subtree + + // Release child back to pool. + st.children[i] = nil returnToPool(child) } - nodes[16] = nilValueNode - h = newHasher(false) - defer returnHasherToPool(h) - h.tmp.Reset() - if err := rlp.Encode(&h.tmp, nodes); err != nil { - panic(err) - } + + nodes.encode(hasher.encbuf) + encodedNode = hasher.encodedBytes() + case extNode: - st.children[0].hash() - h = newHasher(false) - defer returnHasherToPool(h) - h.tmp.Reset() - var valuenode node + st.children[0].hashRec(hasher) + + sz := hexToCompactInPlace(st.key) + n := rawShortNode{Key: st.key[:sz]} if len(st.children[0].val) < 32 { - valuenode = rawNode(st.children[0].val) + n.Val = rawNode(st.children[0].val) } else { - valuenode = hashNode(st.children[0].val) - } - n := struct { - Key []byte - Val node - }{ - Key: hexToCompact(st.key), - Val: valuenode, - } - if err := rlp.Encode(&h.tmp, n); err != nil { - panic(err) + n.Val = hashNode(st.children[0].val) } + + n.encode(hasher.encbuf) + encodedNode = hasher.encodedBytes() + + // Release child back to pool. returnToPool(st.children[0]) - st.children[0] = nil // Reclaim mem from subtree + st.children[0] = nil + case leafNode: - h = newHasher(false) - defer returnHasherToPool(h) - h.tmp.Reset() st.key = append(st.key, byte(16)) sz := hexToCompactInPlace(st.key) - n := [][]byte{st.key[:sz], st.val} - if err := rlp.Encode(&h.tmp, n); err != nil { - panic(err) - } - case emptyNode: - st.val = emptyRoot.Bytes() - st.key = st.key[:0] - st.nodeType = hashedNode - return + n := rawShortNode{Key: st.key[:sz], Val: valueNode(st.val)} + + n.encode(hasher.encbuf) + encodedNode = hasher.encodedBytes() + default: - panic("Invalid node type") + panic("invalid node type") } - st.key = st.key[:0] + st.nodeType = hashedNode - if len(h.tmp) < 32 { - st.val = common.CopyBytes(h.tmp) + st.key = st.key[:0] + if len(encodedNode) < 32 { + st.val = common.CopyBytes(encodedNode) return } + // Write the hash to the 'val'. We allocate a new val here to not mutate // input values - st.val = make([]byte, 32) - h.sha.Reset() - h.sha.Write(h.tmp) - h.sha.Read(st.val) + st.val = hasher.hashData(encodedNode) if st.db != nil { // TODO! Is it safe to Put the slice here? // Do all db implementations copy the value provided? - st.db.Put(st.val, h.tmp) + st.db.Put(st.val, encodedNode) } } -// Hash returns the hash of the current node +// Hash returns the hash of the current node. func (st *StackTrie) Hash() (h common.Hash) { - st.hash() - if len(st.val) != 32 { - // If the node's RLP isn't 32 bytes long, the node will not - // be hashed, and instead contain the rlp-encoding of the - // node. For the top level node, we need to force the hashing. - ret := make([]byte, 32) - h := newHasher(false) - defer returnHasherToPool(h) - h.sha.Reset() - h.sha.Write(st.val) - h.sha.Read(ret) - return common.BytesToHash(ret) + hasher := newHasher(false) + defer returnHasherToPool(hasher) + + st.hashRec(hasher) + if len(st.val) == 32 { + copy(h[:], st.val) + return h } - return common.BytesToHash(st.val) + + // If the node's RLP isn't 32 bytes long, the node will not + // be hashed, and instead contain the rlp-encoding of the + // node. For the top level node, we need to force the hashing. + hasher.sha.Reset() + hasher.sha.Write(st.val) + hasher.sha.Read(h[:]) + return h } // Commit will firstly hash the entrie trie if it's still not hashed @@ -483,23 +484,26 @@ func (st *StackTrie) Hash() (h common.Hash) { // // The associated database is expected, otherwise the whole commit // functionality should be disabled. -func (st *StackTrie) Commit() (common.Hash, error) { +func (st *StackTrie) Commit() (h common.Hash, err error) { if st.db == nil { return common.Hash{}, ErrCommitDisabled } - st.hash() - if len(st.val) != 32 { - // If the node's RLP isn't 32 bytes long, the node will not - // be hashed (and committed), and instead contain the rlp-encoding of the - // node. For the top level node, we need to force the hashing+commit. - ret := make([]byte, 32) - h := newHasher(false) - defer returnHasherToPool(h) - h.sha.Reset() - h.sha.Write(st.val) - h.sha.Read(ret) - st.db.Put(ret, st.val) - return common.BytesToHash(ret), nil + + hasher := newHasher(false) + defer returnHasherToPool(hasher) + + st.hashRec(hasher) + if len(st.val) == 32 { + copy(h[:], st.val) + return h, nil } - return common.BytesToHash(st.val), nil + + // If the node's RLP isn't 32 bytes long, the node will not + // be hashed (and committed), and instead contain the rlp-encoding of the + // node. For the top level node, we need to force the hashing+commit. + hasher.sha.Reset() + hasher.sha.Write(st.val) + hasher.sha.Read(h[:]) + st.db.Put(h[:], st.val) + return h, nil } diff --git a/trie/trie.go b/trie/trie.go index 13343112b8d7..e40b03be38c3 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -514,6 +514,15 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { return nil, &MissingNodeError{NodeHash: hash, Path: prefix} } +func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { + hash := common.BytesToHash(n) + blob, _ := t.db.Node(hash) + if len(blob) != 0 { + return blob, nil + } + return nil, &MissingNodeError{NodeHash: hash, Path: prefix} +} + // Hash returns the root hash of the trie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *Trie) Hash() common.Hash { diff --git a/trie/trie_test.go b/trie/trie_test.go index be0df8a54426..a1fdc8cd58c4 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -414,8 +414,9 @@ func runRandTest(rt randTest) bool { values := make(map[string]string) // tracks content of the trie for i, step := range rt { - fmt.Printf("{op: %d, key: common.Hex2Bytes(\"%x\"), value: common.Hex2Bytes(\"%x\")}, // step %d\n", - step.op, step.key, step.value, i) + // fmt.Printf("{op: %d, key: common.Hex2Bytes(\"%x\"), value: common.Hex2Bytes(\"%x\")}, // step %d\n", + // step.op, step.key, step.value, i) + switch step.op { case opUpdate: tr.Update(step.key, step.value) @@ -675,6 +676,8 @@ func (s *spongeDb) Has(key []byte) (bool, error) { panic("implement func (s *spongeDb) Get(key []byte) ([]byte, error) { return nil, errors.New("no such elem") } func (s *spongeDb) Delete(key []byte) error { panic("implement me") } func (s *spongeDb) NewBatch() ethdb.Batch { return &spongeBatch{s} } +func (s *spongeDb) NewBatchWithSize(size int) ethdb.Batch { return &spongeBatch{s} } +func (s *spongeDb) NewSnapshot() (ethdb.Snapshot, error) { panic("implement me") } func (s *spongeDb) Stat(property string) (string, error) { panic("implement me") } func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Close() error { return nil } @@ -884,7 +887,8 @@ func TestCommitSequenceSmallRoot(t *testing.T) { if stRoot != root { t.Fatalf("root wrong, got %x exp %x", stRoot, root) } - fmt.Printf("root: %x\n", stRoot) + + t.Logf("root: %x\n", stRoot) if got, exp := stackTrieSponge.sponge.Sum(nil), s.sponge.Sum(nil); !bytes.Equal(got, exp) { t.Fatalf("test, disk write sequence wrong:\ngot %x exp %x\n", got, exp) }