e3: CanonicalHash to return ok as found-marker (#11942)

to fix: 
```
curl --data '{"method":"debug_traceBlockByNumber","params":["0x13C2924"],"id":1,"jsonrpc":"2.0"}' -H "Content-Type: application/json" -X POST localhost:8549
{"jsonrpc":"2.0","id":1,"result":null,"error":{"code":-32000,"message":"invalid arguments; block with hash 0000000000000000000000000000000000000000000000000000000000000000 not found"}}
```

accounts/abi/bind/backends/simulated.go

@@ -317,10 +317,13 @@ func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash libcomm
 	if !ok {
 		return nil, false, ethereum.NotFound
 	}
-	blockHash, err := b.BlockReader().CanonicalHash(ctx, tx, blockNumber)
+	blockHash, ok, err := b.BlockReader().CanonicalHash(ctx, tx, blockNumber)
 	if err != nil {
 		return nil, false, err
 	}
+	if !ok {
+		return nil, false, ethereum.NotFound
+	}
 	body, err := b.BlockReader().BodyWithTransactions(ctx, tx, blockHash, blockNumber)
 	if err != nil {
 		return nil, false, err

cmd/hack/hack.go

@@ -223,7 +223,7 @@ func extractHashes(chaindata string, blockStep uint64, blockTotalOrOffset int64,
 		blockTotal := getBlockTotal(tx, b, blockTotalOrOffset)
 		// Note: blockTotal used here as block number rather than block count
 		for b <= blockTotal {
-			hash, err := br.CanonicalHash(context.Background(), tx, b)
+			hash, _, err := br.CanonicalHash(context.Background(), tx, b)
 			if err != nil {
 				return err
 			}
@@ -340,7 +340,7 @@ func extractBodies(datadir string) error {
 		blockNumber := binary.BigEndian.Uint64(k[:8])
 		blockHash := libcommon.BytesToHash(k[8:])
 		var hash libcommon.Hash
-		if hash, err = br.CanonicalHash(context.Background(), tx, blockNumber); err != nil {
+		if hash, _, err = br.CanonicalHash(context.Background(), tx, blockNumber); err != nil {
 			return err
 		}
 		_, baseTxnID, txCount := rawdb.ReadBody(tx, blockHash, blockNumber)

cmd/integration/commands/stages.go

@@ -743,10 +743,13 @@ func stageHeaders(db kv.RwDB, ctx context.Context, logger log.Logger) error {
 		if err = rawdb.TruncateTd(tx, progress+1); err != nil {
 			return err
 		}
-		hash, err := br.CanonicalHash(ctx, tx, progress-1)
+		hash, ok, err := br.CanonicalHash(ctx, tx, progress-1)
 		if err != nil {
 			return err
 		}
+		if !ok {
+			return fmt.Errorf("canonical hash not found: %d", progress-1)
+		}
 		if err = rawdb.WriteHeadHeaderHash(tx, hash); err != nil {
 			return err
 		}

cmd/rpcdaemon/rpcservices/eth_backend.go

@@ -91,11 +91,11 @@ func (back *RemoteBackend) ReadAncestor(db kv.Getter, hash common.Hash, number,
 	panic("not implemented")
 }
 func (back *RemoteBackend) BlockByNumber(ctx context.Context, db kv.Tx, number uint64) (*types.Block, error) {
-	hash, err := back.CanonicalHash(ctx, db, number)
+	hash, ok, err := back.CanonicalHash(ctx, db, number)
 	if err != nil {
 		return nil, fmt.Errorf("failed ReadCanonicalHash: %w", err)
 	}
-	if hash == (common.Hash{}) {
+	if !ok || hash == (common.Hash{}) {
 		return nil, nil
 	}
 	block, _, err := back.BlockWithSenders(ctx, db, hash, number)
@@ -298,7 +298,7 @@ func (back *RemoteBackend) HeaderByNumber(ctx context.Context, tx kv.Getter, blo
 func (back *RemoteBackend) HeaderByHash(ctx context.Context, tx kv.Getter, hash common.Hash) (*types.Header, error) {
 	return back.blockReader.HeaderByHash(ctx, tx, hash)
 }
-func (back *RemoteBackend) CanonicalHash(ctx context.Context, tx kv.Getter, blockNum uint64) (common.Hash, error) {
+func (back *RemoteBackend) CanonicalHash(ctx context.Context, tx kv.Getter, blockNum uint64) (common.Hash, bool, error) {
 	return back.blockReader.CanonicalHash(ctx, tx, blockNum)
 }
 func (back *RemoteBackend) HeaderNumber(ctx context.Context, tx kv.Getter, hash common.Hash) (*uint64, error) {

cmd/state/verify/verify_txlookup.go

@@ -74,10 +74,14 @@ func ValidateTxLookups(chaindata string, logger log.Logger) error {
 		if err := libcommon.Stopped(quitCh); err != nil {
 			return err
 		}
-		blockHash, err := br.CanonicalHash(ctx, tx, blockNum)
+		blockHash, ok, err := br.CanonicalHash(ctx, tx, blockNum)
 		if err != nil {
 			return err
 		}
+		if !ok {
+			logger.Error("no canonnical hash", "blocknum", blockNum)
+			break
+		}
 		body, err := br.BodyWithTransactions(ctx, tx, blockHash, blockNum)
 		if err != nil {
 			return err

core/rawdb/accessors_chain_test.go

@@ -378,7 +378,7 @@ func TestCanonicalMappingStorage(t *testing.T) {
 
 	// Create a test canonical number and assinged hash to move around
 	hash, number := libcommon.Hash{0: 0xff}, uint64(314)
-	entry, err := br.CanonicalHash(m.Ctx, tx, number)
+	entry, _, err := br.CanonicalHash(m.Ctx, tx, number)
 	if err != nil {
 		t.Fatalf("ReadCanonicalHash failed: %v", err)
 	}
@@ -390,7 +390,7 @@ func TestCanonicalMappingStorage(t *testing.T) {
 	if err != nil {
 		t.Fatalf("WriteCanoncalHash failed: %v", err)
 	}
-	entry, err = br.CanonicalHash(m.Ctx, tx, number)
+	entry, _, err = br.CanonicalHash(m.Ctx, tx, number)
 	if err != nil {
 		t.Fatalf("ReadCanonicalHash failed: %v", err)
 	}
@@ -404,7 +404,7 @@ func TestCanonicalMappingStorage(t *testing.T) {
 	if err != nil {
 		t.Fatalf("DeleteCanonicalHash failed: %v", err)
 	}
-	entry, err = br.CanonicalHash(m.Ctx, tx, number)
+	entry, _, err = br.CanonicalHash(m.Ctx, tx, number)
 	if err != nil {
 		t.Error(err)
 	}

core/rawdb/accessors_indexes_test.go

@@ -122,11 +122,11 @@ func readTransactionByHash(db kv.Tx, hash libcommon.Hash, br services.FullBlockR
 	if blockNumber == nil {
 		return nil, libcommon.Hash{}, 0, 0, nil
 	}
-	blockHash, err := br.CanonicalHash(context.Background(), db, *blockNumber)
+	blockHash, ok, err := br.CanonicalHash(context.Background(), db, *blockNumber)
 	if err != nil {
 		return nil, libcommon.Hash{}, 0, 0, err
 	}
-	if blockHash == (libcommon.Hash{}) {
+	if !ok || blockHash == (libcommon.Hash{}) {
 		return nil, libcommon.Hash{}, 0, 0, nil
 	}
 	body, _ := br.BodyWithTransactions(context.Background(), db, blockHash, *blockNumber)

eth/integrity/no_gaps_in_canonical_headers.go

@@ -45,11 +45,11 @@ func NoGapsInCanonicalHeaders(tx kv.Tx, ctx context.Context, br services.FullBlo
 	}
 
 	for i := firstBlockInDB; i < lastBlockNum; i++ {
-		hash, err := br.CanonicalHash(ctx, tx, i)
+		hash, ok, err := br.CanonicalHash(ctx, tx, i)
 		if err != nil {
 			panic(err)
 		}
-		if hash == (common.Hash{}) {
+		if !ok || hash == (common.Hash{}) {
 			err = fmt.Errorf("canonical marker not found: %d", i)
 			panic(err)
 		}

eth/stagedsync/stage_execute.go

@@ -191,11 +191,13 @@ func unwindExec3(u *UnwindState, s *StageState, txc wrap.TxContainer, ctx contex
 	t := time.Now()
 	var changeset *[kv.DomainLen][]libstate.DomainEntryDiff
 	for currentBlock := u.CurrentBlockNumber; currentBlock > u.UnwindPoint; currentBlock-- {
-		currentHash, err := br.CanonicalHash(ctx, txc.Tx, currentBlock)
+		currentHash, ok, err := br.CanonicalHash(ctx, txc.Tx, currentBlock)
 		if err != nil {
 			return err
 		}
-		var ok bool
+		if !ok {
+			return fmt.Errorf("canonical hash not found %d", currentBlock)
+		}
 		var currentKeys [kv.DomainLen][]libstate.DomainEntryDiff
 		currentKeys, ok, err = domains.GetDiffset(txc.Tx, currentHash, currentBlock)
 		if !ok {
@@ -362,10 +364,13 @@ func unwindExecutionStage(u *UnwindState, s *StageState, txc wrap.TxContainer, c
 	if cfg.stateStream && s.BlockNumber-u.UnwindPoint < stateStreamLimit {
 		accumulator = cfg.notifications.Accumulator
 
-		hash, err := cfg.blockReader.CanonicalHash(ctx, txc.Tx, u.UnwindPoint)
+		hash, ok, err := cfg.blockReader.CanonicalHash(ctx, txc.Tx, u.UnwindPoint)
 		if err != nil {
 			return fmt.Errorf("read canonical hash of unwind point: %w", err)
 		}
+		if !ok {
+			return fmt.Errorf("canonical hash not found %d", u.UnwindPoint)
+		}
 		txs, err := cfg.blockReader.RawTransactions(ctx, txc.Tx, u.UnwindPoint, s.BlockNumber)
 		if err != nil {
 			return err

eth/stagedsync/stage_headers.go

@@ -141,16 +141,16 @@ func HeadersPOW(s *StageState, u Unwinder, ctx context.Context, tx kv.RwTx, cfg
 	defer logEvery.Stop()
 
 	// Check if this is called straight after the unwinds, which means we need to create new canonical markings
-	hash, err := cfg.blockReader.CanonicalHash(ctx, tx, startProgress)
+	hash, ok, err := cfg.blockReader.CanonicalHash(ctx, tx, startProgress)
 	if err != nil {
 		return err
 	}
-	if hash == (libcommon.Hash{}) { // restore canonical markers after unwind
+	if !ok || hash == (libcommon.Hash{}) { // restore canonical markers after unwind
 		headHash := rawdb.ReadHeadHeaderHash(tx)
 		if err = fixCanonicalChain(logPrefix, logEvery, startProgress, headHash, tx, cfg.blockReader, logger); err != nil {
 			return err
 		}
-		hash, err = cfg.blockReader.CanonicalHash(ctx, tx, startProgress)
+		hash, _, err = cfg.blockReader.CanonicalHash(ctx, tx, startProgress)
 		if err != nil {
 			return err
 		}
@@ -377,7 +377,7 @@ func fixCanonicalChain(logPrefix string, logEvery *time.Ticker, height uint64, h
 
 	var ch libcommon.Hash
 	var err error
-	for ch, err = headerReader.CanonicalHash(context.Background(), tx, ancestorHeight); err == nil && ch != ancestorHash; ch, err = headerReader.CanonicalHash(context.Background(), tx, ancestorHeight) {
+	for ch, _, err = headerReader.CanonicalHash(context.Background(), tx, ancestorHeight); err == nil && ch != ancestorHash; ch, _, err = headerReader.CanonicalHash(context.Background(), tx, ancestorHeight) {
 		if err = rawdb.WriteCanonicalHash(tx, ancestorHash, ancestorHeight); err != nil {
 			return fmt.Errorf("marking canonical header %d %x: %w", ancestorHeight, ancestorHash, err)
 		}
@@ -497,9 +497,14 @@ func HeadersUnwind(ctx context.Context, u *UnwindState, s *StageState, tx kv.RwT
 		*/
 		if maxNum == 0 {
 			maxNum = u.UnwindPoint
-			if maxHash, err = cfg.blockReader.CanonicalHash(ctx, tx, maxNum); err != nil {
+			var ok bool
+			if maxHash, ok, err = cfg.blockReader.CanonicalHash(ctx, tx, maxNum); err != nil {
 				return err
 			}
+			if !ok {
+				return fmt.Errorf("not found canonical marker: %d", maxNum)
+			}
+
 		}
 		if err = rawdb.WriteHeadHeaderHash(tx, maxHash); err != nil {
 			return err

eth/stagedsync/stage_polygon_sync.go

@@ -220,10 +220,13 @@ func UnwindPolygonSyncStage(ctx context.Context, tx kv.RwTx, u *UnwindState, cfg
 		return err
 	}
 
-	canonicalHash, err := cfg.blockReader.CanonicalHash(ctx, tx, u.UnwindPoint)
+	canonicalHash, ok, err := cfg.blockReader.CanonicalHash(ctx, tx, u.UnwindPoint)
 	if err != nil {
 		return err
 	}
+	if !ok {
+		return fmt.Errorf("canonical marker not found: %d", u.UnwindPoint)
+	}
 
 	if err = rawdb.WriteHeadHeaderHash(tx, canonicalHash); err != nil {
 		return err
@@ -1564,10 +1567,14 @@ func (e *polygonSyncStageExecutionEngine) connectTip(
 	var emptyHash common.Hash
 	var ch common.Hash
 	for {
-		ch, err = e.blockReader.CanonicalHash(ctx, tx, blockNum)
+		var ok bool
+		ch, ok, err = e.blockReader.CanonicalHash(ctx, tx, blockNum)
 		if err != nil {
 			return nil, nil, fmt.Errorf("connectTip reading canonical hash for %d: %w", blockNum, err)
 		}
+		if !ok {
+			return nil, nil, fmt.Errorf("connectTip canonical hash not found. blockNum %d", blockNum)
+		}
 		if ch == blockHash {
 			break
 		}

eth/stagedsync/stage_snapshots.go

@@ -434,10 +434,13 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, dirs
 			if err := h2n.Load(tx, kv.HeaderNumber, etl.IdentityLoadFunc, etl.TransformArgs{}); err != nil {
 				return err
 			}
-			canonicalHash, err := blockReader.CanonicalHash(ctx, tx, blocksAvailable)
+			canonicalHash, ok, err := blockReader.CanonicalHash(ctx, tx, blocksAvailable)
 			if err != nil {
 				return err
 			}
+			if !ok {
+				return fmt.Errorf("canonical marker not found: %d", blocksAvailable)
+			}
 			if err = rawdb.WriteHeadHeaderHash(tx, canonicalHash); err != nil {
 				return err
 			}

ethdb/privateapi/ethbackend.go

@@ -284,10 +284,13 @@ func (s *EthBackendServer) CanonicalHash(ctx context.Context, req *remote.Canoni
 	}
 	defer tx.Rollback()
 
-	hash, err := s.blockReader.CanonicalHash(ctx, tx, req.BlockNumber)
+	hash, ok, err := s.blockReader.CanonicalHash(ctx, tx, req.BlockNumber)
 	if err != nil {
 		return nil, err
 	}
+	if !ok {
+		return nil, nil
+	}
 	return &remote.CanonicalHashReply{Hash: gointerfaces.ConvertHashToH256(hash)}, nil
 }
 

tests/block_test_util.go

@@ -224,7 +224,7 @@ func (bt *BlockTest) insertBlocks(m *mock.MockSentry) ([]btBlock, error) {
 				return nil, err
 			}
 			defer roTx.Rollback()
-			canonical, cErr := bt.br.CanonicalHash(context.Background(), roTx, cb.NumberU64())
+			canonical, _, cErr := bt.br.CanonicalHash(context.Background(), roTx, cb.NumberU64())
 			if cErr != nil {
 				return nil, cErr
 			}

turbo/engineapi/engine_helpers/fork_validator.go

@@ -320,10 +320,15 @@ func (fv *ForkValidator) validateAndStorePayload(txc wrap.TxContainer, header *t
 		if criticalError != nil {
 			return
 		}
-		latestValidHash, criticalError = fv.blockReader.CanonicalHash(fv.ctx, txc.Tx, latestValidNumber)
+		var ok bool
+		latestValidHash, ok, criticalError = fv.blockReader.CanonicalHash(fv.ctx, txc.Tx, latestValidNumber)
 		if criticalError != nil {
 			return
 		}
+		if !ok {
+			criticalError = fmt.Errorf("canonical hash not found: %d", latestValidNumber)
+			return
+		}
 		status = engine_types.InvalidStatus
 		if fv.sharedDom != nil {
 			fv.sharedDom.Close()

turbo/execution/eth1/ethereum_execution.go

@@ -160,11 +160,18 @@ func (e *EthereumExecutionModule) canonicalHash(ctx context.Context, tx kv.Tx, b
 	var err error
 	if e.blockReader == nil {
 		canonical, err = rawdb.ReadCanonicalHash(tx, blockNumber)
+		if err != nil {
+			return libcommon.Hash{}, err
+		}
 	} else {
-		canonical, err = e.blockReader.CanonicalHash(ctx, tx, blockNumber)
-	}
-	if err != nil {
-		return libcommon.Hash{}, err
+		var ok bool
+		canonical, ok, err = e.blockReader.CanonicalHash(ctx, tx, blockNumber)
+		if err != nil {
+			return libcommon.Hash{}, err
+		}
+		if !ok {
+			return libcommon.Hash{}, nil
+		}
 	}
 
 	td, err := rawdb.ReadTd(tx, canonical, blockNumber)

turbo/jsonrpc/debug_api.go

@@ -313,7 +313,13 @@ func (api *PrivateDebugAPIImpl) AccountAt(ctx context.Context, blockHash common.
 	if number == nil {
 		return nil, nil // not error, see https://github.com/erigontech/erigon/issues/1645
 	}
-	canonicalHash, _ := api._blockReader.CanonicalHash(ctx, tx, *number)
+	canonicalHash, ok, err := api._blockReader.CanonicalHash(ctx, tx, *number)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, fmt.Errorf("canonical hash not found %d", *number)
+	}
 	isCanonical := canonicalHash == blockHash
 	if !isCanonical {
 		return nil, errors.New("block hash is not canonical")

turbo/jsonrpc/debug_api_test.go

@@ -465,11 +465,11 @@ func TestAccountAt(t *testing.T) {
 
 	var blockHash0, blockHash1, blockHash3, blockHash10, blockHash12 common.Hash
 	_ = m.DB.View(m.Ctx, func(tx kv.Tx) error {
-		blockHash0, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 0)
-		blockHash1, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 1)
-		blockHash3, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 3)
-		blockHash10, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 10)
-		blockHash12, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 12)
+		blockHash0, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 0)
+		blockHash1, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 1)
+		blockHash3, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 3)
+		blockHash10, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 10)
+		blockHash12, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 12)
 		_, _, _, _, _ = blockHash0, blockHash1, blockHash3, blockHash10, blockHash12
 		return nil
 	})

turbo/jsonrpc/eth_api.go

@@ -195,10 +195,13 @@ func (api *BaseAPI) txnLookup(ctx context.Context, tx kv.Tx, txnHash common.Hash
 }
 
 func (api *BaseAPI) blockByNumberWithSenders(ctx context.Context, tx kv.Tx, number uint64) (*types.Block, error) {
-	hash, hashErr := api._blockReader.CanonicalHash(ctx, tx, number)
+	hash, ok, hashErr := api._blockReader.CanonicalHash(ctx, tx, number)
 	if hashErr != nil {
 		return nil, hashErr
 	}
+	if !ok {
+		return nil, nil
+	}
 	return api.blockWithSenders(ctx, tx, hash, number)
 }
 

turbo/jsonrpc/eth_block.go

@@ -344,10 +344,13 @@ func (api *APIImpl) GetBlockTransactionCountByNumber(ctx context.Context, blockN
 		return nil, nil
 	}
 
-	_, txCount, err := api._blockReader.Body(ctx, tx, blockHash, blockNum)
+	body, txCount, err := api._blockReader.Body(ctx, tx, blockHash, blockNum)
 	if err != nil {
 		return nil, err
 	}
+	if body == nil {
+		return nil, nil
+	}
 
 	chainConfig, err := api.chainConfig(ctx, tx)
 	if err != nil {