Merge remote-tracking branch 'boba/boba-develop' into jyellick/merge-…

…usptream-plus-fixes-20230821

.github/workflows/boba-publish-develop.yml

@@ -0,0 +1,59 @@
+name: Publish Packages (boba-develop)
+
+on:
+  push:
+    branches:
+      - 'boba-develop'
+
+jobs:
+  publish-develop:
+    name: Publish Packages
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build l1 component
+        working-directory: ./ops-bedrock
+        run: |
+          docker build . -f ./Dockerfile.l1 -t bobanetwork/geth:latest
+
+      - name: Build l2 components
+        working-directory: .
+        run: |
+          docker build . -f ./ops-bedrock/Dockerfile.l2-erigon -t bobanetwork/erigon:latest
+          docker build . -f ./op-node/Dockerfile -t bobanetwork/op-node:latest
+          docker build . -f ./op-proposer/Dockerfile -t bobanetwork/op-proposer:latest
+          docker build . -f ./op-batcher/Dockerfile -t bobanetwork/op-batcher:latest
+
+      - name: Print images
+        working-directory: .
+        run: docker images
+
+      - name: Rename and retag images
+        working-directory: .
+        run: |
+          for i in $(docker images --format "{{.Repository}}:{{.Tag}}" | grep bobanetwork); do
+            docker image tag "$i" bobanetwork/$(echo $i | awk -F'/' '{print $2}' | awk -F':' '{print $1}'):latest
+          done
+          docker images
+
+      - name: Push images
+        working-directory: .
+        run: |
+          for i in $(docker images --format "{{.Repository}}:{{.Tag}}" | grep bobanetwork); do
+            echo "$1"
+            docker push "$i"
+          done

.github/workflows/boba-publish-master.yml

@@ -0,0 +1,58 @@
+name: Publish Packages (boba-master)
+
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  publish-develop:
+    name: Publish Packages
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Get the version
+        id: tag
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
+      - name: Print current tag
+        run: echo ${{ steps.tag.outputs.VERSION }}
+
+      - name: Build l1 component
+        working-directory: ./ops-bedrock
+        run: |
+          docker build . -f ./Dockerfile.l1 -t bobanetwork/geth:${{ steps.tag.outputs.VERSION }}
+
+      - name: Build l2 components
+        working-directory: .
+        run: |
+          docker build . -f ./ops-bedrock/Dockerfile.l2-erigon -t bobanetwork/erigon:${{ steps.tag.outputs.VERSION }}
+          docker build . -f ./op-node/Dockerfile -t bobanetwork/op-node:${{ steps.tag.outputs.VERSION }}
+          docker build . -f ./op-proposer/Dockerfile -t bobanetwork/op-proposer:${{ steps.tag.outputs.VERSION }}
+          docker build . -f ./op-batcher/Dockerfile -t bobanetwork/op-batcher:${{ steps.tag.outputs.VERSION }}
+
+      - name: Print images
+        working-directory: .
+        run: docker images
+
+      - name: Push images
+        working-directory: .
+        run: |
+          for i in $(docker images --format "{{.Repository}}:{{.Tag}}" | grep bobanetwork); do
+            echo "$1"
+            docker push "$i"
+          done

bedrock-devnet/devnet/__init__.py

@@ -208,7 +208,7 @@ def devnet_deploy(paths):
     log.info(f'Using batch inbox {batch_inbox_address}')
 
     log.info('Bringing up everything else.')
-    run_command(['docker-compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher'], cwd=paths.ops_bedrock_dir, env={
+    run_command(['docker-compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher', 'kms'], cwd=paths.ops_bedrock_dir, env={
         'PWD': paths.ops_bedrock_dir,
         'L2OO_ADDRESS': l2_output_oracle,
         'SEQUENCER_BATCH_INBOX_ADDRESS': batch_inbox_address

boba-chain-ops/ether/migrate_test.go

@@ -356,21 +356,6 @@ func TestMigrateBalancesRandomMissing(t *testing.T) {
 		err := doMigration(g, addresses, allowances, big.NewInt(0), false)
 		require.ErrorContains(t, err, "unknown storage slot")
 	}
-
-	for i := 0; i < 100; i++ {
-		g, addresses, allowances, _ := setupRandTest(t)
-
-		if len(addresses) == 0 {
-			continue
-		}
-
-		// Remove a random allowance from the list of witnesses
-		idx := rand.Intn(len(allowances))
-		allowances = append(allowances[:idx], allowances[idx+1:]...)
-
-		err := doMigration(g, addresses, allowances, big.NewInt(0), false)
-		require.ErrorContains(t, err, "unknown storage slot")
-	}
 }
 
 func randAddr(t *testing.T) common.Address {

boba-chain-ops/immutables/immutables.go

@@ -260,7 +260,7 @@ func l2Deployer(backend *backends.SimulatedBackend, opts *bind.TransactOpts, dep
 		_, tx, _, err = bindings.DeployLegacyERC20ETH(opts, backend)
 	case "BobaTuringCredit":
 		addr, tx, _, err = bindings.DeployBobaTuringCredit(opts, backend, big.NewInt(10))
-		log.Info("MMDBG BobaTuringCredit", "addr", addr)
+		log.Info("BobaTuringCredit Deployment", "err", err, "addr", addr)
 	case "BobaL2":
 		l2Bridge, ok := deployment.Args[0].(common.Address)
 		if !ok {
@@ -291,10 +291,10 @@ func l2Deployer(backend *backends.SimulatedBackend, opts *bind.TransactOpts, dep
 			_symbol,
 			uint8(_decimals),
 		)
-		log.Info("MMDBG BobaL2 Deployment", "err", err, "addr", addr)
+		log.Info("BobaL2 Deployment", "err", err, "addr", addr)
 	case "BobaGasPriceOracle":
 		addr, tx, _, err = bindings.DeployBobaGasPriceOracle(opts, backend)
-		log.Info("MMDBG BobaGasPriceOracle", "addr", addr)
+		log.Info("BobaGasPriceOracle Deployment", "err", err, "addr", addr)
 	default:
 		return tx, fmt.Errorf("unknown contract: %s", deployment.Name)
 	}

boba-community/README.md

@@ -0,0 +1,77 @@
+# Running a replica node
+
+## Docker configuration
+
+Here are instructions if you want to run boba erigon version as the replica node for OP Mainnet or Testnet.
+
+### Get the data dir
+
+The first step is to download the initial data for `op-erigon`. Thanks for the contribution from [Test in Prod](https://www.testinprod.io).
+
+1. Download the correct data directory snapshot.
+
+* [OP Mainnet](https://op-erigon-backup.mainnet.testinprod.io)
+* [OP Goerli](https://op-erigon-backup.goerli.testinprod.io)
+
+2. Create the data directory and fill it.
+
+  ```bash
+  cd op-anchorage/boba-community
+  mkdir op-db
+  cd ./op-db
+  tar xvf ~/[DIR]/op-erigon-goerli.tar
+  ```
+
+3. Create a shared secret (JWT token)
+
+  ```bash
+  cd op-anchorage/boba-community
+  openssl rand -hex 32 > jwt-secret.txt
+  ```
+
+### Create a .env file
+
+Create a  `.env` file in `boba-community`.
+
+```
+VERSION=
+ETH1_HTTP=
+```
+
+> This step is optional, but we recommand you to use a latest release image for `VERSION`. Otherwise, it pulls the image with the `latest` tag.
+
+### Modify volume location
+
+The volumes of l2 and op-node should be modified to your file locations.
+
+```yaml
+l2:
+  volumes:
+    - ./jwt-secret.txt:/config/jwt-secret.txt
+    - DATA_DIR:/db
+op-node:
+  volumes:
+  	- ./jwt-secret.txt:/config/jwt-secret.txt
+```
+
+### Start your replica node
+
+```bash
+docker-compose -f docker-compose-op-goerli.yml up -d
+```
+
+### The initial synchornization
+
+During the initial synchonization, you get log messages from `op-node`, and nothing else appears to happen.
+
+```bash
+INFO [08-04|16:36:07.150] Advancing bq origin                      origin=df76ff..48987e:8301316 originBehind=false
+```
+
+After a few minutes, `op-node` finds the right batch and then it starts synchronizing. During this synchonization process, you get log messags from `op-node`.
+
+```bash
+INFO [08-04|16:36:01.204] Found next batch                         epoch=44e203..fef9a5:8301309 batch_epoch=8301309                batch_timestamp=1,673,567,518
+INFO [08-04|16:36:01.205] generated attributes in payload queue    txs=2  timestamp=1,673,567,518
+INFO [08-04|16:36:01.265] inserted block                           hash=ee61ee..256300 number=4,069,725 state_root=a582ae..33a7c5 timestamp=1,673,567,518 parent=5b102e..13196c prev_randao=4758ca..11ff3a fee_recipient=0x4200000000000000000000000000000000000011 txs=2  update_safe=true
+```

boba-community/docker-compose-op-goerli.yml

@@ -0,0 +1,32 @@
+version: '3.4'
+
+services:
+  l2:
+    image: bobanetwork/erigon:${VERSION:-latest}
+    environment:
+      - CHAIN_NAME=optimism-goerli
+    ports:
+      - "9545:8545"
+      - "8551:8551"
+    volumes:
+      - ./jwt-secret.txt:/config/jwt-secret.txt
+      - DATA_DIR:/db
+  op-node:
+    depends_on:
+      - l2
+    image: bobanetwork/op-node:${VERSION:-latest}
+    command: >
+      op-node
+      --l1=${ETH1_HTTP:-https://ethereum-goerli.publicnode.com}
+      --l2=http://l2:8551
+      --l2.jwt-secret=/config/jwt-secret.txt
+      --network=goerli
+      --rpc.addr=0.0.0.0
+      --rpc.port=8547
+      --l1.trustrpc=true
+      --p2p.disable=true
+    ports:
+      - "8547:8547"
+    volumes:
+      - ./jwt-secret.txt:/config/jwt-secret.txt
+    restart: always

boba-community/docker-compose-op-mainnet.yml

@@ -0,0 +1,32 @@
+version: '3.4'
+
+services:
+  l2:
+    image: bobanetwork/erigon:${VERSION:-latest}
+    environment:
+      - CHAIN_NAME=optimism-mainnet
+    ports:
+      - "9545:8545"
+      - "8551:8551"
+    volumes:
+      - ./jwt-secret.txt:/config/jwt-secret.txt
+      - DATA_DIR:/db
+  op-node:
+    depends_on:
+      - l2
+    image: bobanetwork/op-node:${VERSION:-latest}
+    command: >
+      op-node
+      --l1=${ETH1_HTTP:-https://ethereum.publicnode.com}
+      --l2=http://l2:8551
+      --l2.jwt-secret=/config/jwt-secret.txt
+      --network=mainnet
+      --rpc.addr=0.0.0.0
+      --rpc.port=8547
+      --l1.trustrpc=true
+      --p2p.disable=true
+    ports:
+      - "8547:8547"
+    volumes:
+      - ./jwt-secret.txt:/config/jwt-secret.txt
+    restart: always

go.mod

@@ -48,10 +48,16 @@ require (
 	gorm.io/gorm v1.25.3
 )
 
+require (
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+)
+
 require (
 	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
 	github.com/allegro/bigcache v1.2.1 // indirect
+	github.com/aws/aws-sdk-go v1.42.6
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.7.0 // indirect
@@ -187,6 +193,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.5.0 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
+	github.com/welthee/go-ethereum-aws-kms-tx-signer v0.0.0-20211013075913-ca566ae7abeb
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.uber.org/atomic v1.10.0 // indirect

op-chain-ops/state/state.go

@@ -49,7 +49,6 @@ func EncodeStorage(entry solc.StorageLayoutEntry, value any, storageType solc.St
 // SetStorage will set the storage values in a db given a contract name,
 // address and the storage values
 func SetStorage(name string, address common.Address, values StorageValues, db vm.StateDB) error {
-	log.Warn("MMDBG SetStorage", "name", name, "address", address)
 	if name == "BobaL2" {
 		name = "OptimismMintableERC20"
 	}

op-node/rollup/derive/attributes_queue.go

@@ -49,7 +49,6 @@ func (aq *AttributesQueue) Origin() eth.L1BlockRef {
 }
 
 func (aq *AttributesQueue) NextAttributes(ctx context.Context, l2SafeHead eth.L2BlockRef) (*eth.PayloadAttributes, error) {
-	log.Debug("MMDBG attributes_queue NextAttributes")
 	// Get a batch if we need it
 	if aq.batch == nil {
 		batch, err := aq.prev.NextBatch(ctx, l2SafeHead)

op-node/rollup/derive/batch_queue.go

@@ -60,7 +60,6 @@ func (bq *BatchQueue) Origin() eth.L1BlockRef {
 }
 
 func (bq *BatchQueue) NextBatch(ctx context.Context, safeL2Head eth.L2BlockRef) (*BatchData, error) {
-	log.Debug("MMDBG batch_queue NextBatch")
 	// Note: We use the origin that we will have to determine if it's behind. This is important
 	// because it's the future origin that gets saved into the l1Blocks array.
 	// We always update the origin of this stage if it is not the same so after the update code

op-node/rollup/derive/l1_traversal.go

@@ -48,7 +48,6 @@ func (l1t *L1Traversal) Origin() eth.L1BlockRef {
 // NextL1Block returns the next block. It does not advance, but it can only be
 // called once before returning io.EOF
 func (l1t *L1Traversal) NextL1Block(_ context.Context) (eth.L1BlockRef, error) {
-	log.Debug("MMDBG l1_traversal Step")
 	if !l1t.done {
 		l1t.done = true
 		return l1t.block, nil

op-node/rollup/derive/pipeline.go

@@ -193,9 +193,6 @@ func (dp *DerivationPipeline) Step(ctx context.Context) error {
 	defer dp.metrics.RecordL1Ref("l1_derived", dp.Origin())
 
 	// if any stages need to be reset, do that first.
-	//log.Debug("MMDBG pipeline.go Step", "dp.resetting", dp.resetting, "stages", dp.stages)
-	log.Debug("MMDBG pipeline.go Step", "dp", dp)
-
 	if dp.resetting < len(dp.stages) {
 		if err := dp.stages[dp.resetting].Reset(ctx, dp.eng.Origin(), dp.eng.SystemConfig()); err == io.EOF {
 			dp.log.Debug("reset of stage completed", "stage", dp.resetting, "origin", dp.eng.Origin())

op-node/rollup/driver/state.go

@@ -316,7 +316,6 @@ func (s *Driver) eventLoop() {
 			s.metrics.SetDerivationIdle(false)
 			s.log.Debug("Derivation process step", "onto_origin", s.derivation.Origin(), "attempts", stepAttempts)
 			err := s.derivation.Step(context.Background())
-			s.log.Debug("MMDBG derivation.Step", "err", err)
 			stepAttempts += 1 // count as attempt by default. We reset to 0 if we are making healthy progress.
 			if err == io.EOF {
 				s.log.Debug("Derivation process went idle", "progress", s.derivation.Origin(), "err", err)

op-node/sources/eth_client.go

@@ -374,7 +374,7 @@ func (s *EthClient) FetchReceipts(ctx context.Context, blockHash common.Hash) (e
 func (s *EthClient) GetProof(ctx context.Context, address common.Address, storage []common.Hash, blockTag string) (*eth.AccountResult, error) {
 	var getProofResponse *eth.AccountResult
 	err := s.client.CallContext(ctx, &getProofResponse, "eth_getProof", address, storage, blockTag)
-	s.log.Info("MMDBG eth_client GetProof", "err", err, "address", address, "blockTag", blockTag, "Response", getProofResponse)
+	s.log.Debug("eth_client GetProof", "err", err, "address", address, "blockTag", blockTag, "Response", getProofResponse)
 	if err != nil {
 		return nil, err
 	}