Merge branch 'master' into das-electra

CHANGELOG.md

@@ -9,7 +9,10 @@
 ### Breaking Changes
 
 ### Additions and Improvements
-- improve block publishing performance, especially relevant with locally produced blocks
+- Improved block publishing performance, especially relevant with locally produced blocks
+- Delayed blob publishing until the block is published to at least 1 peer, especially relevant with locally produced blocks with low upload bandwidth connections. Can be disabled via `--Xp2p-gossip-blobs-after-block-enabled=false`
+- Added single_attestation event type for electra attestations
 
 ### Bug Fixes
 - Added a startup script for unix systems to ensure that when jemalloc is installed the script sets the LD_PRELOAD environment variable to the use the jemalloc library
+- Set `is_syncing` to `false` instead of `true` for the `/eth/v1/node/syncing` API endpoint when the head is optimistic and the sync distance is 0

beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/forward/multipeer/chains/ThrottlingSyncSource.java

@@ -81,6 +81,10 @@ public SafeFuture<Void> requestBlocksByRange(
       LOG.debug("Sending request for {} blocks", count);
       return delegate.requestBlocksByRange(startSlot, count, listener);
     } else {
+      LOG.debug(
+          "Rate limiting request for {} blocks. Retry in {} seconds",
+          count,
+          PEER_REQUEST_DELAY.toSeconds());
       return asyncRunner.runAfterDelay(
           () -> requestBlocksByRange(startSlot, count, listener), PEER_REQUEST_DELAY);
     }
@@ -93,6 +97,10 @@ public SafeFuture<Void> requestBlobSidecarsByRange(
       LOG.debug("Sending request for {} blob sidecars", count);
       return delegate.requestBlobSidecarsByRange(startSlot, count, listener);
     } else {
+      LOG.debug(
+          "Rate limiting request for {} blob sidecars. Retry in {} seconds",
+          count,
+          PEER_REQUEST_DELAY.toSeconds());
       return asyncRunner.runAfterDelay(
           () -> requestBlobSidecarsByRange(startSlot, count, listener), PEER_REQUEST_DELAY);
     }

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandler.java

@@ -586,13 +586,20 @@ public SafeFuture<List<SubmitDataError>> sendSignedAttestations(
   }
 
   private SafeFuture<InternalValidationResult> processAttestation(final Attestation attestation) {
+    final ValidatableAttestation validatableAttestation =
+        ValidatableAttestation.fromValidator(spec, attestation);
     return attestationManager
-        .addAttestation(ValidatableAttestation.fromValidator(spec, attestation), Optional.empty())
+        .addAttestation(validatableAttestation, Optional.empty())
         .thenPeek(
             result -> {
               if (!result.isReject()) {
-                dutyMetrics.onAttestationPublished(attestation.getData().getSlot());
-                performanceTracker.saveProducedAttestation(attestation);
+                // When saving the attestation in performance tracker, we want to make sure we save
+                // the converted attestation.
+                // The conversion happens during processing and is saved in the validatable
+                // attestation.
+                final Attestation convertedAttestation = validatableAttestation.getAttestation();
+                dutyMetrics.onAttestationPublished(convertedAttestation.getData().getSlot());
+                performanceTracker.saveProducedAttestation(convertedAttestation);
               } else {
                 VALIDATOR_LOGGER.producedInvalidAttestation(
                     attestation.getData().getSlot(),

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/performance/DefaultPerformanceTracker.java

@@ -14,6 +14,7 @@
 package tech.pegasys.teku.validator.coordinator.performance;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 
 import com.google.common.annotations.VisibleForTesting;
 import it.unimi.dsi.fastutil.ints.Int2IntMap;
@@ -424,6 +425,7 @@ private SafeFuture<Map<UInt64, List<Attestation>>> getAttestationsIncludedInEpoc
 
   @Override
   public void saveProducedAttestation(final Attestation attestation) {
+    checkState(!attestation.isSingleAttestation(), "Single attestation is not supported");
     final UInt64 epoch = spec.computeEpochAtSlot(attestation.getData().getSlot());
     final Set<Attestation> attestationsInEpoch =
         producedAttestationsByEpoch.computeIfAbsent(epoch, __ -> concurrentSet());

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java

@@ -819,6 +819,53 @@ public void sendSignedAttestations_shouldAddAttestationToAttestationManager() {
         .addAttestation(ValidatableAttestation.from(spec, attestation), Optional.empty());
   }
 
+  @Test
+  void sendSignedAttestations_shouldSaveConvertedAttestationFromSingleAttestation() {
+    spec = TestSpecFactory.createMinimalElectra();
+    dataStructureUtil = new DataStructureUtil(spec);
+    validatorApiHandler =
+        new ValidatorApiHandler(
+            chainDataProvider,
+            nodeDataProvider,
+            networkDataProvider,
+            chainDataClient,
+            syncStateProvider,
+            blockFactory,
+            attestationPool,
+            attestationManager,
+            attestationTopicSubscriptions,
+            activeValidatorTracker,
+            dutyMetrics,
+            performanceTracker,
+            spec,
+            forkChoiceTrigger,
+            proposersDataManager,
+            syncCommitteeMessagePool,
+            syncCommitteeContributionPool,
+            syncCommitteeSubscriptionManager,
+            blockProductionPerformanceFactory,
+            blockPublisher);
+
+    final Attestation attestation = dataStructureUtil.randomSingleAttestation();
+    final Attestation convertedAttestation = dataStructureUtil.randomAttestation();
+    doAnswer(
+            invocation -> {
+              invocation
+                  .getArgument(0, ValidatableAttestation.class)
+                  .convertToAggregatedFormatFromSingleAttestation(convertedAttestation);
+              return completedFuture(InternalValidationResult.ACCEPT);
+            })
+        .when(attestationManager)
+        .addAttestation(any(ValidatableAttestation.class), any());
+
+    final SafeFuture<List<SubmitDataError>> result =
+        validatorApiHandler.sendSignedAttestations(List.of(attestation));
+    assertThat(result).isCompletedWithValue(emptyList());
+
+    verify(dutyMetrics).onAttestationPublished(convertedAttestation.getData().getSlot());
+    verify(performanceTracker).saveProducedAttestation(convertedAttestation);
+  }
+
   @Test
   void sendSignedAttestations_shouldAddToDutyMetricsAndPerformanceTrackerWhenNotInvalid() {
     final Attestation attestation = dataStructureUtil.randomAttestation();

build.gradle

@@ -325,7 +325,7 @@ allprojects {
 }
 
 def nightly = System.getenv("NIGHTLY") != null
-def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.8"
+def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.9"
 def blsRefTestVersion = 'v0.1.2'
 def slashingProtectionInterchangeRefTestVersion = 'v5.3.0'
 def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download'

data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2ElectraIntegrationTest.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.beaconrestapi.v2.beacon;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import okhttp3.Response;
+import org.junit.jupiter.api.BeforeEach;
+import tech.pegasys.teku.beaconrestapi.handlers.v2.beacon.PostAttestationsV2;
+import tech.pegasys.teku.infrastructure.json.JsonUtil;
+import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.datastructures.operations.Attestation;
+import tech.pegasys.teku.spec.datastructures.operations.SingleAttestation;
+import tech.pegasys.teku.spec.schemas.SchemaDefinitionsElectra;
+import tech.pegasys.teku.spec.util.DataStructureUtil;
+
+public class PostAttestationsV2ElectraIntegrationTest extends PostAttestationsV2IntegrationTest {
+  protected SerializableTypeDefinition<List<SingleAttestation>> attestationsListTypeDef;
+
+  @Override
+  @BeforeEach
+  void setup() {
+    spec = TestSpecFactory.createMinimalElectra();
+    specMilestone = SpecMilestone.ELECTRA;
+    startRestAPIAtGenesis(specMilestone);
+    dataStructureUtil = new DataStructureUtil(spec);
+    attestationsListTypeDef =
+        SerializableTypeDefinition.listOf(
+            SchemaDefinitionsElectra.required(spec.getGenesisSchemaDefinitions())
+                .getSingleAttestationSchema()
+                .getJsonTypeDefinition());
+  }
+
+  @Override
+  protected List<Attestation> getAttestationList(final int listSize) {
+    final List<Attestation> attestations = new ArrayList<>(listSize);
+    for (int i = 0; i < listSize; i++) {
+      attestations.add(dataStructureUtil.randomSingleAttestation());
+    }
+    return attestations;
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  protected Response postAttestations(final List<?> attestations, final String milestone)
+      throws IOException {
+    final SerializableTypeDefinition<List<SingleAttestation>> attestationsListTypeDef =
+        SerializableTypeDefinition.listOf(
+            SchemaDefinitionsElectra.required(spec.getGenesisSchemaDefinitions())
+                .getSingleAttestationSchema()
+                .getJsonTypeDefinition());
+    if (milestone == null) {
+      return post(
+          PostAttestationsV2.ROUTE,
+          JsonUtil.serialize((List<SingleAttestation>) attestations, attestationsListTypeDef));
+    }
+    return post(
+        PostAttestationsV2.ROUTE,
+        JsonUtil.serialize((List<SingleAttestation>) attestations, attestationsListTypeDef),
+        Collections.emptyMap(),
+        Optional.of(milestone));
+  }
+}