KAFKA-16448: Add ProcessingExceptionHandler in Streams configuration

Co-authored-by: Dabz <[email protected]>
Co-authored-by: sebastienviale <[email protected]>

.github/workflows/docker_build_and_test.yml

@@ -23,6 +23,7 @@ on:
         description: Docker image type to build and test
         options: 
           - "jvm"
+          - "native"
       kafka_url:
         description: Kafka url to be used to build the docker image
         required: true

.github/workflows/docker_official_image_build_and_test.yml

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Docker Official Image Build Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_type:
+        type: choice
+        description: Docker image type to build and test
+        options: 
+          - "jvm"
+      kafka_version:
+        description: Kafka version for the docker official image. This should be >=3.7.0
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r docker/requirements.txt
+    - name: Build image and run tests
+      working-directory: ./docker
+      run: |
+        python docker_official_image_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -v=${{ github.event.inputs.kafka_version }}
+    - name: Run CVE scan
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: 'kafka/test:test'
+        format: 'table'
+        severity: 'CRITICAL,HIGH'
+        output: scan_report_${{ github.event.inputs.image_type }}.txt
+        exit-code: '1'
+    - name: Upload test report
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: report_${{ github.event.inputs.image_type }}.html
+        path: docker/test/report_${{ github.event.inputs.image_type }}.html
+    - name: Upload CVE scan report
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: scan_report_${{ github.event.inputs.image_type }}.txt
+        path: scan_report_${{ github.event.inputs.image_type }}.txt

.github/workflows/docker_promote.yml

@@ -19,10 +19,10 @@ on:
   workflow_dispatch:
     inputs:
       rc_docker_image:
-        description: RC docker image that needs to be promoted (Example:- apache/kafka:3.6.0-rc0)
+        description: RC docker image that needs to be promoted (Example:- apache/kafka:3.8.0-rc0 (OR) apache/kafka-native:3.8.0-rc0)
         required: true
       promoted_docker_image:
-        description: Docker image name of the promoted image (Example:- apache/kafka:3.6.0)
+        description: Docker image name of the promoted image (Example:- apache/kafka:3.8.0 (OR) apache/kafka-native:3.8.0)
         required: true
 
 jobs:

.github/workflows/docker_rc_release.yml

@@ -23,8 +23,9 @@ on:
         description: Docker image type to be built and pushed
         options: 
           - "jvm"
+          - "native"
       rc_docker_image:
-        description: RC docker image that needs to be built and pushed to Dockerhub (Example:- apache/kafka:3.6.0-rc0)
+        description: RC docker image that needs to be built and pushed to Dockerhub (Example:- apache/kafka:3.8.0-rc0 (OR) apache/kafka-native:3.8.0-rc0)
         required: true
       kafka_url:
         description: Kafka url to be used to build the docker image

.github/workflows/prepare_docker_official_image_source.yml

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Docker Prepare Docker Official Image Source
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_type:
+        type: choice
+        description: Docker image type to build and test
+        options: 
+          - "jvm"
+      kafka_version:
+        description: Kafka version for the docker official image. This should be >=3.7.0
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r docker/requirements.txt
+    - name: Build Docker Official Image Artifact
+      working-directory: ./docker
+      run: |
+        python prepare_docker_official_image_source.py -type=${{ github.event.inputs.image_type }} -v=${{ github.event.inputs.kafka_version }}
+    - name: Upload Docker Official Image Artifact
+      if: success()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ github.event.inputs.kafka_version }}
+        path: docker/docker_official_images/${{ github.event.inputs.kafka_version }}

Jenkinsfile

@@ -36,6 +36,12 @@ def doTest(env, target = "test") {
   junit '**/build/test-results/**/TEST-*.xml'
 }
 
+def runTestOnDevBranch(env) {
+  if (!isChangeRequest(env)) {
+    doTest(env)
+  }
+}
+
 def doStreamsArchetype() {
   echo 'Verify that Kafka Streams archetype compiles'
 
@@ -132,7 +138,7 @@ pipeline {
           }
           steps {
             doValidation()
-            doTest(env)
+            runTestOnDevBranch(env)
             echo 'Skipping Kafka Streams archetype test for Java 11'
           }
         }
@@ -151,7 +157,7 @@ pipeline {
           }
           steps {
             doValidation()
-            doTest(env)
+            runTestOnDevBranch(env)
             echo 'Skipping Kafka Streams archetype test for Java 17'
           }
         }