[DPE-3721] - chore: add cruise-control to snap

.github/workflows/ci.yaml

@@ -60,11 +60,14 @@ jobs:
 
       - name: Set default config
         run: |
-          sudo mkdir -p /var/snap/charmed-kafka/current/etc/kafka
-          sudo mkdir -p /var/snap/charmed-zookeeper/current/etc/zookeeper
           sudo cp /snap/charmed-kafka/current/opt/kafka/config/server.properties /var/snap/charmed-kafka/current/etc/kafka
           sudo cp /snap/charmed-zookeeper/current/opt/zookeeper/conf/zoo_sample.cfg /var/snap/charmed-zookeeper/current/etc/zookeeper/zoo.cfg
 
+          sudo cp /snap/charmed-kafka/current/opt/cruise-control/config/cruisecontrol.properties /var/snap/charmed-kafka/current/etc/cruise-control
+          sudo sed -i -e 's/sample.store.topic.replication.factor=2/sample.store.topic.replication.factor=1/g' /var/snap/charmed-kafka/current/etc/cruise-control/cruisecontrol.properties
+          sudo sed -i -e 's|capacity.config.file=config/capacityJBOD.json|capacity.config.file=/var/snap/charmed-kafka/current/etc/cruise-control/capacityJBOD.json|g' /var/snap/charmed-kafka/current/etc/cruise-control/cruisecontrol.properties
+          sudo cp /snap/charmed-kafka/current/opt/cruise-control/config/capacityJBOD.json /var/snap/charmed-kafka/current/etc/cruise-control
+
       - name: Start snap services
         run: |
           sudo snap start charmed-zookeeper.daemon
@@ -78,3 +81,9 @@ jobs:
           if [ "$topic_creation" != "Created topic test." ]; then
               exit 1
           fi
+
+      - name: Check CruiseControl state
+        run: |
+          sudo snap start charmed-kafka.cruise-control
+          sleep 20
+          curl -s http://localhost:9090/kafkacruisecontrol/state | grep "state: RUNNING"

README.md

@@ -51,3 +51,26 @@ Logs should be available at `/var/snap/charmed-kafka/common/var/log/kafka`.
 Place your custom kafka configuration in the snap data directory at `/var/snap/charmed-kafka/current/etc/kafka/server.properties`.
 
 If you want to use custom log4j properties, place your custom log4j properties file at the snap data directory at `/var/snap/charmed-kafka/etc/kafka/log4j.properties`.
+
+### Cruise Control
+
+To get started with Cruise Control, on a local system already running the Kafka and ZooKeeper services, you can use the following:
+
+```bash
+# copying necessary configuration files
+sudo cp /snap/charmed-kafka/current/opt/cruise-control/config/cruisecontrol.properties /var/snap/charmed-kafka/current/etc/cruise-control
+sudo cp /snap/charmed-kafka/current/opt/cruise-control/config/capacityJBOD.json /var/snap/charmed-kafka/current/etc/cruise-control
+
+# overriding defaults
+sudo sed -i -e 's/sample.store.topic.replication.factor=2/sample.store.topic.replication.factor=1/g' /var/snap/charmed-kafka/current/etc/cruise-control/cruisecontrol.properties
+sudo sed -i -e 's|capacity.config.file=config/capacityJBOD.json|capacity.config.file=/var/snap/charmed-kafka/current/etc/cruise-control/capacityJBOD.json|g' /var/snap/charmed-kafka/current/etc/cruise-control/cruisecontrol.properties
+
+# starting services
+sudo snap start charmed-kafka.cruise-control
+```
+
+Then, after a little time, you can interact with the API webserver using the following:
+
+```bash
+curl http://localhost:9090/kafkacruisecontrol/state
+```

licenses/LICENSE-cruisecontrol

@@ -0,0 +1,26 @@
+BSD 2-CLAUSE LICENSE
+
+Copyright 2017, 2018, 2019, 2020, 2021, 2022 LinkedIn Corporation.
+All Rights Reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

snap/hooks/install

@@ -10,30 +10,31 @@
 
 set -eux
 
-export COMPONENT=kafka
+for COMPONENT in kafka cruise-control
+do
+    export CONF="${SNAP_DATA}"/etc/$COMPONENT
+    export LOGS="${SNAP_COMMON}"/var/log/$COMPONENT
+    export DATA="${SNAP_COMMON}"/var/lib/$COMPONENT
 
-export CONF="${SNAP_DATA}"/etc/$COMPONENT
-export LOGS="${SNAP_COMMON}"/var/log/$COMPONENT
-export DATA="${SNAP_COMMON}"/var/lib/$COMPONENT
-
-if [stat -c '%u' "$DATA" == 0 -o ! -d "$DATA"]; then
-    mkdir -p $DATA
-    mkdir -p $LOGS
-
-    chmod -R 770 "${SNAP_COMMON}"
-    chown -R 584788:root "${SNAP_COMMON}"
-    chmod 750 "${DATA}"
-    chown 584788:root "${DATA}"
-else
+    mkdir -p $CONF
     mkdir -p $LOGS
-    chmod -R 770 "${LOGS}"
-    chown -R 584788:root "${LOGS}"
-fi
-
-mkdir -p $CONF
-
-cp "${SNAP}"/etc/$COMPONENT/* $CONF
+    mkdir -p $DATA
 
+    # setting dir permissions given re-attached storage
+    if [stat -c '%u' "$DATA" == 0 -o ! -d "$DATA"]; then
+        chmod -R 770 "${SNAP_COMMON}"
+        chown -R 584788:root "${SNAP_COMMON}"
+        chmod 750 "${DATA}"
+        chown 584788:root "${DATA}"
+    else
+        chmod -R 770 "${LOGS}"
+        chown -R 584788:root "${LOGS}"
+    fi
+
+    # copying default config from squashfs
+    cp "${SNAP}"/etc/$COMPONENT/* $CONF
+done
+
+# setting blanket permissions for the installing snap data dir
 chmod -R 770 "${SNAP_DATA}"/*
-
 chown -R 584788:root "${SNAP_DATA}"/*

snap/local/etc/cruise-control/log4j.properties

@@ -0,0 +1,66 @@
+#
+# Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information.
+#
+
+cruisecontrol.log.level=INFO
+
+rootLogger.level=${sys:cruisecontrol.log.level}
+appenders=kafkaCruiseControlAppender, operationAppender, requestAppender
+
+property.filename=${sys:cruisecontrol.logs.dir}
+
+appender.kafkaCruiseControlAppender.type=RollingFile
+appender.kafkaCruiseControlAppender.name=kafkaCruiseControlFile
+appender.kafkaCruiseControlAppender.fileName=${filename}/kafkacruisecontrol.log
+appender.kafkaCruiseControlAppender.filePattern=${filename}/kafkacruisecontrol.log.%d{yyyy-MM-dd-HH}
+appender.kafkaCruiseControlAppender.layout.type=PatternLayout
+appender.kafkaCruiseControlAppender.layout.pattern=[%d] %p %m (%c)%n
+appender.kafkaCruiseControlAppender.policies.type=Policies
+appender.kafkaCruiseControlAppender.policies.size.type = SizeBasedTriggeringPolicy
+appender.kafkaCruiseControlAppender.policies.size.size=100MB
+appender.kafkaCruiseControlAppender.strategy.type = DefaultRolloverStrategy
+appender.kafkaCruiseControlAppender.strategy.max = 10
+
+appender.operationAppender.type=RollingFile
+appender.operationAppender.name=operationFile
+appender.operationAppender.fileName=${filename}/kafkacruisecontrol-operation.log
+appender.operationAppender.filePattern=${filename}/kafkacruisecontrol-operation.log.%d{yyyy-MM-dd}
+appender.operationAppender.layout.type=PatternLayout
+appender.operationAppender.layout.pattern=[%d] %p [%c] %m %n
+appender.operationAppender.policies.type=Policies
+appender.operationAppender.policies.size.type = SizeBasedTriggeringPolicy
+appender.operationAppender.policies.size.size=100MB
+appender.operationAppender.strategy.type = DefaultRolloverStrategy
+appender.operationAppender.strategy.max = 10
+
+appender.requestAppender.type=RollingFile
+appender.requestAppender.name=requestFile
+appender.requestAppender.fileName=${filename}/kafkacruisecontrol-request.log
+appender.requestAppender.filePattern=${filename}/kafkacruisecontrol-request.log.%d{yyyy-MM-dd-HH}
+appender.requestAppender.layout.type=PatternLayout
+appender.requestAppender.layout.pattern=[%d] %p %m (%c)%n
+appender.requestAppender.policies.type=Policies
+appender.requestAppender.policies.size.type = SizeBasedTriggeringPolicy
+appender.requestAppender.policies.size.size=100MB
+appender.requestAppender.strategy.type = DefaultRolloverStrategy
+appender.requestAppender.strategy.max = 10
+
+# Loggers
+logger.cruisecontrol.name=com.linkedin.kafka.cruisecontrol
+logger.cruisecontrol.level=${sys:cruisecontrol.log.level}
+logger.cruisecontrol.appenderRef.kafkaCruiseControlAppender.ref=kafkaCruiseControlFile
+
+logger.detector.name=com.linkedin.kafka.cruisecontrol.detector
+logger.detector.level=${sys:cruisecontrol.log.level}
+logger.detector.appenderRef.kafkaCruiseControlAppender.ref=kafkaCruiseControlFile
+
+logger.operationLogger.name=operationLogger
+logger.operationLogger.level=${sys:cruisecontrol.log.level}
+logger.operationLogger.appenderRef.operationAppender.ref=operationFile
+
+logger.CruiseControlPublicAccessLogger.name=CruiseControlPublicAccessLogger
+logger.CruiseControlPublicAccessLogger.level=${sys:cruisecontrol.log.level}
+logger.CruiseControlPublicAccessLogger.appenderRef.requestAppender.ref=requestFile
+
+rootLogger.appenderRefs=kafkaCruiseControlAppender
+rootLogger.appenderRef.kafkaCruiseControlAppender.ref=kafkaCruiseControlFile

snap/local/opt/cruise-control/bin/kafka-cruise-control-start.sh

@@ -0,0 +1,174 @@
+#!/bin/bash
+# Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License").
+# See License in the project root for license information.
+
+if [ $# -lt 1 ];
+then
+  echo "USAGE: $0 [-daemon] [-name servicename] [-loggc] [-jars jar_path] config_path [port]"
+  exit 1
+fi
+
+# CYGINW == 1 if Cygwin is detected, else 0.
+if [[ $(uname -a) =~ "CYGWIN" ]]; then
+  CYGWIN=1
+else
+  CYGWIN=0
+fi
+
+copyJars() {
+  if [ -z $1 ]; then
+    return 0
+  fi
+  files=$1
+  jars=(${files//,/ })
+  for usrJar in ${jars[@]};
+  do
+    cp $usrJar "$base_dir"/cruise-control/build/dependant-libs/
+  done
+}
+
+base_dir=$(dirname $0)/..
+
+if [ -z "$SCALA_VERSION" ]; then
+  SCALA_VERSION=2.13.6
+fi
+
+if [ -z "$SCALA_BINARY_VERSION" ]; then
+  SCALA_BINARY_VERSION=$(echo $SCALA_VERSION | cut -f 1-2 -d '.')
+fi
+
+# JMX settings
+if [ -z "$KAFKA_JMX_OPTS" ]; then
+  KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+# JMX port to use
+if [  $JMX_PORT ]; then
+  KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
+fi
+
+# Log directory to use
+if [ "x$LOG_DIR" = "x" ]; then
+  LOG_DIR="$base_dir/logs"
+fi
+
+# Log4j settings
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+  # Log to console. This is a tool.
+  LOG4J_DIR="$base_dir/config/log4j.properties"
+  # If Cygwin is detected, LOG4J_DIR is converted to Windows format.
+  (( CYGWIN )) && LOG4J_DIR=$(cygpath --path --mixed "${LOG4J_DIR}")
+  KAFKA_LOG4J_OPTS="-Dlog4j.configurationFile=file:${LOG4J_DIR}"
+else
+  # create logs directory
+  if [ ! -d "$LOG_DIR" ]; then
+    mkdir -p "$LOG_DIR"
+  fi
+fi
+
+# If Cygwin is detected, LOG_DIR is converted to Windows format.
+(( CYGWIN )) && LOG_DIR=$(cygpath --path --mixed "${LOG_DIR}")
+KAFKA_LOG4J_OPTS="-Dkafka.logs.dir=$LOG_DIR $KAFKA_LOG4J_OPTS"
+
+# Generic jvm settings you want to add
+if [ -z "$KAFKA_OPTS" ]; then
+  KAFKA_OPTS=""
+fi
+
+# Set Debug options if enabled
+if [ "x$KAFKA_DEBUG" != "x" ]; then
+
+    # Use default ports
+    DEFAULT_JAVA_DEBUG_PORT="5005"
+
+    if [ -z "$JAVA_DEBUG_PORT" ]; then
+        JAVA_DEBUG_PORT="$DEFAULT_JAVA_DEBUG_PORT"
+    fi
+
+    # Use the defaults if JAVA_DEBUG_OPTS was not set
+    DEFAULT_JAVA_DEBUG_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=${DEBUG_SUSPEND_FLAG:-n},address=$JAVA_DEBUG_PORT"
+    if [ -z "$JAVA_DEBUG_OPTS" ]; then
+        JAVA_DEBUG_OPTS="$DEFAULT_JAVA_DEBUG_OPTS"
+    fi
+
+    echo "Enabling Java debug options: $JAVA_DEBUG_OPTS"
+    KAFKA_OPTS="$JAVA_DEBUG_OPTS $KAFKA_OPTS"
+fi
+
+# Which java to use
+if [ -z "$JAVA_HOME" ]; then
+  JAVA="java"
+else
+  JAVA="$JAVA_HOME/bin/java"
+fi
+
+# Memory options
+if [ -z "$KAFKA_HEAP_OPTS" ]; then
+  KAFKA_HEAP_OPTS="-Xmx1G"
+fi
+
+# JVM performance options
+if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then
+  KAFKA_JVM_PERFORMANCE_OPTS="-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true"
+fi
+
+#Add jaas file to KAFKA_OPTS if present
+if [ -f $base_dir/config/cruise_control_jaas.conf ]
+then
+  KAFKA_OPTS="-Djava.security.auth.login.config=$base_dir/config/cruise_control_jaas.conf $KAFKA_OPTS"
+fi
+
+# classpath addition for release
+for file in "$base_dir"/libs/*;
+do
+    CLASSPATH="$CLASSPATH":"$file"
+done
+
+DAEMON_NAME="kafka-cruise-control"
+CONSOLE_OUTPUT_FILE="${LOG_DIR}"/"${DAEMON_NAME}".out
+while [ $# -gt 0 ]; do
+  COMMAND=$1
+  case $COMMAND in
+    -name)
+      DAEMON_NAME=$2
+      CONSOLE_OUTPUT_FILE="${LOG_DIR}"/"${DAEMON_NAME}".out
+      shift 2
+      ;;
+    -loggc)
+      if [ -z "$KAFKA_GC_LOG_OPTS" ]; then
+        GC_LOG_ENABLED="true"
+      fi
+      shift
+      ;;
+    -daemon)
+      DAEMON_MODE="true"
+      shift
+      ;;
+    -jars)
+      JAR_PATHS=$2
+      copyJars "$JAR_PATHS"
+      shift 2
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+# GC options
+GC_FILE_SUFFIX='-gc.log'
+GC_LOG_FILE_NAME=''
+if [ "x$GC_LOG_ENABLED" = "xtrue" ]; then
+  GC_LOG_FILE_NAME=$DAEMON_NAME$GC_FILE_SUFFIX
+  KAFKA_GC_LOG_OPTS="-Xloggc:$LOG_DIR/$GC_LOG_FILE_NAME -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
+fi
+
+# If Cygwin is detected, classpath is converted to Windows format.
+(( CYGWIN )) && CLASSPATH=$(cygpath --path --mixed "${CLASSPATH}")
+
+# Launch mode
+if [ "x$DAEMON_MODE" = "xtrue" ]; then
+  nohup $JAVA $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_OPTS -cp $CLASSPATH $KAFKA_OPTS com.linkedin.kafka.cruisecontrol.KafkaCruiseControlMain "$@" > "$CONSOLE_OUTPUT_FILE" 2>&1 < /dev/null &
+else
+  exec $JAVA $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_OPTS -cp $CLASSPATH $KAFKA_OPTS com.linkedin.kafka.cruisecontrol.KafkaCruiseControlMain "$@"
+fi

snap/local/opt/cruise-control/bin/start-wrapper.bash

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+unset KAFKA_JMX_OPTS
+
+if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
+    export KAFKA_LOG4J_OPTS="-Dcruisecontrol.logs.dir=${LOG_DIR} -Dlog4j.configurationFile=${SNAP_DATA}/etc/cruise-control/log4j.properties -Dcruisecontrol.log.level=${KAFKA_CFG_LOGLEVEL:-INFO}"
+fi
+
+"${SNAP}"/usr/bin/setpriv \
+    --clear-groups \
+    --reuid snap_daemon \
+    --regid snap_daemon -- \
+    "${SNAP}/opt/cruise-control/bin/kafka-cruise-control-start.sh" "${SNAP_DATA}"/etc/cruise-control/cruisecontrol.properties
+