diff --git a/README.md b/README.md index 23282bc87..8f56af549 100644 --- a/README.md +++ b/README.md @@ -587,6 +587,7 @@ A docker-compose is provide to start a development environnement. Just install docker & docker-compose, clone the repository and issue a simple `docker-compose -f docker-compose-dev.yml up` to start a dev server. Dev server is a java server & webpack-dev-server with live reload. +The configuration for the dev server is in `application.dev.yml`. ## Schema references diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 13163ae7d..5f7e9102a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -15,118 +15,108 @@ volumes: services: akhq: image: gradle:6.6.1-jdk11 - command: "gradle run --continuous" + command: 'gradle run --continuous' working_dir: /app volumes: - - ./:/app - - /app/.gradle - - ./gradle/gradle.properties:/root/.gradle/gradle.properties + - ./:/app:z + - /app/.gradle:Z + - ./gradle/gradle.properties:/root/.gradle/gradle.properties:Z ports: - - 127.11.8.17:8080:8080 + - 8080:8080 environment: - MICRONAUT_IO_WATCH_RESTART: "true" + MICRONAUT_IO_WATCH_RESTART: 'true' + MICRONAUT_CONFIG_FILES: '/app/application.dev.yml' depends_on: - kafka - schema-registry - webpack: - image: node:12 - command: "sh -c 'npm install && npm run dev'" - working_dir: /app - volumes: - - ./:/app - ports: - - 127.11.8.17:8081:8081 - depends_on: - - akhq - ui: image: node:12 - command: "sh -c 'npm install && npm start'" + command: 'sh -c "npm install && npm start"' working_dir: /app/client tty: true volumes: - - ./:/app - - ui-modules:/app/client/node_modules - - ui-build:/app/client/build + - ./:/app:z + - ui-modules:/app/client/node_modules:Z + - ui-build:/app/client/build:Z ports: - - 127.11.8.17:3000:3000 + - 3000:3000 depends_on: - akhq zookeeper: image: confluentinc/cp-zookeeper volumes: - - zookeeper-data:/var/lib/zookeeper/data - - zookeeper-log:/var/lib/zookeeper/log + - zookeeper-data:/var/lib/zookeeper/data:Z + - zookeeper-log:/var/lib/zookeeper/log:Z environment: - ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_CLIENT_PORT: '2181' + ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false' ports: - - 127.11.8.17:2181:2181 + - 2181:2181 kafka: image: confluentinc/cp-kafka volumes: - - kafka-data:/var/lib/kafka/data + - kafka-data:/var/lib/kafka:Z environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_NUM_PARTITIONS: 12 - KAFKA_COMPRESSION_TYPE: gzip - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_BROKER_ID: '0' + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_NUM_PARTITIONS: '12' + KAFKA_COMPRESSION_TYPE: 'gzip' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - KAFKA_JMX_PORT: 9091 + KAFKA_JMX_PORT: '9091' KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer' KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' links: - zookeeper ports: - - 127.11.8.17:9092:9092 - - 127.11.8.17:9091:9091 + - 9092:9092 + - 9091:9091 schema-registry: image: confluentinc/cp-schema-registry depends_on: - kafka environment: - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "PLAINTEXT://kafka:9092" - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8085" - SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka:9092' + SCHEMA_REGISTRY_HOST_NAME: 'schema-registry' + SCHEMA_REGISTRY_LISTENERS: 'http://0.0.0.0:8085' + SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: 'INFO' ports: - - 127.11.8.17:8085:8085 + - 8085:8085 connect: image: confluentinc/cp-kafka-connect ports: - - 127.11.8.17:8083:8083 + - 8083:8083 depends_on: - kafka - schema-registry environment: - CONNECT_BOOTSTRAP_SERVERS: kafka:9092 - CONNECT_REST_PORT: 8083 - CONNECT_REST_LISTENERS: http://0.0.0.0:8083 - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_CONFIG_STORAGE_TOPIC: __connect-config - CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets - CONNECT_STATUS_STORAGE_TOPIC: __connect-status - CONNECT_GROUP_ID: "kafka-connect" - CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true" - CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085 - CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true" - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085 - CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_BOOTSTRAP_SERVERS: 'kafka:9092' + CONNECT_REST_PORT: '8083' + CONNECT_REST_LISTENERS: 'http://0.0.0.0:8083' + CONNECT_REST_ADVERTISED_HOST_NAME: 'connect' + CONNECT_CONFIG_STORAGE_TOPIC: '__connect-config' + CONNECT_OFFSET_STORAGE_TOPIC: '__connect-offsets' + CONNECT_STATUS_STORAGE_TOPIC: '__connect-status' + CONNECT_GROUP_ID: 'kafka-connect' + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1' CONNECT_PLUGIN_PATH: ' /usr/share/java/' - volumes: - - ./connects-plugins:/usr/share/java/connects-plugins diff --git a/docker-compose-multiple-clusters.yml b/docker-compose-multiple-clusters.yml index 2dfdf2ee9..013b140bb 100644 --- a/docker-compose-multiple-clusters.yml +++ b/docker-compose-multiple-clusters.yml @@ -43,32 +43,34 @@ services: zookeeper-0: image: confluentinc/cp-zookeeper volumes: - - zookeeper-data-0:/var/lib/zookeeper/data - - zookeeper-log-0:/var/lib/zookeeper/log + - zookeeper-data-0:/var/lib/zookeeper/data:Z + - zookeeper-log-0:/var/lib/zookeeper/log:Z environment: - ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_CLIENT_PORT: '2181' + ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false' kafka-0: image: confluentinc/cp-kafka volumes: - - kafka-data-0:/var/lib/kafka + - kafka-data-0:/var/lib/kafka:Z ports: - 9092:9092 environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper-0:2181 - KAFKA_NUM_PARTITIONS: 12 - KAFKA_COMPRESSION_TYPE: gzip - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_BROKER_ID: '0' + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-0:2181' + KAFKA_NUM_PARTITIONS: '12' + KAFKA_COMPRESSION_TYPE: 'gzip' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1' KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - KAFKA_JMX_PORT: 9091 + KAFKA_JMX_PORT: '9091' KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer' KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' - KAFKA_LISTENERS: INTERNAL://kafka-0:29092,OUTSIDE://kafka-0:9092 - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-0:29092,OUTSIDE://localhost:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_LISTENERS: 'INTERNAL://kafka-0:29092,OUTSIDE://kafka-0:9092' + KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka-0:29092,OUTSIDE://localhost:9092' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT' + KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERNAL' links: - zookeeper-0 schema-registry-0: @@ -78,10 +80,10 @@ services: depends_on: - kafka-0 environment: - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "PLAINTEXT://kafka-0:29092" - SCHEMA_REGISTRY_HOST_NAME: schema-registry-0 - SCHEMA_REGISTRY_LISTENERS: "http://schema-registry-0:8085" - SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka-0:29092' + SCHEMA_REGISTRY_HOST_NAME: 'schema-registry-0' + SCHEMA_REGISTRY_LISTENERS: 'http://schema-registry-0:8085' + SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: 'INFO' connect-0: image: confluentinc/cp-kafka-connect ports: @@ -90,27 +92,27 @@ services: - kafka-0 - schema-registry-0 environment: - CONNECT_BOOTSTRAP_SERVERS: kafka-0:29092 - CONNECT_REST_PORT: 8083 - CONNECT_HOST_NAME: connect-0 - CONNECT_REST_LISTENERS: http://connect-0:8083 - CONNECT_LISTENERS: http://connect-0:8083 - CONNECT_REST_ADVERTISED_HOST_NAME: connect-0 - CONNECT_CONFIG_STORAGE_TOPIC: __connect-0-config - CONNECT_OFFSET_STORAGE_TOPIC: __connect-0-offsets - CONNECT_STATUS_STORAGE_TOPIC: __connect-0-status - CONNECT_GROUP_ID: "kafka-connect-0" - CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true" - CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry-0:8085 - CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true" - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry-0:8085 - CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_BOOTSTRAP_SERVERS: 'kafka-0:29092' + CONNECT_REST_PORT: '8083' + CONNECT_HOST_NAME: 'connect-0' + CONNECT_REST_LISTENERS: 'http://connect-0:8083' + CONNECT_LISTENERS: 'http://connect-0:8083' + CONNECT_REST_ADVERTISED_HOST_NAME: 'connect-0' + CONNECT_CONFIG_STORAGE_TOPIC: '__connect-0-config' + CONNECT_OFFSET_STORAGE_TOPIC: '__connect-0-offsets' + CONNECT_STATUS_STORAGE_TOPIC: '__connect-0-status' + CONNECT_GROUP_ID: 'kafka-connect-0' + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry-0:8085' + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry-0:8085' + CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1' CONNECT_PLUGIN_PATH: ' /usr/share/java/' connect-1: image: confluentinc/cp-kafka-connect @@ -120,57 +122,59 @@ services: - kafka-0 - schema-registry-0 environment: - CONNECT_BOOTSTRAP_SERVERS: kafka-0:29092 - CONNECT_REST_PORT: 8084 - CONNECT_HOST_NAME: connect-1 - CONNECT_REST_LISTENERS: http://connect-1:8084 - CONNECT_LISTENERS: http://connect-1:8084 - CONNECT_REST_ADVERTISED_HOST_NAME: connect-1 - CONNECT_CONFIG_STORAGE_TOPIC: __connect-1-config - CONNECT_OFFSET_STORAGE_TOPIC: __connect-1-offsets - CONNECT_STATUS_STORAGE_TOPIC: __connect-1-status - CONNECT_GROUP_ID: "kafka-connect-1" - CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true" - CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry-0:8085 - CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true" - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry-0:8085 - CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_BOOTSTRAP_SERVERS: 'kafka-0:29092' + CONNECT_REST_PORT: '8084' + CONNECT_HOST_NAME: 'connect-1' + CONNECT_REST_LISTENERS: 'http://connect-1:8084' + CONNECT_LISTENERS: 'http://connect-1:8084' + CONNECT_REST_ADVERTISED_HOST_NAME: 'connect-1' + CONNECT_CONFIG_STORAGE_TOPIC: '__connect-1-config' + CONNECT_OFFSET_STORAGE_TOPIC: '__connect-1-offsets' + CONNECT_STATUS_STORAGE_TOPIC: '__connect-1-status' + CONNECT_GROUP_ID: 'kafka-connect-1' + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry-0:8085' + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry-0:8085' + CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1' CONNECT_PLUGIN_PATH: ' /usr/share/java/' # ANOTHER CLUSTER zookeeper-1: image: confluentinc/cp-zookeeper volumes: - - zookeeper-data-1:/var/lib/zookeeper/data - - zookeeper-log-1:/var/lib/zookeeper/log + - zookeeper-data-1:/var/lib/zookeeper/data:Z + - zookeeper-log-1:/var/lib/zookeeper/log:Z environment: - ZOOKEEPER_CLIENT_PORT: 2171 + ZOOKEEPER_CLIENT_PORT: '2171' + ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false' kafka-1: image: confluentinc/cp-kafka volumes: - - kafka-data-1:/var/lib/kafka + - kafka-data-1:/var/lib/kafka:Z ports: - 9072:9072 environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2171 - KAFKA_NUM_PARTITIONS: 12 - KAFKA_COMPRESSION_TYPE: gzip - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_BROKER_ID: '0' + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2171' + KAFKA_NUM_PARTITIONS: '12' + KAFKA_COMPRESSION_TYPE: 'gzip' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1' KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - KAFKA_JMX_PORT: 9071 + KAFKA_JMX_PORT: '9071' KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer' KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' - KAFKA_LISTENERS: INTERNAL://kafka-1:29072,OUTSIDE://kafka-1:9072 - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:29072,OUTSIDE://localhost:9072 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_LISTENERS: 'INTERNAL://kafka-1:29072,OUTSIDE://kafka-1:9072' + KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka-1:29072,OUTSIDE://localhost:9072' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT' + KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERNAL' links: - zookeeper-1 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0b6bcd371..daf285469 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,9 @@ services: schema-registry: url: "http://schema-registry:8085" connect: - url: "http://connect:8083" + - name: "connect" + url: "http://connect:8083" + ports: - 8080:8080 links: @@ -33,26 +35,30 @@ services: zookeeper: image: confluentinc/cp-zookeeper volumes: - - zookeeper-data:/var/lib/zookeeper/data - - zookeeper-log:/var/lib/zookeeper/log + - zookeeper-data:/var/lib/zookeeper/data:Z + - zookeeper-log:/var/lib/zookeeper/log:Z environment: - ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_CLIENT_PORT: '2181' + ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false' kafka: image: confluentinc/cp-kafka volumes: - - kafka-data:/var/lib/kafka + - kafka-data:/var/lib/kafka:Z environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_NUM_PARTITIONS: 12 - KAFKA_COMPRESSION_TYPE: gzip - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_BROKER_ID: '0' + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_NUM_PARTITIONS: '12' + KAFKA_COMPRESSION_TYPE: 'gzip' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - KAFKA_JMX_PORT: 9091 + KAFKA_JMX_PORT: '9091' KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer' + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' links: - zookeeper @@ -61,10 +67,10 @@ services: depends_on: - kafka environment: - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "PLAINTEXT://kafka:9092" - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8085" - SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka:9092' + SCHEMA_REGISTRY_HOST_NAME: 'schema-registry' + SCHEMA_REGISTRY_LISTENERS: 'http://0.0.0.0:8085' + SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: 'INFO' connect: image: confluentinc/cp-kafka-connect @@ -72,25 +78,25 @@ services: - kafka - schema-registry environment: - CONNECT_BOOTSTRAP_SERVERS: kafka:9092 - CONNECT_REST_PORT: 8083 - CONNECT_REST_LISTENERS: http://0.0.0.0:8083 - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_CONFIG_STORAGE_TOPIC: __connect-config - CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets - CONNECT_STATUS_STORAGE_TOPIC: __connect-status - CONNECT_GROUP_ID: "kafka-connect" - CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true" - CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085 - CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true" - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085 - CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_BOOTSTRAP_SERVERS: 'kafka:9092' + CONNECT_REST_PORT: '8083' + CONNECT_REST_LISTENERS: 'http://0.0.0.0:8083' + CONNECT_REST_ADVERTISED_HOST_NAME: 'connect' + CONNECT_CONFIG_STORAGE_TOPIC: '__connect-config' + CONNECT_OFFSET_STORAGE_TOPIC: '__connect-offsets' + CONNECT_STATUS_STORAGE_TOPIC: '__connect-status' + CONNECT_GROUP_ID: 'kafka-connect' + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1' CONNECT_PLUGIN_PATH: ' /usr/share/java/' test-data: @@ -98,16 +104,18 @@ services: command: "gradle --no-daemon testInjectData" working_dir: /app volumes: - - ./:/app + - ./:/app:z links: - kafka - schema-registry kafkacat: image: confluentinc/cp-kafkacat - command: - - bash - - -c + depends_on: + - kafka + command: + - bash + - -c - | kafkacat -P -b kafka:9092 -t json << EOF {"_id":"5c4b2b45ab234c86955f0802","index":0,"guid":"d3637b06-9940-4958-9f82-639001c14c34"} @@ -117,7 +125,7 @@ services: {"_id":"5c4b2b45d1103ce30dfe1947","index":4,"guid":"14d53f2c-def3-406f-9dfb-c29963fdc37e"} {"_id":"5c4b2b45d6d3b5c51d3dacb7","index":5,"guid":"a20cfc3a-934a-4b93-9a03-008ec651b5a4"} EOF - + kafkacat -P -b kafka:9092 -t csv << EOF 1,Sauncho,Attfield,sattfield0@netlog.com,Male,221.119.13.246 2,Luci,Harp,lharp1@wufoo.com,Female,161.14.184.150