mirror of
https://github.com/rodneyosodo/uber4freefood.git
synced 2026-06-23 04:10:18 +00:00
Merge pull request #2 from 0x6f736f646f/kafka-streaming
Kafka streaming
This commit is contained in:
b1ackd0t
GitHub
@@ -0,0 +1,110 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
|
||||
# Weights
|
||||
yolov3.weights
|
||||
|
||||
#vscode
|
||||
.vscode/
|
||||
@@ -0,0 +1,15 @@
|
||||
from confluent_kafka import Consumer, KafkaError
|
||||
|
||||
c = Consumer({'bootstrap.servers': 'localhost:9092', 'group.id': 'mygroup',
|
||||
'default.topic.config': {'auto.offset.reset': 'smallest'}})
|
||||
c.subscribe(['Qualis'])
|
||||
running = True
|
||||
while running:
|
||||
msg = c.poll()
|
||||
if not msg.error():
|
||||
m = msg.value()
|
||||
print('Received message: %s' % m)
|
||||
elif msg.error().code() != KafkaError._PARTITION_EOF:
|
||||
print(msg.error())
|
||||
running = False
|
||||
c.close()
|
||||
@@ -0,0 +1,28 @@
|
||||
from confluent_kafka import Producer, KafkaError, KafkaException
|
||||
import time
|
||||
import sys
|
||||
|
||||
|
||||
def callback(err, msg):
|
||||
if err:
|
||||
sys.stderr.write("Message failed: ", err)
|
||||
else:
|
||||
sys.stderr.write("Message delivered to ", msg.topic())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
counter = 0
|
||||
while True:
|
||||
producer = Producer({'bootstrap.servers': 'localhost:9092'})
|
||||
topic = "Qualis"
|
||||
try:
|
||||
message = "{} Test message {}".format(counter, time.asctime()[11:][:8])
|
||||
producer.produce(topic, key="Osodo Rodney", value=message)
|
||||
counter = counter + 1
|
||||
#producer.produce(topic, b'Test message')#, callback=callback)
|
||||
except BufferError:
|
||||
pass
|
||||
#sys.stderr.write("Local producer queue is full")
|
||||
producer.flush()
|
||||
# time.sleep(1)
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
---
|
||||
version: '2'
|
||||
services:
|
||||
zookeeper:
|
||||
image: confluentinc/cp-zookeeper:5.4.0
|
||||
hostname: zookeeper
|
||||
container_name: zookeeper
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
ZOOKEEPER_CLIENT_PORT: 2181
|
||||
ZOOKEEPER_TICK_TIME: 2000
|
||||
|
||||
broker:
|
||||
image: confluentinc/cp-server:5.4.0
|
||||
hostname: broker
|
||||
container_name: broker
|
||||
depends_on:
|
||||
- zookeeper
|
||||
ports:
|
||||
- "9092:9092"
|
||||
environment:
|
||||
KAFKA_BROKER_ID: 1
|
||||
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
|
||||
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
|
||||
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
|
||||
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
|
||||
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
|
||||
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
|
||||
CONFLUENT_METRICS_ENABLE: 'true'
|
||||
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
|
||||
|
||||
schema-registry:
|
||||
image: confluentinc/cp-schema-registry:5.4.0
|
||||
hostname: schema-registry
|
||||
container_name: schema-registry
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- broker
|
||||
ports:
|
||||
- "8081:8081"
|
||||
environment:
|
||||
SCHEMA_REGISTRY_HOST_NAME: schema-registry
|
||||
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
|
||||
|
||||
connect:
|
||||
image: cnfldemos/cp-server-connect-datagen:0.2.0-5.4.0
|
||||
hostname: connect
|
||||
container_name: connect
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- broker
|
||||
- schema-registry
|
||||
ports:
|
||||
- "8083:8083"
|
||||
environment:
|
||||
CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
|
||||
CONNECT_REST_ADVERTISED_HOST_NAME: connect
|
||||
CONNECT_REST_PORT: 8083
|
||||
CONNECT_GROUP_ID: compose-connect-group
|
||||
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
|
||||
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
|
||||
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
|
||||
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
|
||||
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
|
||||
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
|
||||
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
|
||||
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
|
||||
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
|
||||
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
|
||||
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
|
||||
# CLASSPATH required due to CC-2422
|
||||
CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-5.4.0.jar
|
||||
CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
|
||||
CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
|
||||
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
|
||||
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
|
||||
|
||||
control-center:
|
||||
image: confluentinc/cp-enterprise-control-center:5.4.0
|
||||
hostname: control-center
|
||||
container_name: control-center
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- broker
|
||||
- schema-registry
|
||||
- connect
|
||||
- ksql-server
|
||||
ports:
|
||||
- "9021:9021"
|
||||
environment:
|
||||
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
|
||||
CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181'
|
||||
CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083'
|
||||
CONTROL_CENTER_KSQL_URL: "http://ksql-server:8088"
|
||||
CONTROL_CENTER_KSQL_ADVERTISED_URL: "http://localhost:8088"
|
||||
CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
|
||||
CONTROL_CENTER_REPLICATION_FACTOR: 1
|
||||
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
|
||||
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
|
||||
CONFLUENT_METRICS_TOPIC_REPLICATION: 1
|
||||
PORT: 9021
|
||||
|
||||
ksql-server:
|
||||
image: confluentinc/cp-ksql-server:5.4.0
|
||||
hostname: ksql-server
|
||||
container_name: ksql-server
|
||||
depends_on:
|
||||
- broker
|
||||
- connect
|
||||
ports:
|
||||
- "8088:8088"
|
||||
environment:
|
||||
KSQL_CONFIG_DIR: "/etc/ksql"
|
||||
KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties"
|
||||
KSQL_BOOTSTRAP_SERVERS: "broker:29092"
|
||||
KSQL_HOST_NAME: ksql-server
|
||||
KSQL_LISTENERS: "http://0.0.0.0:8088"
|
||||
KSQL_CACHE_MAX_BYTES_BUFFERING: 0
|
||||
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
|
||||
KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
|
||||
KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
|
||||
|
||||
ksql-cli:
|
||||
image: confluentinc/cp-ksql-cli:5.4.0
|
||||
container_name: ksql-cli
|
||||
depends_on:
|
||||
- broker
|
||||
- connect
|
||||
- ksql-server
|
||||
entrypoint: /bin/sh
|
||||
tty: true
|
||||
|
||||
ksql-datagen:
|
||||
# Downrev ksql-examples to 5.1.2 due to DEVX-798 (work around issues in 5.2.0)
|
||||
image: confluentinc/ksql-examples:5.4.0
|
||||
hostname: ksql-datagen
|
||||
container_name: ksql-datagen
|
||||
depends_on:
|
||||
- ksql-server
|
||||
- broker
|
||||
- schema-registry
|
||||
- connect
|
||||
command: "bash -c 'echo Waiting for Kafka to be ready... && \
|
||||
cub kafka-ready -b broker:29092 1 40 && \
|
||||
echo Waiting for Confluent Schema Registry to be ready... && \
|
||||
cub sr-ready schema-registry 8081 40 && \
|
||||
echo Waiting a few seconds for topic creation to finish... && \
|
||||
sleep 11 && \
|
||||
tail -f /dev/null'"
|
||||
environment:
|
||||
KSQL_CONFIG_DIR: "/etc/ksql"
|
||||
KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties"
|
||||
STREAMS_BOOTSTRAP_SERVERS: broker:29092
|
||||
STREAMS_SCHEMA_REGISTRY_HOST: schema-registry
|
||||
STREAMS_SCHEMA_REGISTRY_PORT: 8081
|
||||
|
||||
rest-proxy:
|
||||
image: confluentinc/cp-kafka-rest:5.4.0
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- broker
|
||||
- schema-registry
|
||||
ports:
|
||||
- 8082:8082
|
||||
hostname: rest-proxy
|
||||
container_name: rest-proxy
|
||||
environment:
|
||||
KAFKA_REST_HOST_NAME: rest-proxy
|
||||
KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
|
||||
KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
|
||||
KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
|
||||
+566
@@ -0,0 +1,566 @@
|
||||
apiVersion: v1
|
||||
items:
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: broker
|
||||
name: broker
|
||||
spec:
|
||||
ports:
|
||||
- name: "9092"
|
||||
port: 9092
|
||||
targetPort: 9092
|
||||
selector:
|
||||
io.kompose.service: broker
|
||||
status:
|
||||
loadBalancer: {}
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: connect
|
||||
name: connect
|
||||
spec:
|
||||
ports:
|
||||
- name: "8083"
|
||||
port: 8083
|
||||
targetPort: 8083
|
||||
selector:
|
||||
io.kompose.service: connect
|
||||
status:
|
||||
loadBalancer: {}
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: control-center
|
||||
name: control-center
|
||||
spec:
|
||||
ports:
|
||||
- name: "9021"
|
||||
port: 9021
|
||||
targetPort: 9021
|
||||
selector:
|
||||
io.kompose.service: control-center
|
||||
status:
|
||||
loadBalancer: {}
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: ksql-server
|
||||
name: ksql-server
|
||||
spec:
|
||||
ports:
|
||||
- name: "8088"
|
||||
port: 8088
|
||||
targetPort: 8088
|
||||
selector:
|
||||
io.kompose.service: ksql-server
|
||||
status:
|
||||
loadBalancer: {}
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: rest-proxy
|
||||
name: rest-proxy
|
||||
spec:
|
||||
ports:
|
||||
- name: "8082"
|
||||
port: 8082
|
||||
targetPort: 8082
|
||||
selector:
|
||||
io.kompose.service: rest-proxy
|
||||
status:
|
||||
loadBalancer: {}
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: schema-registry
|
||||
name: schema-registry
|
||||
spec:
|
||||
ports:
|
||||
- name: "8081"
|
||||
port: 8081
|
||||
targetPort: 8081
|
||||
selector:
|
||||
io.kompose.service: schema-registry
|
||||
status:
|
||||
loadBalancer: {}
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: zookeeper
|
||||
name: zookeeper
|
||||
spec:
|
||||
ports:
|
||||
- name: "2181"
|
||||
port: 2181
|
||||
targetPort: 2181
|
||||
selector:
|
||||
io.kompose.service: zookeeper
|
||||
status:
|
||||
loadBalancer: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: broker
|
||||
name: broker
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: broker
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: CONFLUENT_METRICS_ENABLE
|
||||
value: "true"
|
||||
- name: CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS
|
||||
value: broker:29092
|
||||
- name: CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS
|
||||
value: "1"
|
||||
- name: CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT
|
||||
value: zookeeper:2181
|
||||
- name: CONFLUENT_SUPPORT_CUSTOMER_ID
|
||||
value: anonymous
|
||||
- name: KAFKA_ADVERTISED_LISTENERS
|
||||
value: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
|
||||
- name: KAFKA_BROKER_ID
|
||||
value: "1"
|
||||
- name: KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR
|
||||
value: "1"
|
||||
- name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS
|
||||
value: "0"
|
||||
- name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
|
||||
value: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
- name: KAFKA_METRIC_REPORTERS
|
||||
value: io.confluent.metrics.reporter.ConfluentMetricsReporter
|
||||
- name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
|
||||
value: "1"
|
||||
- name: KAFKA_ZOOKEEPER_CONNECT
|
||||
value: zookeeper:2181
|
||||
image: confluentinc/cp-server:5.4.0
|
||||
name: broker
|
||||
ports:
|
||||
- containerPort: 9092
|
||||
resources: {}
|
||||
hostname: broker
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: connect
|
||||
name: connect
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: connect
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: CLASSPATH
|
||||
value: /usr/share/java/monitoring-interceptors/monitoring-interceptors-5.4.0.jar
|
||||
- name: CONNECT_BOOTSTRAP_SERVERS
|
||||
value: broker:29092
|
||||
- name: CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR
|
||||
value: "1"
|
||||
- name: CONNECT_CONFIG_STORAGE_TOPIC
|
||||
value: docker-connect-configs
|
||||
- name: CONNECT_CONSUMER_INTERCEPTOR_CLASSES
|
||||
value: io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor
|
||||
- name: CONNECT_GROUP_ID
|
||||
value: compose-connect-group
|
||||
- name: CONNECT_INTERNAL_KEY_CONVERTER
|
||||
value: org.apache.kafka.connect.json.JsonConverter
|
||||
- name: CONNECT_INTERNAL_VALUE_CONVERTER
|
||||
value: org.apache.kafka.connect.json.JsonConverter
|
||||
- name: CONNECT_KEY_CONVERTER
|
||||
value: org.apache.kafka.connect.storage.StringConverter
|
||||
- name: CONNECT_LOG4J_LOGGERS
|
||||
value: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
|
||||
- name: CONNECT_OFFSET_FLUSH_INTERVAL_MS
|
||||
value: "10000"
|
||||
- name: CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR
|
||||
value: "1"
|
||||
- name: CONNECT_OFFSET_STORAGE_TOPIC
|
||||
value: docker-connect-offsets
|
||||
- name: CONNECT_PLUGIN_PATH
|
||||
value: /usr/share/java,/usr/share/confluent-hub-components
|
||||
- name: CONNECT_PRODUCER_INTERCEPTOR_CLASSES
|
||||
value: io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor
|
||||
- name: CONNECT_REST_ADVERTISED_HOST_NAME
|
||||
value: connect
|
||||
- name: CONNECT_REST_PORT
|
||||
value: "8083"
|
||||
- name: CONNECT_STATUS_STORAGE_REPLICATION_FACTOR
|
||||
value: "1"
|
||||
- name: CONNECT_STATUS_STORAGE_TOPIC
|
||||
value: docker-connect-status
|
||||
- name: CONNECT_VALUE_CONVERTER
|
||||
value: io.confluent.connect.avro.AvroConverter
|
||||
- name: CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL
|
||||
value: http://schema-registry:8081
|
||||
- name: CONNECT_ZOOKEEPER_CONNECT
|
||||
value: zookeeper:2181
|
||||
image: cnfldemos/cp-server-connect-datagen:0.2.0-5.4.0
|
||||
name: connect
|
||||
ports:
|
||||
- containerPort: 8083
|
||||
resources: {}
|
||||
hostname: connect
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: control-center
|
||||
name: control-center
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: control-center
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: CONFLUENT_METRICS_TOPIC_REPLICATION
|
||||
value: "1"
|
||||
- name: CONTROL_CENTER_BOOTSTRAP_SERVERS
|
||||
value: broker:29092
|
||||
- name: CONTROL_CENTER_CONNECT_CLUSTER
|
||||
value: connect:8083
|
||||
- name: CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS
|
||||
value: "1"
|
||||
- name: CONTROL_CENTER_KSQL_ADVERTISED_URL
|
||||
value: http://localhost:8088
|
||||
- name: CONTROL_CENTER_KSQL_URL
|
||||
value: http://ksql-server:8088
|
||||
- name: CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS
|
||||
value: "1"
|
||||
- name: CONTROL_CENTER_REPLICATION_FACTOR
|
||||
value: "1"
|
||||
- name: CONTROL_CENTER_SCHEMA_REGISTRY_URL
|
||||
value: http://schema-registry:8081
|
||||
- name: CONTROL_CENTER_ZOOKEEPER_CONNECT
|
||||
value: zookeeper:2181
|
||||
- name: PORT
|
||||
value: "9021"
|
||||
image: confluentinc/cp-enterprise-control-center:5.4.0
|
||||
name: control-center
|
||||
ports:
|
||||
- containerPort: 9021
|
||||
resources: {}
|
||||
hostname: control-center
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: ksql-cli
|
||||
name: ksql-cli
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: ksql-cli
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/sh
|
||||
image: confluentinc/cp-ksql-cli:5.4.0
|
||||
name: ksql-cli
|
||||
resources: {}
|
||||
tty: true
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: ksql-datagen
|
||||
name: ksql-datagen
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: ksql-datagen
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- bash
|
||||
- -c
|
||||
- echo Waiting for Kafka to be ready... && cub kafka-ready -b broker:29092
|
||||
1 40 && echo Waiting for Confluent Schema Registry to be ready... && cub
|
||||
sr-ready schema-registry 8081 40 && echo Waiting a few seconds for topic
|
||||
creation to finish... && sleep 11 && tail -f /dev/null
|
||||
env:
|
||||
- name: KSQL_CONFIG_DIR
|
||||
value: /etc/ksql
|
||||
- name: KSQL_LOG4J_OPTS
|
||||
value: -Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties
|
||||
- name: STREAMS_BOOTSTRAP_SERVERS
|
||||
value: broker:29092
|
||||
- name: STREAMS_SCHEMA_REGISTRY_HOST
|
||||
value: schema-registry
|
||||
- name: STREAMS_SCHEMA_REGISTRY_PORT
|
||||
value: "8081"
|
||||
image: confluentinc/ksql-examples:5.4.0
|
||||
name: ksql-datagen
|
||||
resources: {}
|
||||
hostname: ksql-datagen
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: ksql-server
|
||||
name: ksql-server
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: ksql-server
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: KSQL_BOOTSTRAP_SERVERS
|
||||
value: broker:29092
|
||||
- name: KSQL_CACHE_MAX_BYTES_BUFFERING
|
||||
value: "0"
|
||||
- name: KSQL_CONFIG_DIR
|
||||
value: /etc/ksql
|
||||
- name: KSQL_CONSUMER_INTERCEPTOR_CLASSES
|
||||
value: io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor
|
||||
- name: KSQL_HOST_NAME
|
||||
value: ksql-server
|
||||
- name: KSQL_KSQL_SCHEMA_REGISTRY_URL
|
||||
value: http://schema-registry:8081
|
||||
- name: KSQL_LISTENERS
|
||||
value: http://0.0.0.0:8088
|
||||
- name: KSQL_LOG4J_OPTS
|
||||
value: -Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties
|
||||
- name: KSQL_PRODUCER_INTERCEPTOR_CLASSES
|
||||
value: io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor
|
||||
image: confluentinc/cp-ksql-server:5.4.0
|
||||
name: ksql-server
|
||||
ports:
|
||||
- containerPort: 8088
|
||||
resources: {}
|
||||
hostname: ksql-server
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: rest-proxy
|
||||
name: rest-proxy
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: rest-proxy
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: KAFKA_REST_BOOTSTRAP_SERVERS
|
||||
value: broker:29092
|
||||
- name: KAFKA_REST_HOST_NAME
|
||||
value: rest-proxy
|
||||
- name: KAFKA_REST_LISTENERS
|
||||
value: http://0.0.0.0:8082
|
||||
- name: KAFKA_REST_SCHEMA_REGISTRY_URL
|
||||
value: http://schema-registry:8081
|
||||
image: confluentinc/cp-kafka-rest:5.4.0
|
||||
name: rest-proxy
|
||||
ports:
|
||||
- containerPort: 8082
|
||||
resources: {}
|
||||
hostname: rest-proxy
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: schema-registry
|
||||
name: schema-registry
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: schema-registry
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: SCHEMA_REGISTRY_HOST_NAME
|
||||
value: schema-registry
|
||||
- name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL
|
||||
value: zookeeper:2181
|
||||
image: confluentinc/cp-schema-registry:5.4.0
|
||||
name: schema-registry
|
||||
ports:
|
||||
- containerPort: 8081
|
||||
resources: {}
|
||||
hostname: schema-registry
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
- apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: zookeeper
|
||||
name: zookeeper
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose convert -f docker-compose.yml -o kube.yaml
|
||||
kompose.version: 1.19.0 (f63a961c)
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
io.kompose.service: zookeeper
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: ZOOKEEPER_CLIENT_PORT
|
||||
value: "2181"
|
||||
- name: ZOOKEEPER_TICK_TIME
|
||||
value: "2000"
|
||||
image: confluentinc/cp-zookeeper:5.4.0
|
||||
name: zookeeper
|
||||
ports:
|
||||
- containerPort: 2181
|
||||
resources: {}
|
||||
hostname: zookeeper
|
||||
restartPolicy: Always
|
||||
status: {}
|
||||
kind: List
|
||||
metadata: {}
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
confluent-kafka==1.3.0
|
||||
pkg-resources==0.0.0
|
||||
Reference in New Issue
Block a user