Implement kafka in Docker compose and set group_id in handlers

bite/bite/settings.py

@@ -151,6 +151,10 @@ STATIC_URL = '/static/'
 
 STATIC_ROOT = '/srv/appdata/bite/static'
 
+REST_FRAMEWORK = {
+    'DEFAULT_AUTHENTICATION_CLASSES': []
+}
+
 SKIP_WHITELIST = True
 
 MQTT_BROKER = {

bite/telemetry/management/commands/handler.py

@@ -55,10 +55,11 @@ class Command(BaseCommand):
             try:
                 consumer = KafkaConsumer(
                     "telemetry",
-                    value_deserializer=lambda m: json.loads(m.decode('utf8')),
                     bootstrap_servers='{}:{}'.format(
                         self.KAFKA_HOST, self.KAFKA_PORT
                     ),
+                    group_id="handler",
+                    value_deserializer=lambda m: json.loads(m.decode('utf8')),
                 )
                 break
             except NoBrokersAvailable:

docker/django/Dockerfile

@@ -17,20 +17,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-FROM python:3.9-alpine AS builder
+FROM python:3.11-alpine AS builder
 RUN apk update && apk add gcc musl-dev postgresql-dev \
         && pip install psycopg2-binary
 
 # ---
 
-FROM python:3.9-alpine
+FROM python:3.11-alpine
 ENV PYTHONUNBUFFERED 1
 ENV DJANGO_SETTINGS_MODULE "bite.settings"
 
 RUN apk update && apk add --no-cache postgresql-libs \
-        && wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-alpine-linux-amd64-v0.6.1.tar.gz -qO- \
+        && wget https://github.com/jwilder/dockerize/releases/download/v0.7.0/dockerize-alpine-linux-amd64-v0.7.0.tar.gz -qO- \
         | tar -xz -C /usr/local/bin
-COPY --from=builder /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/site-packages/
+COPY --from=builder /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
 COPY --chown=1000:1000 requirements.txt /srv/app/bite/requirements.txt
 RUN pip3 install --no-cache-dir -r /srv/app/bite/requirements.txt
 

docker/docker-compose.dev.yml

@@ -36,6 +36,10 @@ services:
     ports:
         - "${CUSTOM_DOCKER_IP:-0.0.0.0}:8000:8000"
 
+  kafka:
+    ports:
+        - "${CUSTOM_DOCKER_IP:-0.0.0.0}:9092:9092"
+
   data-migration:
     volumes:
       - ../bite:/srv/app/bite

docker/docker-compose.yml

@@ -62,6 +62,30 @@ services:
     ports:
       - "${CUSTOM_DOCKER_IP:-0.0.0.0}:1883:1883"
 
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    networks:
+      - net
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - 22181:2181
+
+  kafka:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
+    networks:
+      - net
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+
   ingress:
     <<: *service_default
     image: nginx:stable-alpine

docker/ntpd/Dockerfile

@@ -17,9 +17,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-FROM alpine:3.15
+FROM alpine:3.18
 
-RUN apk update && apk add chrony && \
+RUN apk add --no-cache chrony && \
     chown -R chrony:chrony /var/lib/chrony
 COPY ./chrony.conf /etc/chrony/chrony.conf
 

docker/simulator/Dockerfile

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-FROM python:3.9-alpine
+FROM python:3.11-alpine
 
 RUN pip3 install urllib3 paho-mqtt
 COPY ./device_simulator.py /opt/bite/device_simulator.py

docker/simulator/device_simulator.py

@@ -94,7 +94,7 @@ def main():
     parser.add_argument('-s', '--serial',
                         default=os.environ.get('IOT_SERIAL'),
                         help='IoT device serial number')
-    parser.add_argument('-d', '--delay', metavar='s', type=int,
+    parser.add_argument('-d', '--delay', metavar='s', type=float,
                         default=os.environ.get('IOT_DELAY', 10),
                         help='Delay between requests')
     args = parser.parse_args()

requirements.txt

@@ -4,7 +4,8 @@ djangorestframework
 django-health-check
 psycopg2-binary
 paho-mqtt
-asyncio-mqtt
+kafka-python
+aiomqtt
 PyYAML
 uritemplate
 pygments