Implement MQTT ingestion and improve dockerization (#13)

* Add mqtt-to-db command

* Minor fixes

* Ignore production.py on git

* Add a production conf

* Add django container

* Add gunicorn for prod and traefik

.gitignore

@@ -129,3 +129,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+##
+production.py

docker/django/Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.8-alpine AS builder
+RUN apk update && apk add gcc musl-dev postgresql-dev \
+        && pip install psycopg2-binary
+
+# ---
+
+FROM python:3.8-alpine
+ENV PYTHONUNBUFFERED 1
+ENV DJANGO_SETTINGS_MODULE "freedcs.settings"
+
+RUN apk update && apk add --no-cache postgresql-libs
+COPY --from=builder /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/site-packages/
+COPY --chown=1000:1000 freedcs /srv/app/freedcs
+COPY --chown=1000:1000 requirements.txt /tmp/requirements.txt
+
+RUN pip3 install -r /tmp/requirements.txt && rm /tmp/requirements.txt
+
+USER 1000:1000
+WORKDIR /srv/app/freedcs
+EXPOSE 8000/tcp
+CMD ["python3", "manage.py", "runserver"]

docker/django/production.py.sample

@@ -0,0 +1,29 @@
+# vim: syntax=python
+
+from freedcs import settings
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'i4z%50+4b4ek(l0#!w2-r1hpo%&r6tk7p$p_-(=6d!c9n=g5m&'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ['*']
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': 'freedcs',
+        'USER': 'freedcs',
+        'PASSWORD': 'password',
+        'HOST': 'timescale',
+        'PORT': '5432',
+    }
+}
+
+MQTT_BROKER = {
+    'HOST': 'rabbitmq',
+    'PORT': 1883,
+}
+
+SKIP_WHITELIST = True

docker/docker-compose.prod.yml

@@ -0,0 +1,20 @@
+version: "3.7"
+
+services:
+  ingress:
+    command: --providers.docker
+    ports:
+      - "80:80"
+
+  freedcs:
+    volumes:
+      - "./django/production.py.sample:/srv/freedcs/freedcs/production.py"
+    command: ["gunicorn", "-b", "0.0.0.0:8000", "freedcs.wsgi:application"]
+
+  data-migration:
+    volumes:
+      - "./django/production.py.sample:/srv/freedcs/freedcs/production.py"
+
+  mqtt-to-db:
+    volumes:
+      - "./django/production.py.sample:/srv/freedcs/freedcs/production.py"

docker/docker-compose.yml

@@ -19,7 +19,7 @@ services:
     networks:
       - net
     ports:
-      - "127.0.0.1:123:123/udp"
+      - "123:123/udp"
 
   timescale:
     <<: *service_default
@@ -31,22 +31,6 @@ services:
       - "pgdata:/var/lib/postgresql/data"
     networks:
       - net
-    ports:
-      - "127.0.0.1:5432:5432"
-
-  # mosquitto simple deployment
-  # mqtt:
-  #   <<: *service_default
-  #   # image: vernemq/vernemq
-  #   # environment:
-  #   #   DOCKER_VERNEMQ_ALLOW_ANONYMOUS: "on"
-  #   #   DOCKER_VERNEMQ_ACCEPT_EULA: "yes"
-  #   image: eclipse-mosquitto
-  #   networks:
-  #     - net
-  #   ports:
-  #     - "1883:1883"
-  #   #  - "9001:9001"  # mqtt via websocket
 
   rabbitmq:
     <<: *service_default
@@ -61,16 +45,65 @@ services:
       - net
     ports:
       - "1883:1883"
-      - "5672:5672"
       - "15672:15672"
 
-  edge:
+  # edge:
-    <<: *service_default
+  #   <<: *service_default
-    image: docker:dind
+  #   image: docker:dind
-    privileged: true
+  #   privileged: true
-    environment:
+  #   environment:
-      DOCKER_TLS_CERTDIR:
+  #     DOCKER_TLS_CERTDIR:
+  #   networks:
+  #     - net
+  #   ports:
+  #     - "127.0.0.1:22375:2375"
+
+  ingress:
+    image: traefik:v2.2
+    command: --api.insecure=true --providers.docker
+    ports:
+      - "8000:80"
+      - "8080:8080"
     networks:
       - net
-    ports:
+    volumes:
-      - "127.0.0.1:22375:2375"
+      # So that Traefik can listen to the Docker events
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  freedcs:
+    <<: *service_default
+    build:
+      context: ..
+      dockerfile: ./docker/django/Dockerfile
+    image: daniviga/freedcs
+    volumes:
+      - "../freedcs:/srv/freedcs"
+    command: ["python3", "manage.py", "runserver", "0.0.0.0:8000"]
+    networks:
+      - net
+    depends_on:
+      - data-migration
+      - timescale
+    labels:
+      - "traefik.http.routers.freedcs.rule=PathPrefix(`/`)"
+
+  data-migration:
+    image: daniviga/freedcs
+    volumes:
+      - "../freedcs:/srv/freedcs"
+    command: ["python3", "manage.py", "migrate", "--noinput"]
+    networks:
+      - net
+
+  mqtt-to-db:
+    <<: *service_default
+    image: daniviga/freedcs
+    volumes:
+      - "../freedcs:/srv/freedcs"
+    command: ["python3", "manage.py", "mqtt-to-db"]
+    networks:
+      - net
+    depends_on:
+      - data-migration
+      - timescale
+      - rabbitmq

freedcs/freedcs/production.py.sample

@@ -0,0 +1,27 @@
+from freedcs import settings
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'i4z%50+4b4ek(l0#!w2-r1hpo%&r6tk7p$p_-(=6d!c9n=g5m&'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ['*']
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': 'freedcs',
+        'USER': 'freedcs',
+        'PASSWORD': 'password',
+        'HOST': 'timescale',
+        'PORT': '5432',
+    }
+}
+
+MQTT_BROKER = {
+    'HOST': 'rabbitmq',
+    'PORT': '1883',
+}
+
+SKIP_WHITELIST = True

freedcs/freedcs/settings.py

@@ -82,7 +82,7 @@ DATABASES = {
         'NAME': 'freedcs',
         'USER': 'freedcs',
         'PASSWORD': 'password',
-        'HOST': '127.0.0.1',
+        'HOST': 'timescale',
         'PORT': '5432',
     }
 }
@@ -127,3 +127,17 @@ USE_TZ = True
 STATIC_URL = '/static/'
 
 SKIP_WHITELIST = True
+
+MQTT_BROKER = {
+    'HOST': 'rabbitmq',
+    'PORT': '1883',
+}
+
+# If no local_settings.py is availble in the current folder let's try to
+# load it from the application root
+try:
+    from freedcs.production import *
+except ImportError:
+    # If a local_setting.py does not exist
+    # settings in this file only will be used
+    pass

freedcs/telemetry/admin.py

@@ -4,7 +4,7 @@ from telemetry.models import Telemetry
 
 @admin.register(Telemetry)
 class TelemetryAdmin(admin.ModelAdmin):
-    readonly_fields = ('device', 'time', 'clock', 'payload',)
+    readonly_fields = ('device', 'transport', 'time', 'clock', 'payload',)
     list_display = ('__str__', 'device')
     list_filter = ('time', 'device__serial')
     search_fields = ('device__serial',)

freedcs/telemetry/management/commands/mqtt-to-db.py

@@ -0,0 +1,52 @@
+import asyncio
+import json
+import time
+import paho.mqtt.client as mqtt
+from asgiref.sync import sync_to_async
+from asyncio_mqtt import Client
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from api.models import Device
+from telemetry.models import Telemetry
+
+MQTT_HOST = settings.MQTT_BROKER['HOST']
+MQTT_PORT = int(settings.MQTT_BROKER['PORT'])
+
+
+class Command(BaseCommand):
+    help = 'MQTT to DB deamon'
+
+    @sync_to_async
+    def get_device(self, serial):
+        return Device.objects.get(serial=serial)
+
+    @sync_to_async
+    def store_telemetry(self, device, payload):
+        Telemetry.objects.create(
+            device=device,
+            transport='mqtt',
+            clock=payload['clock'],
+            payload=payload['payload']
+        )
+
+    async def mqtt_broker(self):
+        async with Client(MQTT_HOST, port=MQTT_PORT) as client:
+            await client.subscribe("#")
+            async with client.unfiltered_messages() as messages:
+                async for message in messages:
+                    payload = json.loads(message.payload.decode('utf-8'))
+                    device = await self.get_device(message.topic)
+                    await self.store_telemetry(device, payload)
+
+    def handle(self, *args, **options):
+        client = mqtt.Client()
+        while True:
+            try:
+                client.connect(MQTT_HOST, MQTT_PORT)
+                break
+            except ConnectionRefusedError:
+                self.stdout.write('WARNING: Broker not available')
+                time.sleep(5)
+        client.disconnect()
+        asyncio.run(self.mqtt_broker())

freedcs/telemetry/urls.py

@@ -23,9 +23,9 @@ urlpatterns = [
     path('<str:device>/',
          TelemetryView.as_view({'get': 'list'}),
          name='device-telemetry'),
-    path('<str:device>/latest/',
+    path('<str:device>/last/',
          TelemetryLatest.as_view({'get': 'retrieve'}),
-         name='device-telemetry-latest'),
+         name='device-telemetry-last'),
     path('<str:device>/<str:time_from>/',
          TelemetryRange.as_view({'get': 'list'}),
          name='device-telemetry-single'),

requirements.txt

@@ -1,4 +1,6 @@
 Django
 djangorestframework
 psycopg2-binary
-kombu
+paho-mqtt==1.5.0
+asyncio-mqtt==0.5.0
+gunicorn