From 8e5a407b280a1a90eaae1d158fda619bde96d287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniele=20Vigan=C3=B2?= Date: Mon, 15 Jun 2020 22:47:55 +0200 Subject: [PATCH] Implement MQTT ingestion and improve dockerization (#13) * Add mqtt-to-db command * Minor fixes * Ignore production.py on git * Add a production conf * Add django container * Add gunicorn for prod and traefik --- .gitignore | 3 + docker/django/Dockerfile | 21 +++++ docker/django/production.py.sample | 29 +++++++ docker/docker-compose.prod.yml | 20 +++++ docker/docker-compose.yml | 85 +++++++++++++------ docker/simulator/device_simulator.py | 0 freedcs/freedcs/production.py.sample | 27 ++++++ freedcs/freedcs/settings.py | 16 +++- freedcs/telemetry/admin.py | 2 +- .../management/commands/mqtt-to-db.py | 52 ++++++++++++ freedcs/telemetry/urls.py | 4 +- requirements.txt | 4 +- 12 files changed, 232 insertions(+), 31 deletions(-) create mode 100644 docker/django/Dockerfile create mode 100644 docker/django/production.py.sample create mode 100644 docker/docker-compose.prod.yml mode change 100644 => 100755 docker/simulator/device_simulator.py create mode 100644 freedcs/freedcs/production.py.sample create mode 100644 freedcs/telemetry/management/commands/mqtt-to-db.py diff --git a/.gitignore b/.gitignore index 9dd28e5..e182b94 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,6 @@ dmypy.json # Pyre type checker .pyre/ + +## +production.py diff --git a/docker/django/Dockerfile b/docker/django/Dockerfile new file mode 100644 index 0000000..95f0005 --- /dev/null +++ b/docker/django/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.8-alpine AS builder +RUN apk update && apk add gcc musl-dev postgresql-dev \ + && pip install psycopg2-binary + +# --- + +FROM python:3.8-alpine +ENV PYTHONUNBUFFERED 1 +ENV DJANGO_SETTINGS_MODULE "freedcs.settings" + +RUN apk update && apk add --no-cache postgresql-libs +COPY --from=builder /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/site-packages/ +COPY --chown=1000:1000 freedcs /srv/app/freedcs +COPY --chown=1000:1000 requirements.txt /tmp/requirements.txt + +RUN pip3 install -r /tmp/requirements.txt && rm /tmp/requirements.txt + +USER 1000:1000 +WORKDIR /srv/app/freedcs +EXPOSE 8000/tcp +CMD ["python3", "manage.py", "runserver"] diff --git a/docker/django/production.py.sample b/docker/django/production.py.sample new file mode 100644 index 0000000..0fee18e --- /dev/null +++ b/docker/django/production.py.sample @@ -0,0 +1,29 @@ +# vim: syntax=python + +from freedcs import settings + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'i4z%50+4b4ek(l0#!w2-r1hpo%&r6tk7p$p_-(=6d!c9n=g5m&' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['*'] + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'freedcs', + 'USER': 'freedcs', + 'PASSWORD': 'password', + 'HOST': 'timescale', + 'PORT': '5432', + } +} + +MQTT_BROKER = { + 'HOST': 'rabbitmq', + 'PORT': 1883, +} + +SKIP_WHITELIST = True diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 0000000..984e518 --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,20 @@ +version: "3.7" + +services: + ingress: + command: --providers.docker + ports: + - "80:80" + + freedcs: + volumes: + - "./django/production.py.sample:/srv/freedcs/freedcs/production.py" + command: ["gunicorn", "-b", "0.0.0.0:8000", "freedcs.wsgi:application"] + + data-migration: + volumes: + - "./django/production.py.sample:/srv/freedcs/freedcs/production.py" + + mqtt-to-db: + volumes: + - "./django/production.py.sample:/srv/freedcs/freedcs/production.py" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ad8d5d6..f56beb3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -19,7 +19,7 @@ services: networks: - net ports: - - "127.0.0.1:123:123/udp" + - "123:123/udp" timescale: <<: *service_default @@ -31,22 +31,6 @@ services: - "pgdata:/var/lib/postgresql/data" networks: - net - ports: - - "127.0.0.1:5432:5432" - - # mosquitto simple deployment - # mqtt: - # <<: *service_default - # # image: vernemq/vernemq - # # environment: - # # DOCKER_VERNEMQ_ALLOW_ANONYMOUS: "on" - # # DOCKER_VERNEMQ_ACCEPT_EULA: "yes" - # image: eclipse-mosquitto - # networks: - # - net - # ports: - # - "1883:1883" - # # - "9001:9001" # mqtt via websocket rabbitmq: <<: *service_default @@ -61,16 +45,65 @@ services: - net ports: - "1883:1883" - - "5672:5672" - "15672:15672" - edge: - <<: *service_default - image: docker:dind - privileged: true - environment: - DOCKER_TLS_CERTDIR: + # edge: + # <<: *service_default + # image: docker:dind + # privileged: true + # environment: + # DOCKER_TLS_CERTDIR: + # networks: + # - net + # ports: + # - "127.0.0.1:22375:2375" + + ingress: + image: traefik:v2.2 + command: --api.insecure=true --providers.docker + ports: + - "8000:80" + - "8080:8080" networks: - net - ports: - - "127.0.0.1:22375:2375" + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock + + freedcs: + <<: *service_default + build: + context: .. + dockerfile: ./docker/django/Dockerfile + image: daniviga/freedcs + volumes: + - "../freedcs:/srv/freedcs" + command: ["python3", "manage.py", "runserver", "0.0.0.0:8000"] + networks: + - net + depends_on: + - data-migration + - timescale + labels: + - "traefik.http.routers.freedcs.rule=PathPrefix(`/`)" + + data-migration: + image: daniviga/freedcs + volumes: + - "../freedcs:/srv/freedcs" + command: ["python3", "manage.py", "migrate", "--noinput"] + networks: + - net + + mqtt-to-db: + <<: *service_default + image: daniviga/freedcs + volumes: + - "../freedcs:/srv/freedcs" + command: ["python3", "manage.py", "mqtt-to-db"] + networks: + - net + depends_on: + - data-migration + - timescale + - rabbitmq diff --git a/docker/simulator/device_simulator.py b/docker/simulator/device_simulator.py old mode 100644 new mode 100755 diff --git a/freedcs/freedcs/production.py.sample b/freedcs/freedcs/production.py.sample new file mode 100644 index 0000000..3e1fedf --- /dev/null +++ b/freedcs/freedcs/production.py.sample @@ -0,0 +1,27 @@ +from freedcs import settings + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'i4z%50+4b4ek(l0#!w2-r1hpo%&r6tk7p$p_-(=6d!c9n=g5m&' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['*'] + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'freedcs', + 'USER': 'freedcs', + 'PASSWORD': 'password', + 'HOST': 'timescale', + 'PORT': '5432', + } +} + +MQTT_BROKER = { + 'HOST': 'rabbitmq', + 'PORT': '1883', +} + +SKIP_WHITELIST = True diff --git a/freedcs/freedcs/settings.py b/freedcs/freedcs/settings.py index 17d4151..0f94a53 100644 --- a/freedcs/freedcs/settings.py +++ b/freedcs/freedcs/settings.py @@ -82,7 +82,7 @@ DATABASES = { 'NAME': 'freedcs', 'USER': 'freedcs', 'PASSWORD': 'password', - 'HOST': '127.0.0.1', + 'HOST': 'timescale', 'PORT': '5432', } } @@ -127,3 +127,17 @@ USE_TZ = True STATIC_URL = '/static/' SKIP_WHITELIST = True + +MQTT_BROKER = { + 'HOST': 'rabbitmq', + 'PORT': '1883', +} + +# If no local_settings.py is availble in the current folder let's try to +# load it from the application root +try: + from freedcs.production import * +except ImportError: + # If a local_setting.py does not exist + # settings in this file only will be used + pass diff --git a/freedcs/telemetry/admin.py b/freedcs/telemetry/admin.py index 6f229f5..b5884d4 100644 --- a/freedcs/telemetry/admin.py +++ b/freedcs/telemetry/admin.py @@ -4,7 +4,7 @@ from telemetry.models import Telemetry @admin.register(Telemetry) class TelemetryAdmin(admin.ModelAdmin): - readonly_fields = ('device', 'time', 'clock', 'payload',) + readonly_fields = ('device', 'transport', 'time', 'clock', 'payload',) list_display = ('__str__', 'device') list_filter = ('time', 'device__serial') search_fields = ('device__serial',) diff --git a/freedcs/telemetry/management/commands/mqtt-to-db.py b/freedcs/telemetry/management/commands/mqtt-to-db.py new file mode 100644 index 0000000..5fd9858 --- /dev/null +++ b/freedcs/telemetry/management/commands/mqtt-to-db.py @@ -0,0 +1,52 @@ +import asyncio +import json +import time +import paho.mqtt.client as mqtt +from asgiref.sync import sync_to_async +from asyncio_mqtt import Client + +from django.conf import settings +from django.core.management.base import BaseCommand +from api.models import Device +from telemetry.models import Telemetry + +MQTT_HOST = settings.MQTT_BROKER['HOST'] +MQTT_PORT = int(settings.MQTT_BROKER['PORT']) + + +class Command(BaseCommand): + help = 'MQTT to DB deamon' + + @sync_to_async + def get_device(self, serial): + return Device.objects.get(serial=serial) + + @sync_to_async + def store_telemetry(self, device, payload): + Telemetry.objects.create( + device=device, + transport='mqtt', + clock=payload['clock'], + payload=payload['payload'] + ) + + async def mqtt_broker(self): + async with Client(MQTT_HOST, port=MQTT_PORT) as client: + await client.subscribe("#") + async with client.unfiltered_messages() as messages: + async for message in messages: + payload = json.loads(message.payload.decode('utf-8')) + device = await self.get_device(message.topic) + await self.store_telemetry(device, payload) + + def handle(self, *args, **options): + client = mqtt.Client() + while True: + try: + client.connect(MQTT_HOST, MQTT_PORT) + break + except ConnectionRefusedError: + self.stdout.write('WARNING: Broker not available') + time.sleep(5) + client.disconnect() + asyncio.run(self.mqtt_broker()) diff --git a/freedcs/telemetry/urls.py b/freedcs/telemetry/urls.py index c7a26e0..c1d26da 100644 --- a/freedcs/telemetry/urls.py +++ b/freedcs/telemetry/urls.py @@ -23,9 +23,9 @@ urlpatterns = [ path('/', TelemetryView.as_view({'get': 'list'}), name='device-telemetry'), - path('/latest/', + path('/last/', TelemetryLatest.as_view({'get': 'retrieve'}), - name='device-telemetry-latest'), + name='device-telemetry-last'), path('//', TelemetryRange.as_view({'get': 'list'}), name='device-telemetry-single'), diff --git a/requirements.txt b/requirements.txt index e3e43ba..820ab80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ Django djangorestframework psycopg2-binary -kombu +paho-mqtt==1.5.0 +asyncio-mqtt==0.5.0 +gunicorn