mirror of
https://github.com/daniviga/bite.git
synced 2024-12-23 08:41:24 +01:00
Implement MQTT ingestion and improve dockerization (#13)
* Add mqtt-to-db command * Minor fixes * Ignore production.py on git * Add a production conf * Add django container * Add gunicorn for prod and traefik
This commit is contained in:
parent
0ee0b51078
commit
8e5a407b28
3
.gitignore
vendored
3
.gitignore
vendored
@ -129,3 +129,6 @@ dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
##
|
||||
production.py
|
||||
|
21
docker/django/Dockerfile
Normal file
21
docker/django/Dockerfile
Normal file
@ -0,0 +1,21 @@
|
||||
FROM python:3.8-alpine AS builder
|
||||
RUN apk update && apk add gcc musl-dev postgresql-dev \
|
||||
&& pip install psycopg2-binary
|
||||
|
||||
# ---
|
||||
|
||||
FROM python:3.8-alpine
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
ENV DJANGO_SETTINGS_MODULE "freedcs.settings"
|
||||
|
||||
RUN apk update && apk add --no-cache postgresql-libs
|
||||
COPY --from=builder /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/site-packages/
|
||||
COPY --chown=1000:1000 freedcs /srv/app/freedcs
|
||||
COPY --chown=1000:1000 requirements.txt /tmp/requirements.txt
|
||||
|
||||
RUN pip3 install -r /tmp/requirements.txt && rm /tmp/requirements.txt
|
||||
|
||||
USER 1000:1000
|
||||
WORKDIR /srv/app/freedcs
|
||||
EXPOSE 8000/tcp
|
||||
CMD ["python3", "manage.py", "runserver"]
|
29
docker/django/production.py.sample
Normal file
29
docker/django/production.py.sample
Normal file
@ -0,0 +1,29 @@
|
||||
# vim: syntax=python
|
||||
|
||||
from freedcs import settings
|
||||
|
||||
# SECURITY WARNING: keep the secret key used in production secret!
|
||||
SECRET_KEY = 'i4z%50+4b4ek(l0#!w2-r1hpo%&r6tk7p$p_-(=6d!c9n=g5m&'
|
||||
|
||||
# SECURITY WARNING: don't run with debug turned on in production!
|
||||
DEBUG = True
|
||||
|
||||
ALLOWED_HOSTS = ['*']
|
||||
|
||||
DATABASES = {
|
||||
'default': {
|
||||
'ENGINE': 'django.db.backends.postgresql',
|
||||
'NAME': 'freedcs',
|
||||
'USER': 'freedcs',
|
||||
'PASSWORD': 'password',
|
||||
'HOST': 'timescale',
|
||||
'PORT': '5432',
|
||||
}
|
||||
}
|
||||
|
||||
MQTT_BROKER = {
|
||||
'HOST': 'rabbitmq',
|
||||
'PORT': 1883,
|
||||
}
|
||||
|
||||
SKIP_WHITELIST = True
|
20
docker/docker-compose.prod.yml
Normal file
20
docker/docker-compose.prod.yml
Normal file
@ -0,0 +1,20 @@
|
||||
version: "3.7"
|
||||
|
||||
services:
|
||||
ingress:
|
||||
command: --providers.docker
|
||||
ports:
|
||||
- "80:80"
|
||||
|
||||
freedcs:
|
||||
volumes:
|
||||
- "./django/production.py.sample:/srv/freedcs/freedcs/production.py"
|
||||
command: ["gunicorn", "-b", "0.0.0.0:8000", "freedcs.wsgi:application"]
|
||||
|
||||
data-migration:
|
||||
volumes:
|
||||
- "./django/production.py.sample:/srv/freedcs/freedcs/production.py"
|
||||
|
||||
mqtt-to-db:
|
||||
volumes:
|
||||
- "./django/production.py.sample:/srv/freedcs/freedcs/production.py"
|
@ -19,7 +19,7 @@ services:
|
||||
networks:
|
||||
- net
|
||||
ports:
|
||||
- "127.0.0.1:123:123/udp"
|
||||
- "123:123/udp"
|
||||
|
||||
timescale:
|
||||
<<: *service_default
|
||||
@ -31,22 +31,6 @@ services:
|
||||
- "pgdata:/var/lib/postgresql/data"
|
||||
networks:
|
||||
- net
|
||||
ports:
|
||||
- "127.0.0.1:5432:5432"
|
||||
|
||||
# mosquitto simple deployment
|
||||
# mqtt:
|
||||
# <<: *service_default
|
||||
# # image: vernemq/vernemq
|
||||
# # environment:
|
||||
# # DOCKER_VERNEMQ_ALLOW_ANONYMOUS: "on"
|
||||
# # DOCKER_VERNEMQ_ACCEPT_EULA: "yes"
|
||||
# image: eclipse-mosquitto
|
||||
# networks:
|
||||
# - net
|
||||
# ports:
|
||||
# - "1883:1883"
|
||||
# # - "9001:9001" # mqtt via websocket
|
||||
|
||||
rabbitmq:
|
||||
<<: *service_default
|
||||
@ -61,16 +45,65 @@ services:
|
||||
- net
|
||||
ports:
|
||||
- "1883:1883"
|
||||
- "5672:5672"
|
||||
- "15672:15672"
|
||||
|
||||
edge:
|
||||
<<: *service_default
|
||||
image: docker:dind
|
||||
privileged: true
|
||||
environment:
|
||||
DOCKER_TLS_CERTDIR:
|
||||
# edge:
|
||||
# <<: *service_default
|
||||
# image: docker:dind
|
||||
# privileged: true
|
||||
# environment:
|
||||
# DOCKER_TLS_CERTDIR:
|
||||
# networks:
|
||||
# - net
|
||||
# ports:
|
||||
# - "127.0.0.1:22375:2375"
|
||||
|
||||
ingress:
|
||||
image: traefik:v2.2
|
||||
command: --api.insecure=true --providers.docker
|
||||
ports:
|
||||
- "8000:80"
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- net
|
||||
ports:
|
||||
- "127.0.0.1:22375:2375"
|
||||
volumes:
|
||||
# So that Traefik can listen to the Docker events
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
|
||||
freedcs:
|
||||
<<: *service_default
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: ./docker/django/Dockerfile
|
||||
image: daniviga/freedcs
|
||||
volumes:
|
||||
- "../freedcs:/srv/freedcs"
|
||||
command: ["python3", "manage.py", "runserver", "0.0.0.0:8000"]
|
||||
networks:
|
||||
- net
|
||||
depends_on:
|
||||
- data-migration
|
||||
- timescale
|
||||
labels:
|
||||
- "traefik.http.routers.freedcs.rule=PathPrefix(`/`)"
|
||||
|
||||
data-migration:
|
||||
image: daniviga/freedcs
|
||||
volumes:
|
||||
- "../freedcs:/srv/freedcs"
|
||||
command: ["python3", "manage.py", "migrate", "--noinput"]
|
||||
networks:
|
||||
- net
|
||||
|
||||
mqtt-to-db:
|
||||
<<: *service_default
|
||||
image: daniviga/freedcs
|
||||
volumes:
|
||||
- "../freedcs:/srv/freedcs"
|
||||
command: ["python3", "manage.py", "mqtt-to-db"]
|
||||
networks:
|
||||
- net
|
||||
depends_on:
|
||||
- data-migration
|
||||
- timescale
|
||||
- rabbitmq
|
||||
|
0
docker/simulator/device_simulator.py
Normal file → Executable file
0
docker/simulator/device_simulator.py
Normal file → Executable file
27
freedcs/freedcs/production.py.sample
Normal file
27
freedcs/freedcs/production.py.sample
Normal file
@ -0,0 +1,27 @@
|
||||
from freedcs import settings
|
||||
|
||||
# SECURITY WARNING: keep the secret key used in production secret!
|
||||
SECRET_KEY = 'i4z%50+4b4ek(l0#!w2-r1hpo%&r6tk7p$p_-(=6d!c9n=g5m&'
|
||||
|
||||
# SECURITY WARNING: don't run with debug turned on in production!
|
||||
DEBUG = True
|
||||
|
||||
ALLOWED_HOSTS = ['*']
|
||||
|
||||
DATABASES = {
|
||||
'default': {
|
||||
'ENGINE': 'django.db.backends.postgresql',
|
||||
'NAME': 'freedcs',
|
||||
'USER': 'freedcs',
|
||||
'PASSWORD': 'password',
|
||||
'HOST': 'timescale',
|
||||
'PORT': '5432',
|
||||
}
|
||||
}
|
||||
|
||||
MQTT_BROKER = {
|
||||
'HOST': 'rabbitmq',
|
||||
'PORT': '1883',
|
||||
}
|
||||
|
||||
SKIP_WHITELIST = True
|
@ -82,7 +82,7 @@ DATABASES = {
|
||||
'NAME': 'freedcs',
|
||||
'USER': 'freedcs',
|
||||
'PASSWORD': 'password',
|
||||
'HOST': '127.0.0.1',
|
||||
'HOST': 'timescale',
|
||||
'PORT': '5432',
|
||||
}
|
||||
}
|
||||
@ -127,3 +127,17 @@ USE_TZ = True
|
||||
STATIC_URL = '/static/'
|
||||
|
||||
SKIP_WHITELIST = True
|
||||
|
||||
MQTT_BROKER = {
|
||||
'HOST': 'rabbitmq',
|
||||
'PORT': '1883',
|
||||
}
|
||||
|
||||
# If no local_settings.py is availble in the current folder let's try to
|
||||
# load it from the application root
|
||||
try:
|
||||
from freedcs.production import *
|
||||
except ImportError:
|
||||
# If a local_setting.py does not exist
|
||||
# settings in this file only will be used
|
||||
pass
|
||||
|
@ -4,7 +4,7 @@ from telemetry.models import Telemetry
|
||||
|
||||
@admin.register(Telemetry)
|
||||
class TelemetryAdmin(admin.ModelAdmin):
|
||||
readonly_fields = ('device', 'time', 'clock', 'payload',)
|
||||
readonly_fields = ('device', 'transport', 'time', 'clock', 'payload',)
|
||||
list_display = ('__str__', 'device')
|
||||
list_filter = ('time', 'device__serial')
|
||||
search_fields = ('device__serial',)
|
||||
|
52
freedcs/telemetry/management/commands/mqtt-to-db.py
Normal file
52
freedcs/telemetry/management/commands/mqtt-to-db.py
Normal file
@ -0,0 +1,52 @@
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import paho.mqtt.client as mqtt
|
||||
from asgiref.sync import sync_to_async
|
||||
from asyncio_mqtt import Client
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
from api.models import Device
|
||||
from telemetry.models import Telemetry
|
||||
|
||||
MQTT_HOST = settings.MQTT_BROKER['HOST']
|
||||
MQTT_PORT = int(settings.MQTT_BROKER['PORT'])
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'MQTT to DB deamon'
|
||||
|
||||
@sync_to_async
|
||||
def get_device(self, serial):
|
||||
return Device.objects.get(serial=serial)
|
||||
|
||||
@sync_to_async
|
||||
def store_telemetry(self, device, payload):
|
||||
Telemetry.objects.create(
|
||||
device=device,
|
||||
transport='mqtt',
|
||||
clock=payload['clock'],
|
||||
payload=payload['payload']
|
||||
)
|
||||
|
||||
async def mqtt_broker(self):
|
||||
async with Client(MQTT_HOST, port=MQTT_PORT) as client:
|
||||
await client.subscribe("#")
|
||||
async with client.unfiltered_messages() as messages:
|
||||
async for message in messages:
|
||||
payload = json.loads(message.payload.decode('utf-8'))
|
||||
device = await self.get_device(message.topic)
|
||||
await self.store_telemetry(device, payload)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
client = mqtt.Client()
|
||||
while True:
|
||||
try:
|
||||
client.connect(MQTT_HOST, MQTT_PORT)
|
||||
break
|
||||
except ConnectionRefusedError:
|
||||
self.stdout.write('WARNING: Broker not available')
|
||||
time.sleep(5)
|
||||
client.disconnect()
|
||||
asyncio.run(self.mqtt_broker())
|
@ -23,9 +23,9 @@ urlpatterns = [
|
||||
path('<str:device>/',
|
||||
TelemetryView.as_view({'get': 'list'}),
|
||||
name='device-telemetry'),
|
||||
path('<str:device>/latest/',
|
||||
path('<str:device>/last/',
|
||||
TelemetryLatest.as_view({'get': 'retrieve'}),
|
||||
name='device-telemetry-latest'),
|
||||
name='device-telemetry-last'),
|
||||
path('<str:device>/<str:time_from>/',
|
||||
TelemetryRange.as_view({'get': 'list'}),
|
||||
name='device-telemetry-single'),
|
||||
|
@ -1,4 +1,6 @@
|
||||
Django
|
||||
djangorestframework
|
||||
psycopg2-binary
|
||||
kombu
|
||||
paho-mqtt==1.5.0
|
||||
asyncio-mqtt==0.5.0
|
||||
gunicorn
|
||||
|
Loading…
Reference in New Issue
Block a user