From be4c226955d55ee99c74b0e0772e71326109081b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniele=20Vigan=C3=B2?= Date: Sun, 21 Jun 2020 22:58:46 +0200 Subject: [PATCH] Fix HTTP reverse proxy with embedded IoT (#20) * Add an ENV far for custom docker binding * Delay in Arduino can be customized * Fix 400 error with Arduino telemetry and Nginx This was caused by the client not waiting for a response. Also Content-Length header was bogus. * Do not crash mqtt-to-db if devices isn't proviosioned * Update READMEs --- README.md | 6 +++++- arduino/README.md | 8 +++++--- arduino/tempLightSensor/tempLightSensor.ino | 13 +++++++------ bite/telemetry/management/commands/mqtt-to-db.py | 14 ++++++++++++-- docker/docker-compose.dev.yml | 6 +++--- docker/docker-compose.yml | 6 +++--- docker/ingress/nginx.conf | 4 ++++ 7 files changed, 39 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 533a6e5..71f4037 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ a [PostgreSQL](https://www.postgresql.org/) database with a timeseries extension ## Deployment +The `$CUSTOM_DOCKER_IP` environment variable can be used to set a custom +IP address to bind ports. Default is `0.0.0.0`; `127.0.0.1` is a +safe configuration (see https://github.com/docker/compose/issues/2999). + ### Development ```bash @@ -149,7 +153,7 @@ The sketch reads temperature and light from sensors. /* ... */ void loop(void) { - const int postDelay = 10 * 1000; + const int postDelay = TELEMETRY_DELAY * 1000; unsigned int tempReading = analogRead(A0); unsigned int photocellReading = analogRead(A1); diff --git a/arduino/README.md b/arduino/README.md index a93dac3..dcd7f1b 100644 --- a/arduino/README.md +++ b/arduino/README.md @@ -52,7 +52,9 @@ The `EEPROM` can be completely erased setting the `ERASE_FIRST` macro to `1`. The following macros are available in the firmware (to be set at compile time): ```c -#define DEBUG_TO_SERIAL 1 // debug on serial port -#define USE_MQTT 1 // use mqtt protocol instead of http post -#define USE_INTERNAL_NTP 0 // use default ntp server or the internal one +#define DEBUG_TO_SERIAL 0 // debug on serial port +#define USE_MQTT 1 // use mqtt protocol instead of http post +#define USE_INTERNAL_NTP 1 // use default ntp server or the internal one +#define TELEMETRY_DELAY 10 // second between telemetry samples +#define AREF_VOLTAGE 3.3 // set aref voltage to 3.3v instead of default 5v ``` \ No newline at end of file diff --git a/arduino/tempLightSensor/tempLightSensor.ino b/arduino/tempLightSensor/tempLightSensor.ino index 8da1a55..f6d1f4b 100644 --- a/arduino/tempLightSensor/tempLightSensor.ino +++ b/arduino/tempLightSensor/tempLightSensor.ino @@ -5,10 +5,11 @@ #include #include -#define DEBUG_TO_SERIAL 1 // debug on serial port -#define USE_MQTT 1 // use mqtt protocol instead of http post -#define USE_INTERNAL_NTP 0 // use default ntp server or the internal one -#define AREF_VOLTAGE 3.3 // set aref voltage to 3.3v instead of default 5v +#define DEBUG_TO_SERIAL 0 // debug on serial port +#define USE_MQTT 1 // use mqtt protocol instead of http post +#define USE_INTERNAL_NTP 1 // use default ntp server or the internal one +#define TELEMETRY_DELAY 10 // second between telemetry samples +#define AREF_VOLTAGE 3.3 // set aref voltage to 3.3v instead of default 5v char serial[9]; @@ -96,7 +97,7 @@ void setup(void) { } void loop(void) { - const int postDelay = 10 * 1000; + const int postDelay = TELEMETRY_DELAY * 1000; unsigned int tempReading = analogRead(A0); unsigned int photocellReading = analogRead(A1); @@ -161,7 +162,7 @@ void postData(const netConfig &postAPI, const String &URL, const DynamicJsonDocu ethClient.println(postAPI.port); ethClient.println("Content-Type: application/json"); ethClient.print("Content-Length: "); - ethClient.println(measureJsonPretty(json)); + ethClient.println(measureJson(json)); ethClient.println("Connection: close"); ethClient.println(); serializeJson(json, ethClient); diff --git a/bite/telemetry/management/commands/mqtt-to-db.py b/bite/telemetry/management/commands/mqtt-to-db.py index 0d2de1b..73e1ee8 100644 --- a/bite/telemetry/management/commands/mqtt-to-db.py +++ b/bite/telemetry/management/commands/mqtt-to-db.py @@ -8,6 +8,8 @@ from asyncio_mqtt import Client from django.conf import settings from django.core.management.base import BaseCommand +from django.core.exceptions import ObjectDoesNotExist + from api.models import Device from telemetry.models import Telemetry @@ -20,7 +22,10 @@ class Command(BaseCommand): @sync_to_async def get_device(self, serial): - return Device.objects.get(serial=serial) + try: + return Device.objects.get(serial=serial) + except ObjectDoesNotExist: + return None @sync_to_async def store_telemetry(self, device, payload): @@ -39,7 +44,12 @@ class Command(BaseCommand): async for message in messages: payload = json.loads(message.payload.decode('utf-8')) device = await self.get_device(message.topic) - await self.store_telemetry(device, payload) + if device is not None: + await self.store_telemetry(device, payload) + else: + self.stdout.write( + self.style.ERROR( + 'DEBUG: message discarded')) def handle(self, *args, **options): client = mqtt.Client() diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 60e70b0..f0ac275 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -7,15 +7,15 @@ services: broker: ports: - - "1883:1883" - - "9001:9001" + - "${CUSTOM_DOCKER_IP:-0.0.0.0}:1883:1883" + - "${CUSTOM_DOCKER_IP:-0.0.0.0}:9001:9001" bite: volumes: - ../bite:/srv/app/bite command: ["python3", "manage.py", "runserver", "0.0.0.0:8000"] ports: - - "8000:8000" + - "${CUSTOM_DOCKER_IP:-0.0.0.0}:8000:8000" data-migration: volumes: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 845f169..1aa505a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,7 +20,7 @@ services: networks: - net ports: - - "123:123/udp" + - "${CUSTOM_DOCKER_IP:-0.0.0.0}:123:123/udp" timescale: <<: *service_default @@ -41,13 +41,13 @@ services: networks: - net ports: - - "1883:1883" + - "${CUSTOM_DOCKER_IP:-0.0.0.0}:1883:1883" ingress: <<: *service_default image: nginx:stable-alpine ports: - - "80:80" + - "${CUSTOM_DOCKER_IP:-0.0.0.0}:80:80" networks: - net volumes: diff --git a/docker/ingress/nginx.conf b/docker/ingress/nginx.conf index ae019e3..651bf1a 100644 --- a/docker/ingress/nginx.conf +++ b/docker/ingress/nginx.conf @@ -51,6 +51,7 @@ http { location / { proxy_pass http://bite; + proxy_http_version 1.1; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-Host $host:$server_port; @@ -59,6 +60,9 @@ http { proxy_read_timeout 300; proxy_connect_timeout 300; + + ## !- This is mandatory for IoT that do not wait for a reply -! ## + proxy_ignore_client_abort on; } location /mqtt {