Fix HTTP reverse proxy with embedded IoT (#20)

* Add an ENV far for custom docker binding

* Delay in Arduino can be customized

* Fix 400 error with Arduino telemetry and Nginx
This was caused by the client not waiting for a response.
Also Content-Length header was bogus.

* Do not crash mqtt-to-db if devices isn't proviosioned

* Update READMEs

README.md

@@ -47,6 +47,10 @@ a [PostgreSQL](https://www.postgresql.org/) database with a timeseries extension
 
 ## Deployment
 
+The `$CUSTOM_DOCKER_IP` environment variable can be used to set a custom
+IP address to bind ports. Default is `0.0.0.0`; `127.0.0.1` is a
+safe configuration (see https://github.com/docker/compose/issues/2999).
+
 ### Development
 
 ```bash
@@ -149,7 +153,7 @@ The sketch reads temperature and light from sensors.
 /* ... */
 
 void loop(void) {
-  const int postDelay = 10 * 1000;
+  const int postDelay = TELEMETRY_DELAY * 1000;
 
   unsigned int tempReading = analogRead(A0);
   unsigned int photocellReading = analogRead(A1);

arduino/README.md

@@ -52,7 +52,9 @@ The `EEPROM` can be completely erased setting the `ERASE_FIRST` macro to `1`.
 The following macros are available in the firmware (to be set at compile time):
 
 ```c
-#define DEBUG_TO_SERIAL 1  // debug on serial port
-#define USE_MQTT 1         // use mqtt protocol instead of http post
-#define USE_INTERNAL_NTP 0 // use default ntp server or the internal one
+#define DEBUG_TO_SERIAL   0   // debug on serial port
+#define USE_MQTT          1   // use mqtt protocol instead of http post
+#define USE_INTERNAL_NTP  1   // use default ntp server or the internal one
+#define TELEMETRY_DELAY  10   // second between telemetry samples
+#define AREF_VOLTAGE      3.3 // set aref voltage to 3.3v instead of default 5v
 ```

arduino/tempLightSensor/tempLightSensor.ino

@@ -5,10 +5,11 @@
 #include <NTPClient.h>
 #include <ArduinoJson.h>
 
-#define DEBUG_TO_SERIAL 1  // debug on serial port
-#define USE_MQTT 1         // use mqtt protocol instead of http post
-#define USE_INTERNAL_NTP 0 // use default ntp server or the internal one
-#define AREF_VOLTAGE 3.3   // set aref voltage to 3.3v instead of default 5v
+#define DEBUG_TO_SERIAL   0   // debug on serial port
+#define USE_MQTT          1   // use mqtt protocol instead of http post
+#define USE_INTERNAL_NTP  1   // use default ntp server or the internal one
+#define TELEMETRY_DELAY  10   // second between telemetry samples
+#define AREF_VOLTAGE      3.3 // set aref voltage to 3.3v instead of default 5v
 
 char serial[9];
 
@@ -96,7 +97,7 @@ void setup(void) {
 }
 
 void loop(void) {
-  const int postDelay = 10 * 1000;
+  const int postDelay = TELEMETRY_DELAY * 1000;
 
   unsigned int tempReading = analogRead(A0);
   unsigned int photocellReading = analogRead(A1);
@@ -161,7 +162,7 @@ void postData(const netConfig &postAPI, const String &URL, const DynamicJsonDocu
     ethClient.println(postAPI.port);
     ethClient.println("Content-Type: application/json");
     ethClient.print("Content-Length: ");
-    ethClient.println(measureJsonPretty(json));
+    ethClient.println(measureJson(json));
     ethClient.println("Connection: close");
     ethClient.println();
     serializeJson(json, ethClient);

bite/telemetry/management/commands/mqtt-to-db.py

@@ -8,6 +8,8 @@ from asyncio_mqtt import Client
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.core.exceptions import ObjectDoesNotExist
+
 from api.models import Device
 from telemetry.models import Telemetry
 
@@ -20,7 +22,10 @@ class Command(BaseCommand):
 
     @sync_to_async
     def get_device(self, serial):
-        return Device.objects.get(serial=serial)
+        try:
+            return Device.objects.get(serial=serial)
+        except ObjectDoesNotExist:
+            return None
 
     @sync_to_async
     def store_telemetry(self, device, payload):
@@ -39,7 +44,12 @@ class Command(BaseCommand):
                 async for message in messages:
                     payload = json.loads(message.payload.decode('utf-8'))
                     device = await self.get_device(message.topic)
-                    await self.store_telemetry(device, payload)
+                    if device is not None:
+                        await self.store_telemetry(device, payload)
+                    else:
+                        self.stdout.write(
+                            self.style.ERROR(
+                                'DEBUG: message discarded'))
 
     def handle(self, *args, **options):
         client = mqtt.Client()

docker/docker-compose.dev.yml

@@ -7,15 +7,15 @@ services:
 
   broker:
     ports:
-        - "1883:1883"
-        - "9001:9001"
+        - "${CUSTOM_DOCKER_IP:-0.0.0.0}:1883:1883"
+        - "${CUSTOM_DOCKER_IP:-0.0.0.0}:9001:9001"
 
   bite:
     volumes:
       - ../bite:/srv/app/bite
     command: ["python3", "manage.py", "runserver", "0.0.0.0:8000"]
     ports:
-        - "8000:8000"
+        - "${CUSTOM_DOCKER_IP:-0.0.0.0}:8000:8000"
 
   data-migration:
     volumes:

docker/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     networks:
       - net
     ports:
-      - "123:123/udp"
+      - "${CUSTOM_DOCKER_IP:-0.0.0.0}:123:123/udp"
 
   timescale:
     <<: *service_default
@@ -41,13 +41,13 @@ services:
     networks:
       - net
     ports:
-      - "1883:1883"
+      - "${CUSTOM_DOCKER_IP:-0.0.0.0}:1883:1883"
 
   ingress:
     <<: *service_default
     image: nginx:stable-alpine
     ports:
-      - "80:80"
+      - "${CUSTOM_DOCKER_IP:-0.0.0.0}:80:80"
     networks:
       - net
     volumes:

docker/ingress/nginx.conf

@@ -51,6 +51,7 @@ http {
     
         location / {
             proxy_pass           http://bite;
+            proxy_http_version   1.1;
             proxy_set_header     Host               $http_host;
             proxy_set_header     X-Real-IP          $remote_addr;
             proxy_set_header     X-Forwarded-Host   $host:$server_port;
@@ -59,6 +60,9 @@ http {
     
             proxy_read_timeout      300;
             proxy_connect_timeout   300;
+
+            ## !- This is mandatory for IoT that do not wait for a reply -! ##
+            proxy_ignore_client_abort on;
         }
 
         location /mqtt {