Add testing suite (#18)

* Add some tests

* Add an example of NTP with encryption

* Enable TravisCI

* Run sims via Docker

* Improve simulators stage

* Final fix for travis

* Add README docs [skip ci]

.travis.yml

@@ -0,0 +1,47 @@
+language: python
+dist: bionic
+
+services:
+ - docker
+
+before_install:
+  - pip -q install -U docker-compose
+
+simulator: &simulator
+    stage: simulator
+    install:
+      - docker-compose -f docker/docker-compose.yml pull
+      - docker-compose -f docker/docker-compose.yml build
+    before_script:
+      - docker-compose -f docker/docker-compose.yml -f docker/edge/docker-compose.edge.yml up -d
+      - DOCKER_HOST='127.0.0.1:22375' docker-compose -f docker/docker-compose.yml -f docker/edge/docker-compose.edge.yml pull
+      - DOCKER_HOST='127.0.0.1:22375' docker-compose -f docker/docker-compose.yml -f docker/edge/docker-compose.edge.yml build
+    script:
+      - sleep 5  # warm-up
+      - sed -i 's/# IOT_SERIAL/IOT_SERIAL/g' docker/edge/docker-compose.modules.yml
+      - DOCKER_HOST='127.0.0.1:22375' docker-compose -f docker/edge/docker-compose.modules.yml up -d
+      - sleep 30  # collect some telemetry
+      - curl -sf http://localhost/telemetry/${IOT_TL}1234/last/
+
+jobs:
+  include:
+  - stage: build
+    before_script:
+      - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+    script:
+      - docker-compose -f docker/docker-compose.yml build
+      - docker push daniviga/bite
+      - docker push daniviga/ntpd
+    if: branch = master
+  - stage: django
+    install:
+      - docker-compose -f docker/docker-compose.yml pull
+      - docker-compose -f docker/docker-compose.yml build
+    before_script:
+      - docker-compose -f docker/docker-compose.yml up -d
+    script:
+      - docker-compose -f docker/docker-compose.yml exec bite python manage.py test
+  - <<: *simulator
+    env: IOT_TL=http
+  - <<: *simulator
+    env: IOT_TL=mqtt

README.md

@@ -1,3 +1,123 @@
 # BITE - Basic/IoT/Example
 
 Playing with IoT
+
+This project is for educational purposes only. It does not implement any authentication and/or encryption protocol, so it is not suitable for real production.
+
+## Installation
+
+### Requirements
+
+- `docker-ce` or `moby`
+- `docker-compose`
+
+The project is compatible with Docker for Windows (using Linux executors), but it is advised to directly use a minimal Linux VM instead (via the preferred hypervisor).
+
+The application stack is composed by the following components:
+
+- [Django](https://www.djangoproject.com/) with [Django REST framework](https://www.django-rest-framework.org/) web application (running via `gunicorn` in production mode)
+  - `mqtt-to-db` custom daemon to dump telemetry into the timeseries database
+  - telemetry payload is stored as json object (via PostgreSQL JSON data type)
+- [Timescale](https://www.timescale.com/) DB, a [PostgreSQL](https://www.postgresql.org/) database with a timeseries extension
+- [Mosquitto](https://mosquitto.org/) MQTT broker (see alternatives below)
+- [Nginx](http://nginx.org/) as ingress for HTTP (see alternative below)
+- [Chrony](https://chrony.tuxfamily.org/) as NTP server (with optional `MD5` encryption)
+
+## Deployment
+
+### Development
+
+```bash
+docker-compose -f docker/docker-compose.yml up -d [--scale {bite,mqtt-to-db)=N]
+```
+It exposes:
+- `http://localhost:80` (HTTP)
+- `tcp://localhost:1883` (MQTT)
+- `udp://localhost:123` (NTP)
+
+Django runs with `DEBUG = True` and `SKIP_WHITELIST = True`
+
+### Development with direct access to services
+
+```bash
+docker-compose -f docker/docker-compose.yml -f docker-compose.dev.yml up -d [--scale {bite,mqtt-to-db)=N]
+```
+It exposes:
+- `http://localhost:80` (HTTP)
+- `http://localhost:8080` (Django's `runserver`)
+- `tcp://localhost:1883` (MQTT)
+- `udp://localhost:123` (NTP)
+- `tcp://localhost:5432` (PostgreSQL/Timescale)
+
+Django runs with `DEBUG = True` and `SKIP_WHITELIST = True`
+
+### Production
+
+```bash
+docker-compose -f docker/docker-compose.yml -f docker-compose.prod.yml up -d [--scale {bite,mqtt-to-db)=N]
+```
+It exposes:
+- `http://localhost:80` (HTTP)
+- `tcp://localhost:1883` (MQTT)
+- `udp://localhost:123` (NTP)
+
+Django runs with `DEBUG = False` and `SKIP_WHITELIST = False`
+
+## Extra features
+
+The project provides multiple modules that can be combined with the fore-mentioned configurations.
+
+### Traefik
+
+To use [Traefik](https://containo.us/traefik/) instead of Nginx use:
+```bash
+docker-compose -f docker/docker-compose.yml up -f docker/ingress/docker-compose.traefik.yml -d
+```
+
+### VerneMQ
+
+A ~8x memory usage can be expected compared to Mosquitto.
+
+To use [VerneMQ](https://vernemq.com/) instead of Mosquitto use:
+```bash
+docker-compose -f docker/docker-compose.yml up -f docker/mqtt/docker-compose.vernemq.yml -d
+```
+
+### RabbitMQ
+
+RabbitMQ does provides AMQP protocol too, but ingestion on the application side is not implemented yet.
+A ~10x memory usage can be expected compared to Mosquitto.
+
+To use [RabbitMQ](https://www.rabbitmq.com/) (with the MQTT plugin enabled) instead of Mosquitto use:
+```bash
+docker-compose -f docker/docker-compose.yml up -f docker/mqtt/docker-compose.rabbitmq.yml -d
+```
+
+## EDGE gateway simulation (via dind)
+
+An EDGE gateway, with containers as modules, may be simulated via dind (docker-in-docker).
+
+### Start the EDGE
+
+```bash
+docker-compose -f docker/docker-compose.yml up -f docker/edge/docker-compose.edge.yml -d
+```
+
+### Run the modules inside the EDGE
+
+```bash
+DOCKER_HOST='127.0.0.1:22375' docker-compose -f docker-compose.modules.yml up -d [--scale {device-http,device-mqtt}=N]
+```
+
+## Ardunio
+
+A simple Arduino UNO sketch is provided in the `arduino/tempLightSensor` folder. The sketch reads temperature and light from sensors. The simple schematic is:
+
+![tempLightSensor](./arduino/tempLightSensor/tempLightSensor.svg)
+
+The sketch does require an Ethernet shield and a bunch of libraries which are available as git submodules under `arduino/libraries`.
+Be advised that some libraries (notably the NTP one) are customized.
+
+Configuration parameters are stored and retrieved from the EEPROM. An helper sketch to update the EEPROM is available under `arduino/eeprom_prog`
+
+An `ESP32` board (or similar Arduino) may be used, with some adaptions, too.

bite/api/tests.py

@@ -1,3 +1,26 @@
-from django.test import TestCase
+from django.test import TestCase, Client
+from api.models import Device, WhiteList
 
-# Create your tests here.
+
+class ApiTestCase(TestCase):
+    c = Client()
+
+    def setUp(self):
+        WhiteList.objects.create(serial='test1234')
+        Device.objects.create(serial='test1234')
+
+    def test_no_whitelist(self):
+        response = self.c.post('/api/device/subscribe/',
+                               {'serial': 'test12345'})
+        self.assertEqual(response.status_code, 400)
+
+    def test_subscribe_post(self):
+        WhiteList.objects.create(serial='test12345')
+        response = self.c.post('/api/device/subscribe/',
+                               {'serial': 'test12345'})
+        self.assertEqual(response.status_code, 201)
+
+    def test_subscribe_get(self):
+        response = self.c.get('/api/device/list/')
+        self.assertEqual(
+            response.json()[0]['serial'], 'test1234')

bite/telemetry/migrations/0008_auto_20200619_1627.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.0.7 on 2020-06-19 16:27
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+import telemetry.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('telemetry', '0007_telemetry_transport'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='telemetry',
+            name='payload',
+            field=django.contrib.postgres.fields.jsonb.JSONField(validators=[telemetry.models.telemetry_validation]),
+        ),
+    ]

bite/telemetry/models.py

@@ -1,10 +1,16 @@
 from django.db import models
 from django.core.validators import MinValueValidator
+from django.core.exceptions import ValidationError
 from django.contrib.postgres.fields import JSONField
 
 from api.models import Device
 
 
+def telemetry_validation(value):
+    if not value:
+        raise ValidationError("No telemetry has been sent")
+
+
 class Telemetry(models.Model):
     device = models.ForeignKey(Device, on_delete=models.CASCADE)
     time = models.DateTimeField(primary_key=True, auto_now_add=True)
@@ -14,7 +20,7 @@ class Telemetry(models.Model):
     clock = models.IntegerField(
         validators=[MinValueValidator(0)],
         null=True)
-    payload = JSONField()
+    payload = JSONField(validators=[telemetry_validation])
 
     class Meta:
         ordering = ['-time', 'device']

bite/telemetry/tests.py

@@ -1,3 +1,53 @@
-from django.test import TestCase
+import json
+from django.test import TestCase, Client
+from api.models import Device, WhiteList
 
-# Create your tests here.
+
+class ApiTestCase(TestCase):
+    c = Client()
+
+    payload = {
+        'id': 'sensor.server.domain',
+        'light': 434,
+        'temperature': {
+            'celsius': 27.02149,
+            'raw': 239,
+            'volts': 0.770215
+        }
+    }
+
+    telemetry = {
+        'device': 'test1234',
+        'clock': 1591194712,
+        'payload': json.dumps(payload)
+    }
+
+    def setUp(self):
+        WhiteList.objects.create(serial='test1234')
+        Device.objects.create(serial='test1234')
+
+    def test_no_device(self):
+        fake_telemetry = dict(self.telemetry)  # make a copy of the dict
+        fake_telemetry['device'] = '1234test'
+        response = self.c.post('/telemetry/', fake_telemetry)
+        self.assertEqual(response.status_code, 400)
+
+    def test_empty_telemetry(self):
+        fake_telemetry = dict(self.telemetry)  # make a copy of the dict
+        fake_telemetry['payload'] = ''
+        response = self.c.post('/telemetry/', fake_telemetry)
+        self.assertEqual(response.status_code, 400)
+
+    def test_telemetry_post(self):
+        response = self.c.post('/telemetry/', self.telemetry)
+        self.assertEqual(response.status_code, 201)
+
+    def test_telemetry_get(self):
+        response = self.c.post('/telemetry/', self.telemetry)
+        response = self.c.get('/telemetry/test1234/last/')
+        self.assertEqual(
+            response.json()['device'], 'test1234')
+        self.assertEqual(
+            response.json()['transport'], 'http')
+        self.assertJSONEqual(
+            json.dumps(response.json()['payload']), self.payload)

docker/edge/docker-compose.edge.yml

@@ -2,12 +2,11 @@ version: "3.7"
 
 services:
   edge-host:
-    <<: *service_default
     image: docker:dind
     privileged: true
     environment:
       DOCKER_TLS_CERTDIR:
-    # networks:
+    networks:
-    #   - net
+      - net
     ports:
       - "127.0.0.1:22375:2375"

docker/edge/docker-compose.modules.yml

@@ -1,8 +1,5 @@
 version: "3.7"
 
-networks:
-  localnet:
-
 x-op-service-default: &service_default
   restart: always
   init: true
@@ -15,12 +12,11 @@ services:
       context: ../simulator
     image: daniviga/bite-device-simulator
     environment:
-      IOT_HTTP: "http://192.168.10.123:8000"
+      IOT_HTTP: "http://ingress"
-      # IOT_SERIAL: "abcd1234"
+      # IOT_SERIAL: "http1234"
       # IOT_DELAY: 10
       IOT_DEBUG: 1
-    networks:
+    network_mode: "host"
-      - localnet
 
   device-mqtt:
     <<: *service_default
@@ -28,11 +24,10 @@ services:
       context: ../simulator
     image: daniviga/bite-device-simulator
     environment:
-      IOT_HTTP: "http://192.168.10.123:8000"
+      IOT_HTTP: "http://ingress"
-      IOT_MQTT: "192.168.10.123:1883"
+      IOT_MQTT: "broker:1883"
-      # IOT_SERIAL: "abcd1234"
+      # IOT_SERIAL: "mqtt1234"
       # IOT_DELAY: 10
       IOT_DEBUG: 1
     command: ["/opt/bite/device_simulator.py", "-t", "mqtt"]
-    networks:
+    network_mode: "host"
-      - localnet

docker/ntpd/Dockerfile.enc

@@ -0,0 +1,9 @@
+FROM daniviga/ntpd
+
+COPY ./chrony.keys /etc/chrony/chrony.keys
+
+RUN echo "keyfile /etc/chrony/chrony.keys" \
+        >> /etc/chrony/chrony.conf 
+
+EXPOSE 123/udp
+ENTRYPOINT ["chronyd", "-d", "-s", "-x"]

docker/ntpd/chrony.keys.sample

@@ -0,0 +1 @@
+1 MD5 HEX:D8ACA3A254EF59F83C35CEEFF6EBD2C7C67DB3D9