Merge branch 'master' into podman

Rename API to DPS and othe rimprovements

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,62 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 2 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['python']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.gitignore

@@ -132,3 +132,4 @@ dmypy.json
 
 ##
 production.py
+settings.h

.travis.yml

@@ -11,11 +11,12 @@ before_install:
 _iot-simulator: &iot-simulator
     stage: simulator
     install:
-      - docker-compose -f docker/docker-compose.yml pull
+      - docker-compose -f docker/docker-compose.yml pull -q
       - docker-compose -f docker/docker-compose.yml build
     before_script:
       - docker-compose -f docker/docker-compose.yml -f docker/edge/docker-compose.edge.yml up -d
-      - DOCKER_HOST='127.0.0.1:22375' docker-compose -f docker/edge/docker-compose.modules.yml pull
+      - sleep 30  # warm-up
+      - DOCKER_HOST='127.0.0.1:22375' docker-compose -f docker/edge/docker-compose.modules.yml pull -q
       - DOCKER_HOST='127.0.0.1:22375' docker-compose -f docker/edge/docker-compose.modules.yml build
     script:
       - sleep 5  # warm-up
@@ -36,7 +37,7 @@ jobs:
     if: branch = master
   - stage: django
     install:
-      - docker-compose -f docker/docker-compose.yml pull
+      - docker-compose -f docker/docker-compose.yml pull -q
       - docker-compose -f docker/docker-compose.yml build
     before_script:
       - docker-compose -f docker/docker-compose.yml up -d

README.md

@@ -4,7 +4,7 @@ Playing with IoT
 
 [![Build Status](https://travis-ci.com/daniviga/bite.svg?branch=master)](https://travis-ci.com/daniviga/bite)
 ![AGPLv3](./docs/.badges/agpl3.svg)
-![Python 3.8](./docs/.badges/python.svg)
+![Python 3.11](./docs/.badges/python.svg)
 ![MQTT](./docs/.badges/mqtt.svg)
 ![Moby](./docs/.badges/moby.svg)
 ![docker-compose 3.7+](./docs/.badges/docker-compose.svg)
@@ -15,29 +15,52 @@ production.
 
 ![Application Schema](./docs/application_chart.png)
 
-### Future implementations
-
-- Broker HA via [VerneMQ clustering](https://docs.vernemq.com/clustering/introduction)
-- Stream analytics via [Apache Spark](https://spark.apache.org/)
-
 ## Installation
 
 ### Requirements
 
-- `docker-ce` or `moby`
+- `moby-engine` or `podman-docker` (recommended)
 - `docker-compose`
 
 The project is compatible with Docker for Windows (using Linux executors),
 but it is advised to directly use a minimal Linux VM instead
 (via the preferred hypervisor).
 
+### Podman
+
+`podman`, with `podman-docker` is the recommended way to run BITE, in rootless mode.
+
+Requirements are:
+
+- `podman`
+- `podman-docker`
+- `catatonit`
+- `docker-compose`
+
+On Fedora 33+:
+
+```bash
+sudo dnf install -y podman podman-docker catatonit docker-compose
+```
+
+To setup `podman` run:
+
+```bash
+systemctl start --user podman.socket
+export DOCKER_HOST=unix://run/user/$UID/podman/podman.sock
+```
+
+### Application stack
+
 The application stack is composed by the following components:
 
 - [Django](https://www.djangoproject.com/) with
 [Django REST framework](https://www.django-rest-framework.org/)
 web application (running via `gunicorn` in production mode)
-  - `mqtt-to-db` custom daemon to dump telemetry into the timeseries database
+  - `dispatcher` custom daemon to dump telemetry into the Kafka queue
+  - `handler` custom daemon to dump telemetry into the timeseries database from the Kafka queue
   - telemetry payload is stored as json object (via PostgreSQL JSON data type)
+- [Kafka](https://kafka.apache.org/) broker
 - [Timescale](https://www.timescale.com/) DB,
 a [PostgreSQL](https://www.postgresql.org/) database with a timeseries extension
 - [Mosquitto](https://mosquitto.org/) MQTT broker (see alternatives below)

arduino/eeprom_prog/eeprom_prog.ino

@@ -2,7 +2,7 @@
 * vim: tabstop=2 shiftwidth=2 softtabstop=2
 *
 * BITE - A Basic/IoT/Example
-* Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+* Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 *
 * BITE is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
@@ -20,23 +20,21 @@
 
 #include <EEPROM.h>
 #include <Ethernet.h>
+#include "settings.h"
 
 #define ERASE_FIRST 0
 
 const byte mac[] = {
   0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED
 };
-const char serial[] = "abcd1234";
+const char serial[] = SERIAL;
 
 struct netConfig {
-  IPAddress address;
-  unsigned int port;
-};
-
-netConfig config = {
-  {192, 168, 10, 123},
-  80
-};
+  IPAddress iot_address = IOT_IP;
+  unsigned int iot_port = IOT_PORT;
+  IPAddress ntp_address = NTP_IP;
+  unsigned int ntp_port = NTP_PORT;
+} config;
 
 void setup() {
 

arduino/eeprom_prog/settings.h.tmpl

@@ -0,0 +1,25 @@
+/* -*- coding: utf-8 -*-
+* vim: tabstop=2 shiftwidth=2 softtabstop=2 syntax=c
+*
+* BITE - A Basic/IoT/Example
+* Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
+*
+* BITE is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* BITE is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define SERIAL "uno_1"
+#define IOT_IP {192, 168, 10, 1}
+#define IOT_PORT 80
+#define NTP_IP {192, 168, 10, 1}
+#define NTP_PORT 123

arduino/tempLightSensor/tempLightSensor.ino

@@ -2,7 +2,7 @@
 * vim: tabstop=2 shiftwidth=2 softtabstop=2
 *
 * BITE - A Basic/IoT/Example
-* Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+* Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 *
 * BITE is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
@@ -31,8 +31,6 @@
 #define TELEMETRY_DELAY  10   // second between telemetry samples
 #define AREF_VOLTAGE      3.3 // set aref voltage to 3.3v instead of default 5v
 
-char serial[9];
-
 // const String serverName = "sensor.server.domain";
 const size_t capacity = 2 * JSON_OBJECT_SIZE(3) + JSON_OBJECT_SIZE(2) + 20;
 
@@ -50,12 +48,14 @@ EthernetClient ethClient;
 PubSubClient clientMQTT(ethClient);
 
 struct netConfig {
-  IPAddress address;
-  unsigned int port;
-};
-netConfig config;
+  IPAddress iot_address;
+  unsigned int iot_port;
+  IPAddress ntp_address;
+  unsigned int ntp_port;  // not used
+} config;
 
-const String apiURL = "/api/device/subscribe/";
+char serial[9];
+const String dpsURL = "/dps/device/subscribe/";
 const String telemetryURL = "/telemetry/";
 
 void setup(void) {
@@ -63,7 +63,7 @@ void setup(void) {
 
   analogReference(EXTERNAL);
 
-  StaticJsonDocument<20> api;
+  StaticJsonDocument<20> dps;
 
   byte mac[6];
   int eeAddress = 0;
@@ -89,12 +89,17 @@ void setup(void) {
   Serial.println(Ethernet.localIP());
   Serial.println();
   Serial.print("Connecting to: ");
-  Serial.print(config.address);
+  Serial.print(config.iot_address);
   Serial.print(":");
-  Serial.println(config.port);
+  Serial.print(config.iot_port);
+  Serial.print(" every ");
+  Serial.print(TELEMETRY_DELAY);
+  Serial.println("s");
 
 #if USE_INTERNAL_NTP
-  timeClient.setPoolServerIP(config.address);
+  Serial.print("Using NTP: ");
+  Serial.println(config.ntp_address);
+  timeClient.setPoolServerIP(config.ntp_address);
 #endif
   timeClient.begin();
   if (timeClient.update()) {
@@ -105,14 +110,14 @@ void setup(void) {
   Serial.println("DEBUG: clock updated via NTP.");
 #endif
 
-  api["serial"] = serial;
-  postData(config, apiURL, api);
+  dps["serial"] = serial;
+  postData(config, dpsURL, dps);
 
   telemetry["device"] = serial;
   // payload["id"] = serverName;
 
 #if USE_MQTT
-  clientMQTT.setServer(config.address, 1883);
+  clientMQTT.setServer(config.iot_address, 1883);
 #endif
 }
 
@@ -172,14 +177,14 @@ void publishData(const netConfig &mqtt, const DynamicJsonDocument &json) {
 #endif
 
 void postData(const netConfig &postAPI, const String &URL, const DynamicJsonDocument &json) {
-  if (ethClient.connect(postAPI.address, postAPI.port)) {
+  if (ethClient.connect(postAPI.iot_address, postAPI.iot_port)) {
     ethClient.print("POST ");
     ethClient.print(URL);
     ethClient.println(" HTTP/1.1");
     ethClient.print("Host: ");
-    ethClient.print(postAPI.address);
+    ethClient.print(postAPI.iot_address);
     ethClient.print(":");
-    ethClient.println(postAPI.port);
+    ethClient.println(postAPI.iot_port);
     ethClient.println("Content-Type: application/json");
     ethClient.print("Content-Length: ");
     ethClient.println(measureJson(json));

bite/api/migrations/0002_auto_20200601_1523.py

@@ -1,21 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-01 15:23
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='device',
-            options={'ordering': ['updated_time', 'serial']},
-        ),
-        migrations.AlterModelOptions(
-            name='whitelist',
-            options={'ordering': ['serial', 'updated_time']},
-        ),
-    ]

bite/api/migrations/0003_auto_20200602_2128.py

@@ -1,19 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-02 21:28
-
-import api.models
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0002_auto_20200601_1523'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='device',
-            name='serial',
-            field=models.CharField(max_length=128, unique=True, validators=[api.models.device_validation]),
-        ),
-    ]

bite/api/migrations/0004_device_uuid.py

@@ -1,19 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-05 09:19
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0003_auto_20200602_2128'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='device',
-            name='uuid',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
-        ),
-    ]

bite/bite/__init__.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/bite/admin.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/bite/asgi.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/bite/local_settings.py.sample

@@ -0,0 +1,20 @@
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': 'bite',
+        'USER': 'bite',
+        'PASSWORD': 'password',
+        'HOST': 'localhost',
+        'PORT': '5432',
+    }
+}
+
+MQTT_BROKER = {
+    'HOST': 'localhost',
+    'PORT': '1883',
+}
+
+KAFKA_BROKER = {
+    'HOST': 'localhost',
+    'PORT': '29092',
+}

bite/bite/settings.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -61,7 +61,7 @@ INSTALLED_APPS = [
     # 'health_check.storage',
     'rest_framework',
     'bite',
-    'api',
+    'dps',
     'telemetry',
 ]
 
@@ -151,6 +151,10 @@ STATIC_URL = '/static/'
 
 STATIC_ROOT = '/srv/appdata/bite/static'
 
+REST_FRAMEWORK = {
+    'DEFAULT_AUTHENTICATION_CLASSES': []
+}
+
 SKIP_WHITELIST = True
 
 MQTT_BROKER = {
@@ -158,11 +162,17 @@ MQTT_BROKER = {
     'PORT': '1883',
 }
 
-# If no local_settings.py is availble in the current folder let's try to
-# load it from the application root
+KAFKA_BROKER = {
+    'HOST': 'kafka',
+    'PORT': '9092',
+}
+
+try:
+    from bite.local_settings import *
+except ImportError:
+    pass
+
 try:
     from bite.production import *
 except ImportError:
-    # If a local_setting.py does not exist
-    # settings in this file only will be used
     pass

bite/bite/urls.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -37,13 +37,13 @@ from django.contrib import admin
 from django.conf import settings
 from django.urls import include, path
 
-from api import urls as api_urls
+from dps import urls as dps_urls
 from telemetry import urls as telemetry_urls
 
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('ht/', include('health_check.urls')),
-    path('api/', include(api_urls)),
+    path('dps/', include(dps_urls)),
     path('telemetry/', include(telemetry_urls)),
 ]
 

bite/bite/wsgi.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/dps/__init__.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/dps/admin.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,7 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.contrib import admin
-from api.models import Device, WhiteList
+from dps.models import Device, WhiteList
 
 
 @admin.register(Device)

bite/dps/apps.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -20,5 +20,5 @@
 from django.apps import AppConfig
 
 
-class ApiConfig(AppConfig):
-    name = 'api'
+class DPSConfig(AppConfig):
+    name = 'dps'

bite/dps/migrations/0001_initial.py

@@ -1,6 +1,8 @@
-# Generated by Django 3.0.6 on 2020-06-01 14:13
+# Generated by Django 3.1.3 on 2021-03-19 08:08
 
+import dps.models
 from django.db import migrations, models
+import uuid
 
 
 class Migration(migrations.Migration):
@@ -14,20 +16,25 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Device',
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('serial', models.CharField(max_length=128, unique=True)),
+                ('serial', models.CharField(max_length=128, unique=True, validators=[dps.models.device_validation])),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                 ('creation_time', models.DateTimeField(auto_now_add=True)),
                 ('updated_time', models.DateTimeField(auto_now=True)),
             ],
+            options={
+                'ordering': ['updated_time', 'serial'],
+            },
         ),
         migrations.CreateModel(
             name='WhiteList',
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('serial', models.CharField(max_length=128, unique=True)),
+                ('serial', models.CharField(max_length=128, primary_key=True, serialize=False)),
                 ('creation_time', models.DateTimeField(auto_now_add=True)),
                 ('updated_time', models.DateTimeField(auto_now=True)),
                 ('is_published', models.BooleanField(default=True)),
             ],
+            options={
+                'ordering': ['serial', 'updated_time'],
+            },
         ),
     ]

bite/dps/migrations/__init__.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/dps/models.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -37,7 +37,7 @@ def device_validation(value):
 
 
 class WhiteList(models.Model):
-    serial = models.CharField(max_length=128, unique=True)
+    serial = models.CharField(primary_key=True, max_length=128)
     creation_time = models.DateTimeField(auto_now_add=True)
     updated_time = models.DateTimeField(auto_now=True)
     is_published = models.BooleanField(default=True)
@@ -52,7 +52,7 @@ class WhiteList(models.Model):
 class Device(models.Model):
     serial = models.CharField(max_length=128, unique=True,
                               validators=[device_validation])
-    uuid = models.UUIDField(unique=True, default=uuid.uuid4,
+    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4,
                             editable=False)
     creation_time = models.DateTimeField(auto_now_add=True)
     updated_time = models.DateTimeField(auto_now=True)

bite/dps/serializers.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,7 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from rest_framework import serializers
-from api.models import Device, device_validation
+from dps.models import Device, device_validation
 
 
 class DeviceSerializer(serializers.ModelSerializer):

bite/dps/tests.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,10 +18,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.test import TestCase, Client
-from api.models import Device, WhiteList
+from dps.models import Device, WhiteList
 
 
-class ApiTestCase(TestCase):
+class DPSTestCase(TestCase):
     c = Client()
 
     def setUp(self):
@@ -29,17 +29,17 @@ class ApiTestCase(TestCase):
         Device.objects.create(serial='test1234')
 
     def test_no_whitelist(self):
-        response = self.c.post('/api/device/subscribe/',
+        response = self.c.post('/dps/device/provision/',
                                {'serial': 'test12345'})
         self.assertEqual(response.status_code, 400)
 
-    def test_subscribe_post(self):
+    def test_provision_post(self):
         WhiteList.objects.create(serial='test12345')
-        response = self.c.post('/api/device/subscribe/',
+        response = self.c.post('/dps/device/provision/',
                                {'serial': 'test12345'})
         self.assertEqual(response.status_code, 201)
 
-    def test_subscribe_get(self):
-        response = self.c.get('/api/device/list/')
+    def test_provision_get(self):
+        response = self.c.get('/dps/device/list/')
         self.assertEqual(
             response.json()[0]['serial'], 'test1234')

bite/dps/urls.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -33,13 +33,13 @@ Including another URLconf
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.urls import path
-from api.views import APISubscribe
+from dps.views import DPS
 
 urlpatterns = [
-    path('device/subscribe/',
-         APISubscribe.as_view({'post': 'create'}),
-         name='device-subscribe'),
+    path('device/provision/',
+         DPS.as_view({'post': 'create'}),
+         name='device-provision'),
     path('device/list/',
-         APISubscribe.as_view({'get': 'list'}),
+         DPS.as_view({'get': 'list'}),
          name='device-list'),
 ]

bite/dps/views.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,10 +19,10 @@
 
 from rest_framework.viewsets import ModelViewSet
 
-from api.models import Device
-from api.serializers import DeviceSerializer
+from dps.models import Device
+from dps.serializers import DeviceSerializer
 
 
-class APISubscribe(ModelViewSet):
+class DPS(ModelViewSet):
     queryset = Device.objects.all()
     serializer_class = DeviceSerializer

bite/manage.py

@@ -3,7 +3,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/telemetry/__init__.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/telemetry/admin.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/telemetry/apps.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/telemetry/health.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/telemetry/management/commands/dispatcher.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -22,22 +22,26 @@ import asyncio
 import json
 import time
 import paho.mqtt.client as mqtt
+from kafka import KafkaProducer
+from kafka.errors import NoBrokersAvailable
 from asgiref.sync import sync_to_async
-from asyncio_mqtt import Client
+from aiomqtt import Client
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.core.exceptions import ObjectDoesNotExist
 
-from api.models import Device
-from telemetry.models import Telemetry
-
-MQTT_HOST = settings.MQTT_BROKER['HOST']
-MQTT_PORT = int(settings.MQTT_BROKER['PORT'])
+from dps.models import Device
 
 
 class Command(BaseCommand):
-    help = 'MQTT to DB deamon'
+    help = "Telemetry dispatcher"
+
+    MQTT_HOST = settings.MQTT_BROKER["HOST"]
+    MQTT_PORT = int(settings.MQTT_BROKER["PORT"])
+    KAFKA_HOST = settings.KAFKA_BROKER["HOST"]
+    KAFKA_PORT = int(settings.KAFKA_BROKER["PORT"])
+    producer = None
 
     @sync_to_async
     def get_device(self, serial):
@@ -47,40 +51,54 @@ class Command(BaseCommand):
             return None
 
     @sync_to_async
-    def store_telemetry(self, device, payload):
-        Telemetry.objects.create(
-            device=device,
-            transport='mqtt',
-            clock=payload['clock'],
-            payload=payload['payload']
-        )
+    def dispatch(self, message):
+        self.producer.send("telemetry", {"transport": "mqtt", "body": message})
 
     async def mqtt_broker(self):
-        async with Client(MQTT_HOST, port=MQTT_PORT) as client:
+        async with Client(self.MQTT_HOST, port=self.MQTT_PORT) as client:
             # use shared subscription for HA/balancing
             await client.subscribe("$share/telemetry/#")
-            async with client.unfiltered_messages() as messages:
+            async with client.messages() as messages:
                 async for message in messages:
-                    payload = json.loads(message.payload.decode('utf-8'))
                     device = await self.get_device(message.topic)
                     if device is not None:
-                        await self.store_telemetry(device, payload)
+                        message_body = json.loads(
+                            message.payload.decode("utf-8")
+                        )
+                        await self.dispatch(message_body)
                     else:
                         self.stdout.write(
-                            self.style.ERROR(
-                                'DEBUG: message discarded'))
+                            self.style.ERROR("DEBUG: message discarded")
+                        )
 
     def handle(self, *args, **options):
         client = mqtt.Client()
         while True:
             try:
-                client.connect(MQTT_HOST, MQTT_PORT)
+                client.connect(self.MQTT_HOST, self.MQTT_PORT)
                 break
             except (socket.gaierror, ConnectionRefusedError):
                 self.stdout.write(
-                    self.style.WARNING('WARNING: Broker not available'))
+                    self.style.WARNING("WARNING: MQTT broker not available")
+                )
                 time.sleep(5)
 
-        self.stdout.write(self.style.SUCCESS('INFO: Broker subscribed'))
+        while True:
+            try:
+                self.producer = KafkaProducer(
+                    bootstrap_servers="{}:{}".format(
+                        self.KAFKA_HOST, self.KAFKA_PORT
+                    ),
+                    value_serializer=lambda v: json.dumps(v).encode("utf-8"),
+                    retries=5,
+                )
+                break
+            except NoBrokersAvailable:
+                self.stdout.write(
+                    self.style.WARNING("WARNING: Kafka broker not available")
+                )
+                time.sleep(5)
+
+        self.stdout.write(self.style.SUCCESS("INFO: Brokers subscribed"))
         client.disconnect()
         asyncio.run(self.mqtt_broker())

bite/telemetry/management/commands/handler.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# BITE - A Basic/IoT/Example
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
+#
+# BITE is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# BITE is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import time
+from kafka import KafkaConsumer
+from kafka.errors import NoBrokersAvailable
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.core.exceptions import ObjectDoesNotExist
+
+from dps.models import Device
+from telemetry.models import Telemetry
+
+
+class Command(BaseCommand):
+    help = "Telemetry handler"
+
+    KAFKA_HOST = settings.KAFKA_BROKER["HOST"]
+    KAFKA_PORT = int(settings.KAFKA_BROKER["PORT"])
+
+    def get_device(self, serial):
+        try:
+            return Device.objects.get(serial=serial)
+        except ObjectDoesNotExist:
+            return None
+
+    def store_telemetry(self, transport, message):
+        Telemetry.objects.create(
+            transport=transport,
+            device=self.get_device(message["device"]),
+            clock=message["clock"],
+            payload=message["payload"],
+        )
+
+    def handle(self, *args, **options):
+        while True:
+            try:
+                consumer = KafkaConsumer(
+                    "telemetry",
+                    bootstrap_servers="{}:{}".format(
+                        self.KAFKA_HOST, self.KAFKA_PORT
+                    ),
+                    group_id="handler",
+                    value_deserializer=lambda m: json.loads(m.decode("utf8")),
+                )
+                break
+            except NoBrokersAvailable:
+                self.stdout.write(
+                    self.style.WARNING("WARNING: Kafka broker not available")
+                )
+                time.sleep(5)
+
+        self.stdout.write(self.style.SUCCESS("INFO: Kafka broker subscribed"))
+        for message in consumer:
+            self.store_telemetry(
+                message.value["transport"], message.value["body"]
+            )
+        consumer.unsuscribe()

bite/telemetry/migrations/0001_initial.py

@@ -1,8 +1,9 @@
-# Generated by Django 3.0.6 on 2020-06-01 14:45
+# Generated by Django 3.1.7 on 2021-03-25 10:55
 
-import django.contrib.postgres.fields.jsonb
+import django.core.validators
 from django.db import migrations, models
 import django.db.models.deletion
+import telemetry.models
 
 
 class Migration(migrations.Migration):
@@ -10,7 +11,7 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('api', '0001_initial'),
+        ('dps', '0001_initial'),
     ]
 
     operations = [
@@ -18,9 +19,16 @@ class Migration(migrations.Migration):
             name='Telemetry',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('time', models.DateTimeField(auto_now_add=True)),
-                ('payload', django.contrib.postgres.fields.jsonb.JSONField()),
-                ('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.Device')),
+                ('time', models.DateTimeField(auto_now_add=True, db_index=True)),
+                ('transport', models.CharField(choices=[('http', 'http'), ('mqtt', 'mqtt')], default='http', max_length=4)),
+                ('clock', models.IntegerField(null=True, validators=[django.core.validators.MinValueValidator(0)])),
+                ('payload', models.JSONField(validators=[telemetry.models.telemetry_validation])),
+                ('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dps.device')),
             ],
+            options={
+                'verbose_name_plural': 'Telemetry',
+                'ordering': ['-time', 'device'],
+                'unique_together': {('time', 'device')},
+            },
         ),
     ]

bite/telemetry/migrations/0002_auto_20200601_1557.py

@@ -1,17 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-01 15:57
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('telemetry', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='telemetry',
-            options={'ordering': ['time', 'device']},
-        ),
-    ]

bite/telemetry/migrations/0002_timescale.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.1.7 on 2021-03-25 10:55
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('telemetry', '0001_initial'),
+    ]
+
+    # Timescale requires an hyperscale table to have the field used for the
+    # partitioning ('time') to be in the any UNIQUE constraint.
+    # Because of that we have a unique_together on 'time' and 'device',
+    # however Django always adds an 'id' as PRIMARY_KEY.
+    # Django's 'id' isn't used as a foreign key, so we are dropping
+    # the contraint and simply adding an index to the 'id' column.
+    # We can now create the hypertable.
+    operations = [
+        migrations.RunSQL(
+            "ALTER TABLE telemetry_telemetry DROP CONSTRAINT telemetry_telemetry_pkey ;"  # noqa: E501
+            "CREATE INDEX telemetry_telemetry_id_idx ON telemetry_telemetry(id);"  # noqa: E501
+            "SELECT create_hypertable('telemetry_telemetry', 'time');"),
+    ]

bite/telemetry/migrations/0003_alter_telemetry_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.7 on 2021-09-03 09:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('telemetry', '0002_timescale'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='telemetry',
+            name='id',
+            field=models.AutoField(primary_key=True, serialize=False),
+        ),
+    ]

bite/telemetry/migrations/0003_auto_20200602_2131.py

@@ -1,26 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-02 21:31
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('telemetry', '0002_auto_20200601_1557'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='telemetry',
-            options={'ordering': ['time', 'device'], 'verbose_name_plural': 'Telemetry'},
-        ),
-        migrations.RemoveField(
-            model_name='telemetry',
-            name='id',
-        ),
-        migrations.AlterField(
-            model_name='telemetry',
-            name='time',
-            field=models.DateTimeField(auto_now_add=True, primary_key=True, serialize=False),
-        ),
-    ]

bite/telemetry/migrations/0004_auto_20200602_2132.py

@@ -1,15 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-02 21:32
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('telemetry', '0003_auto_20200602_2131'),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            "SELECT create_hypertable('telemetry_telemetry', 'time');"),
-    ]

bite/telemetry/migrations/0005_telemetry_clock.py

@@ -1,19 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-03 13:08
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('telemetry', '0004_auto_20200602_2132'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='telemetry',
-            name='clock',
-            field=models.IntegerField(null=True, validators=[django.core.validators.MinValueValidator(0)]),
-        ),
-    ]

bite/telemetry/migrations/0006_auto_20200603_1317.py

@@ -1,17 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-03 13:17
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('telemetry', '0005_telemetry_clock'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='telemetry',
-            options={'ordering': ['-time', 'device'], 'verbose_name_plural': 'Telemetry'},
-        ),
-    ]

bite/telemetry/migrations/0007_telemetry_transport.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.0.6 on 2020-06-08 20:07
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('telemetry', '0006_auto_20200603_1317'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='telemetry',
-            name='transport',
-            field=models.CharField(choices=[('http', 'http'), ('mqtt', 'mqtt')], default='http', max_length=4),
-        ),
-    ]

bite/telemetry/migrations/0008_auto_20200619_1627.py

@@ -1,20 +0,0 @@
-# Generated by Django 3.0.7 on 2020-06-19 16:27
-
-import django.contrib.postgres.fields.jsonb
-from django.db import migrations
-import telemetry.models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('telemetry', '0007_telemetry_transport'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='telemetry',
-            name='payload',
-            field=django.contrib.postgres.fields.jsonb.JSONField(validators=[telemetry.models.telemetry_validation]),
-        ),
-    ]

bite/telemetry/migrations/0009_auto_20200823_1340.py

@@ -1,19 +0,0 @@
-# Generated by Django 3.1 on 2020-08-23 13:40
-
-from django.db import migrations, models
-import telemetry.models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('telemetry', '0008_auto_20200619_1627'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='telemetry',
-            name='payload',
-            field=models.JSONField(validators=[telemetry.models.telemetry_validation]),
-        ),
-    ]

bite/telemetry/migrations/__init__.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/telemetry/models.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -21,7 +21,7 @@ from django.db import models
 from django.core.validators import MinValueValidator
 from django.core.exceptions import ValidationError
 
-from api.models import Device
+from dps.models import Device
 
 
 def telemetry_validation(value):
@@ -30,8 +30,9 @@ def telemetry_validation(value):
 
 
 class Telemetry(models.Model):
+    id = models.AutoField(primary_key=True)
     device = models.ForeignKey(Device, on_delete=models.CASCADE)
-    time = models.DateTimeField(primary_key=True, auto_now_add=True)
+    time = models.DateTimeField(db_index=True, auto_now_add=True)
     transport = models.CharField(max_length=4,
                                  choices=[('http', 'http'), ('mqtt', 'mqtt')],
                                  default='http')
@@ -42,6 +43,7 @@ class Telemetry(models.Model):
 
     class Meta:
         ordering = ['-time', 'device']
+        unique_together = ['time', 'device']
         verbose_name_plural = "Telemetry"
 
     def __str__(self):

bite/telemetry/serializers.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,7 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from rest_framework import serializers
-from api.models import Device
+from dps.models import Device
 from telemetry.models import Telemetry
 
 

bite/telemetry/tests.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,7 +19,7 @@
 
 import json
 from django.test import TestCase, Client
-from api.models import Device, WhiteList
+from dps.models import Device, WhiteList
 
 
 class ApiTestCase(TestCase):
@@ -69,4 +69,5 @@ class ApiTestCase(TestCase):
         self.assertEqual(
             response.json()['transport'], 'http')
         self.assertJSONEqual(
-            response.json()['payload'], self.payload)
+            json.dumps(response.json()['payload']),
+            json.dumps(self.payload))

bite/telemetry/urls.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

bite/telemetry/views.py

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,6 +19,8 @@
 
 from datetime import datetime
 from django.http import Http404
+from django.utils import timezone
+from django.utils.dateparse import parse_date, parse_datetime
 from rest_framework.views import APIView
 from rest_framework.viewsets import ModelViewSet
 
@@ -26,6 +28,7 @@ from telemetry.models import Telemetry
 from telemetry.serializers import (TelemetrySerializer,
                                    TelemetrySummarySerializer)
 from rest_framework.response import Response
+from rest_framework.exceptions import ParseError
 
 
 class TelemetryView(ModelViewSet):
@@ -66,8 +69,25 @@ class TelemetryRange(ModelViewSet):
     serializer_class = TelemetrySerializer
     lookup_field = 'device'
 
+    @staticmethod
+    def datetime_validation(time):
+        parsed_datetime = parse_datetime(time)
+        if parsed_datetime is None:
+            parsed_datetime = parse_date(time)
+            if parsed_datetime is None:
+                raise ParseError({
+                    datetime: 'Invalid date format'
+                })
+            parsed_datetime = datetime.combine(
+                parsed_datetime, datetime.min.time())
+        return timezone.make_aware(parsed_datetime)
+
     def list(self, request, device, time_from, time_to=None):
-        time_to = datetime.now() if time_to is None else time_to
+        time_from = self.datetime_validation(time_from)
+        time_to = (
+            timezone.now() if time_to is None else
+            self.datetime_validation(time_to)
+        )
         queryset = Telemetry.objects.filter(
             device__serial=device,
             time__range=[time_from, time_to])

docker/django/Dockerfile

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,20 +17,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-FROM python:3.8-alpine AS builder
+FROM python:3.11-alpine AS builder
 RUN apk update && apk add gcc musl-dev postgresql-dev \
         && pip install psycopg2-binary
 
 # ---
 
-FROM python:3.8-alpine
+FROM python:3.11-alpine
 ENV PYTHONUNBUFFERED 1
 ENV DJANGO_SETTINGS_MODULE "bite.settings"
 
 RUN apk update && apk add --no-cache postgresql-libs \
-        && wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-alpine-linux-amd64-v0.6.1.tar.gz -qO- \
+        && wget https://github.com/jwilder/dockerize/releases/download/v0.7.0/dockerize-alpine-linux-amd64-v0.7.0.tar.gz -qO- \
         | tar -xz -C /usr/local/bin
-COPY --from=builder /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/site-packages/
+COPY --from=builder /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
 COPY --chown=1000:1000 requirements.txt /srv/app/bite/requirements.txt
 RUN pip3 install --no-cache-dir -r /srv/app/bite/requirements.txt
 

docker/django/production.py.sample

@@ -2,7 +2,7 @@
 # vim: syntax=python tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

docker/docker-compose.dev.yml

@@ -2,7 +2,7 @@
 # vim: tabstop=2 shiftwidth=2 softtabstop=2
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -36,10 +36,25 @@ services:
     ports:
         - "${CUSTOM_DOCKER_IP:-0.0.0.0}:8000:8000"
 
+  kafka:
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+    ports:
+        - "${CUSTOM_DOCKER_IP:-0.0.0.0}:29092:29092"
+
   data-migration:
     volumes:
       - ../bite:/srv/app/bite
 
-  mqtt-to-db:
+  static-files:
+    volumes:
+      - ../bite:/srv/app/bite
+
+  dispatcher:
+    volumes:
+      - ../bite:/srv/app/bite
+
+  handler:
     volumes:
       - ../bite:/srv/app/bite

docker/docker-compose.prod.yml

@@ -2,7 +2,7 @@
 # vim: tabstop=2 shiftwidth=2 softtabstop=2
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -29,6 +29,10 @@ services:
     volumes:
       - ./django/production.py.sample:/srv/app/bite/bite/production.py
 
-  mqtt-to-db:
+  dispatcher:
+    volumes:
+      - ./django/production.py.sample:/srv/app/bite/bite/production.py
+
+  handler:
     volumes:
       - ./django/production.py.sample:/srv/app/bite/bite/production.py

docker/docker-compose.yml

@@ -2,7 +2,7 @@
 # vim: tabstop=2 shiftwidth=2 softtabstop=2
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -39,11 +39,11 @@ services:
     networks:
       - net
     ports:
-      - "${CUSTOM_DOCKER_IP:-0.0.0.0}:123:123/udp"
+      - "${CUSTOM_DOCKER_IP:-0.0.0.0}:1230:123/udp"  # PORT 123 CAN BE USED WHEN RUNNING AS PRIVILEGED USER
 
   timescale:
     <<: *service_default
-    image: timescale/timescaledb:latest-pg12
+    image: timescale/timescaledb:latest-pg14
     environment:
       POSTGRES_USER: "bite"
       POSTGRES_PASSWORD: "password"
@@ -54,7 +54,7 @@ services:
 
   broker:
     <<: *service_default
-    image: eclipse-mosquitto
+    image: eclipse-mosquitto:1.6
     volumes:
       - "./mqtt/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf"
     networks:
@@ -62,16 +62,40 @@ services:
     ports:
       - "${CUSTOM_DOCKER_IP:-0.0.0.0}:1883:1883"
 
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    networks:
+      - net
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  kafka:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
+    networks:
+      - net
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+
   ingress:
     <<: *service_default
     image: nginx:stable-alpine
+    environment:
+      NGINX_ENTRYPOINT_QUIET_LOGS: 1
     ports:
-      - "${CUSTOM_DOCKER_IP:-0.0.0.0}:80:80"
+      - "${CUSTOM_DOCKER_IP:-0.0.0.0}:8080:80"  # PORT 80 CAN BE USED WHEN RUNNING AS PRIVILEGED USER
     networks:
       - net
     volumes:
-      - staticdata:/srv/appdata/bite/static
-      - ./ingress/nginx.conf:/etc/nginx/nginx.conf
+      - "staticdata:/srv/appdata/bite/static:U"  # REMOVE ':U' ON MOBY/DOCKER
+      - "./ingress/nginx.conf:/etc/nginx/nginx.conf"
 
 
   bite:
@@ -99,16 +123,24 @@ services:
   static-files:
     image: daniviga/bite
     volumes:
-      - staticdata:/srv/appdata/bite/static
+      - "staticdata:/srv/appdata/bite/static:U"  # REMOVE ':U' ON MOBY/DOCKER
     command: ["python3", "manage.py", "collectstatic", "--noinput"]
 
-  mqtt-to-db:
+  dispatcher:
     <<: *service_default
     image: daniviga/bite
-    command: ["python3", "manage.py", "mqtt-to-db"]
+    command: ["python3", "manage.py", "dispatcher"]
+    networks:
+      - net
+    depends_on:
+      - broker
+
+  handler:
+    <<: *service_default
+    image: daniviga/bite
+    command: ["python3", "manage.py", "handler"]
     networks:
       - net
     depends_on:
       - data-migration
       - timescale
-      - broker

docker/edge/docker-compose.edge.yml

@@ -2,7 +2,7 @@
 # vim: tabstop=2 shiftwidth=2 softtabstop=2
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -24,7 +24,7 @@ services:
     image: docker:dind
     privileged: true
     environment:
-      DOCKER_TLS_CERTDIR:
+      DOCKER_TLS_CERTDIR: ""
     networks:
       - net
     ports:

docker/edge/docker-compose.modules.yml

@@ -2,7 +2,7 @@
 # vim: tabstop=2 shiftwidth=2 softtabstop=2
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

docker/ingress/docker-compose.traefik.yml

@@ -2,7 +2,7 @@
 # vim: tabstop=2 shiftwidth=2 softtabstop=2
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

docker/ingress/nginx.conf

@@ -2,7 +2,7 @@
 # vim: syntax=nginx tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

docker/mqtt/docker-compose.rabbitmq.yml

@@ -2,7 +2,7 @@
 # vim: tabstop=2 shiftwidth=2 softtabstop=2
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

docker/mqtt/docker-compose.vernemq.yml

@@ -2,7 +2,7 @@
 # vim: tabstop=2 shiftwidth=2 softtabstop=2
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

docker/ntpd/Dockerfile

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,9 +17,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-FROM alpine:3.9
+FROM alpine:3.18
 
-RUN apk update && apk add chrony && \
+RUN apk add --no-cache chrony && \
     chown -R chrony:chrony /var/lib/chrony
 COPY ./chrony.conf /etc/chrony/chrony.conf
 

docker/ntpd/Dockerfile.enc

@@ -2,7 +2,7 @@
 # vim: syntax=dockerfile tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

docker/simulator/Dockerfile

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-FROM python:3.8-alpine
+FROM python:3.11-alpine
 
 RUN pip3 install urllib3 paho-mqtt
 COPY ./device_simulator.py /opt/bite/device_simulator.py

docker/simulator/device_simulator.py

@@ -3,7 +3,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # BITE - A Basic/IoT/Example
-# Copyright (C) 2020 Daniele Viganò <daniele@vigano.me>
+# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
 #
 # BITE is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -29,7 +29,7 @@ import argparse
 from time import sleep
 import paho.mqtt.publish as publish
 
-DEBUG = bool(os.environ.get('IOT_DEBUG', False))
+DEBUG = bool(os.environ.get("IOT_DEBUG", False))
 http = urllib3.PoolManager()
 
 
@@ -39,15 +39,16 @@ def post_json(endpoint, url, data):
     if DEBUG:
         print(json_data)
 
-    encoded_data = json_data.encode('utf8')
+    encoded_data = json_data.encode("utf8")
 
     while True:
         try:
             r = http.request(
-                'POST',
+                "POST",
                 endpoint + url,
                 body=encoded_data,
-                headers={'content-type': 'application/json'})
+                headers={"content-type": "application/json"},
+            )
             return r
         except urllib3.exceptions.MaxRetryError:
             pass
@@ -57,74 +58,89 @@ def post_json(endpoint, url, data):
 
 def publish_json(transport, endpoint, data):
     json_data = json.dumps(data)
-    serial = data['device']
+    serial = data["device"]
 
     if DEBUG:
         print(json_data)
 
-    encoded_data = json_data.encode('utf8')
+    encoded_data = json_data.encode("utf8")
 
     publish.single(
         topic=serial,
         payload=encoded_data,
-        hostname=endpoint.split(':')[0],
-        port=int(endpoint.split(':')[1]),
+        hostname=endpoint.split(":")[0],
+        port=int(endpoint.split(":")[1]),
         client_id=serial,
-        transport=('websockets' if transport == 'ws' else 'tcp'),
+        transport=("websockets" if transport == "ws" else "tcp"),
         # auth=auth FIXME
     )
 
 
 def main():
-    parser = argparse.ArgumentParser(
-        description='IoT simulator oprions')
+    parser = argparse.ArgumentParser(description="IoT simulator oprions")
 
-    parser.add_argument('-e', '--endpoint',
-                        default=os.environ.get('IOT_HTTP',
-                                               'http://127.0.0.1:8000'),
-                        help='IoT HTTP endpoint')
-    parser.add_argument('-m', '--mqtt',
-                        default=os.environ.get('IOT_MQTT',
-                                               '127.0.0.1:1883'),
-                        help='IoT MQTT endpoint')
-    parser.add_argument('-t', '--transport',
-                        choices=['mqtt', 'ws', 'http'],
-                        default=os.environ.get('IOT_TL', 'http'),
-                        help='IoT transport layer')
-    parser.add_argument('-s', '--serial',
-                        default=os.environ.get('IOT_SERIAL'),
-                        help='IoT device serial number')
-    parser.add_argument('-d', '--delay', metavar='s', type=int,
-                        default=os.environ.get('IOT_DELAY', 10),
-                        help='Delay between requests')
+    parser.add_argument(
+        "-e",
+        "--endpoint",
+        default=os.environ.get("IOT_HTTP", "http://127.0.0.1:8000"),
+        help="IoT HTTP endpoint",
+    )
+    parser.add_argument(
+        "-m",
+        "--mqtt",
+        default=os.environ.get("IOT_MQTT", "127.0.0.1:1883"),
+        help="IoT MQTT endpoint",
+    )
+    parser.add_argument(
+        "-t",
+        "--transport",
+        choices=["mqtt", "ws", "http"],
+        default=os.environ.get("IOT_TL", "http"),
+        help="IoT transport layer",
+    )
+    parser.add_argument(
+        "-s",
+        "--serial",
+        default=os.environ.get("IOT_SERIAL"),
+        help="IoT device serial number",
+    )
+    parser.add_argument(
+        "-d",
+        "--delay",
+        metavar="s",
+        type=float,
+        default=os.environ.get("IOT_DELAY", 10),
+        help="Delay between requests",
+    )
     args = parser.parse_args()
 
-    subscribe = '/api/device/subscribe/'
-    telemetry = '/telemetry/'
+    dps = "/dps/device/provision/"
+    telemetry = "/telemetry/"
 
     if args.serial is None:
-        args.serial = ''.join(
-            random.choices(string.ascii_lowercase + string.digits, k=8))
+        args.serial = "".join(
+            random.choices(string.ascii_lowercase + string.digits, k=8)
+        )
 
-    data = {'serial': args.serial}
-    post_json(args.endpoint, subscribe, data)
+    data = {"serial": args.serial}
+    post_json(args.endpoint, dps, data)
 
     while True:
         data = {
-            'device': args.serial,
-            'clock': int(datetime.datetime.now().timestamp()),
+            "device": args.serial,
+            "clock": int(datetime.datetime.now().timestamp()),
         }
         payload = {
-            'id': 'device_simulator',
-            'light': random.randint(300, 500),
-            'temperature': {
-                'celsius': round(random.uniform(20, 28), 1)}
+            "id": "device_simulator",
+            "light": random.randint(300, 500),
+            "temperature": {"celsius": round(random.uniform(20, 28), 1)},
         }
-        if args.transport == 'http':
-            post_json(args.endpoint, telemetry, {**data, 'payload': payload})
-        elif args.transport in ('mqtt', 'ws'):
+        if args.transport == "http":
+            post_json(args.endpoint, telemetry, {**data, "payload": payload})
+        elif args.transport in ("mqtt", "ws"):
             publish_json(
-                args.transport, args.mqtt, {**data, 'payload': payload})
+                args.transport, args.mqtt, {**data, "payload": payload}
+            )
         else:
             raise NotImplementedError
         sleep(args.delay)

docs/.badges/python.svg

@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="93.0" height="20"><linearGradient id="smooth" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="round"><rect width="93.0" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#round)"><rect width="65.5" height="20" fill="#555"/><rect x="65.5" width="27.5" height="20" fill="#007ec6"/><rect width="93.0" height="20" fill="url(#smooth)"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"><image x="5" y="3" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxMDAgMTAwIj4KICA8ZGVmcz4KICAgIDxsaW5lYXJHcmFkaWVudCBpZD0icHlZZWxsb3ciIGdyYWRpZW50VHJhbnNmb3JtPSJyb3RhdGUoNDUpIj4KICAgICAgPHN0b3Agc3RvcC1jb2xvcj0iI2ZlNSIgb2Zmc2V0PSIwLjYiLz4KICAgICAgPHN0b3Agc3RvcC1jb2xvcj0iI2RhMSIgb2Zmc2V0PSIxIi8+CiAgICA8L2xpbmVhckdyYWRpZW50PgogICAgPGxpbmVhckdyYWRpZW50IGlkPSJweUJsdWUiIGdyYWRpZW50VHJhbnNmb3JtPSJyb3RhdGUoNDUpIj4KICAgICAgPHN0b3Agc3RvcC1jb2xvcj0iIzY5ZiIgb2Zmc2V0PSIwLjQiLz4KICAgICAgPHN0b3Agc3RvcC1jb2xvcj0iIzQ2OCIgb2Zmc2V0PSIxIi8+CiAgICA8L2xpbmVhckdyYWRpZW50PgogIDwvZGVmcz4KCiAgPHBhdGggZD0iTTI3LDE2YzAtNyw5LTEzLDI0LTEzYzE1LDAsMjMsNiwyMywxM2wwLDIyYzAsNy01LDEyLTExLDEybC0yNCwwYy04LDAtMTQsNi0xNCwxNWwwLDEwbC05LDBjLTgsMC0xMy05LTEzLTI0YzAtMTQsNS0yMywxMy0yM2wzNSwwbDAtM2wtMjQsMGwwLTlsMCwweiBNODgsNTB2MSIgZmlsbD0idXJsKCNweUJsdWUpIi8+CiAgPHBhdGggZD0iTTc0LDg3YzAsNy04LDEzLTIzLDEzYy0xNSwwLTI0LTYtMjQtMTNsMC0yMmMwLTcsNi0xMiwxMi0xMmwyNCwwYzgsMCwxNC03LDE0LTE1bDAtMTBsOSwwYzcsMCwxMyw5LDEzLDIzYzAsMTUtNiwyNC0xMywyNGwtMzUsMGwwLDNsMjMsMGwwLDlsMCwweiBNMTQwLDUwdjEiIGZpbGw9InVybCgjcHlZZWxsb3cpIi8+CgogIDxjaXJjbGUgcj0iNCIgY3g9IjY0IiBjeT0iODgiIGZpbGw9IiNGRkYiLz4KICA8Y2lyY2xlIHI9IjQiIGN4PSIzNyIgY3k9IjE1IiBmaWxsPSIjRkZGIi8+Cjwvc3ZnPgo="/><text x="422.5" y="150" fill="#010101" fill-opacity=".3" transform="scale(0.1)" textLength="385.0" lengthAdjust="spacing">python</text><text x="422.5" y="140" transform="scale(0.1)" textLength="385.0" lengthAdjust="spacing">python</text><text x="782.5" y="150" fill="#010101" fill-opacity=".3" transform="scale(0.1)" textLength="175.0" lengthAdjust="spacing">3.8</text><text x="782.5" y="140" transform="scale(0.1)" textLength="175.0" lengthAdjust="spacing">3.8</text></g></svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="93.0" height="20"><linearGradient id="smooth" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="round"><rect width="93.0" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#round)"><rect width="65.5" height="20" fill="#555"/><rect x="65.5" width="27.5" height="20" fill="#007ec6"/><rect width="93.0" height="20" fill="url(#smooth)"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"><image x="5" y="3" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxMDAgMTAwIj4KICA8ZGVmcz4KICAgIDxsaW5lYXJHcmFkaWVudCBpZD0icHlZZWxsb3ciIGdyYWRpZW50VHJhbnNmb3JtPSJyb3RhdGUoNDUpIj4KICAgICAgPHN0b3Agc3RvcC1jb2xvcj0iI2ZlNSIgb2Zmc2V0PSIwLjYiLz4KICAgICAgPHN0b3Agc3RvcC1jb2xvcj0iI2RhMSIgb2Zmc2V0PSIxIi8+CiAgICA8L2xpbmVhckdyYWRpZW50PgogICAgPGxpbmVhckdyYWRpZW50IGlkPSJweUJsdWUiIGdyYWRpZW50VHJhbnNmb3JtPSJyb3RhdGUoNDUpIj4KICAgICAgPHN0b3Agc3RvcC1jb2xvcj0iIzY5ZiIgb2Zmc2V0PSIwLjQiLz4KICAgICAgPHN0b3Agc3RvcC1jb2xvcj0iIzQ2OCIgb2Zmc2V0PSIxIi8+CiAgICA8L2xpbmVhckdyYWRpZW50PgogIDwvZGVmcz4KCiAgPHBhdGggZD0iTTI3LDE2YzAtNyw5LTEzLDI0LTEzYzE1LDAsMjMsNiwyMywxM2wwLDIyYzAsNy01LDEyLTExLDEybC0yNCwwYy04LDAtMTQsNi0xNCwxNWwwLDEwbC05LDBjLTgsMC0xMy05LTEzLTI0YzAtMTQsNS0yMywxMy0yM2wzNSwwbDAtM2wtMjQsMGwwLTlsMCwweiBNODgsNTB2MSIgZmlsbD0idXJsKCNweUJsdWUpIi8+CiAgPHBhdGggZD0iTTc0LDg3YzAsNy04LDEzLTIzLDEzYy0xNSwwLTI0LTYtMjQtMTNsMC0yMmMwLTcsNi0xMiwxMi0xMmwyNCwwYzgsMCwxNC03LDE0LTE1bDAtMTBsOSwwYzcsMCwxMyw5LDEzLDIzYzAsMTUtNiwyNC0xMywyNGwtMzUsMGwwLDNsMjMsMGwwLDlsMCwweiBNMTQwLDUwdjEiIGZpbGw9InVybCgjcHlZZWxsb3cpIi8+CgogIDxjaXJjbGUgcj0iNCIgY3g9IjY0IiBjeT0iODgiIGZpbGw9IiNGRkYiLz4KICA8Y2lyY2xlIHI9IjQiIGN4PSIzNyIgY3k9IjE1IiBmaWxsPSIjRkZGIi8+Cjwvc3ZnPgo="/><text x="422.5" y="150" fill="#010101" fill-opacity=".3" transform="scale(0.1)" textLength="385.0" lengthAdjust="spacing">python</text><text x="422.5" y="140" transform="scale(0.1)" textLength="385.0" lengthAdjust="spacing">python</text><text x="782.5" y="150" fill="#010101" fill-opacity=".3" transform="scale(0.1)" textLength="175.0" lengthAdjust="spacing">3.11</text><text x="782.5" y="140" transform="scale(0.1)" textLength="175.0" lengthAdjust="spacing">3.11</text></g></svg>

esp32/libraries/ArduinoJson

@@ -0,0 +1 @@
+../../arduino/libraries/ArduinoJson

esp32/libraries/Ethernet

@@ -0,0 +1 @@
+../../arduino/libraries/Ethernet

esp32/libraries/NTPClient

@@ -0,0 +1 @@
+../../arduino/libraries/NTPClient

esp32/libraries/pubsubclient

@@ -0,0 +1 @@
+../../arduino/libraries/pubsubclient

esp32/notebook/ESP32_1 RSSI-HALL.ipynb

@@ -0,0 +1,753 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "endless-short",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# -*- coding: utf-8 -*-\n",
+    "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n",
+    "#\n",
+    "# BITE - A Basic/IoT/Example\n",
+    "# Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>\n",
+    "#\n",
+    "# BITE is free software: you can redistribute it and/or modify\n",
+    "# it under the terms of the GNU Affero General Public License as published by\n",
+    "# the Free Software Foundation, either version 3 of the License, or\n",
+    "# (at your option) any later version.\n",
+    "#\n",
+    "# BITE is distributed in the hope that it will be useful,\n",
+    "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n",
+    "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n",
+    "# GNU Affero General Public License for more details.\n",
+    "#\n",
+    "# You should have received a copy of the GNU Affero General Public License\n",
+    "# along with this program.  If not, see <http://www.gnu.org/licenses/>.\n",
+    "\n",
+    "import requests\n",
+    "import pandas as pd\n",
+    "import ipywidgets as widgets\n",
+    "import plotly.graph_objects as go\n",
+    "\n",
+    "from datetime import date, datetime\n",
+    "from IPython.display import display"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "running-darwin",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "serial = widgets.Text(\n",
+    "    value='esp32_1',\n",
+    "    placeholder='Type something',\n",
+    "    description='Serial:',\n",
+    "    disabled=False\n",
+    ")\n",
+    "date_from = widgets.DatePicker(\n",
+    "    description='From Date',\n",
+    "    disabled=False,\n",
+    "    value=date.today()\n",
+    ")\n",
+    "date_to = widgets.DatePicker(\n",
+    "    description='To Date',\n",
+    "    disabled=False,\n",
+    "    value=date.today()\n",
+    ")\n",
+    "display(serial)\n",
+    "display(date_from)\n",
+    "display(date_to)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "designing-milwaukee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "date_from = datetime.combine(date_from.value, datetime.min.time())\n",
+    "date_to = datetime.combine(date_to.value, datetime.max.time())\n",
+    "\n",
+    "try:\n",
+    "    r = requests.get(\"http://localhost/telemetry/{serial}/{date_from}/{date_to}/\".format(\n",
+    "        serial=serial.value,\n",
+    "        date_from=date_from,\n",
+    "        date_to=date_to\n",
+    "    ))\n",
+    "    \n",
+    "except requests.exceptions.RequestException as e:\n",
+    "    raise SystemExit(e)\n",
+    "\n",
+    "r.raise_for_status()\n",
+    "\n",
+    "df = pd.json_normalize(r.json())\n",
+    "if 'time' in df:\n",
+    "    index = pd.to_datetime(df['time'])\n",
+    "    df = df.set_index(index)\n",
+    "    df = df.resample('5min').agg(['min', 'max', 'mean'])\n",
+    "else:\n",
+    "    print('No data to compute')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "related-place",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig = go.Figure([go.Scatter(x=df.index, y=df['payload.wifi-rssi']['max'], showlegend=False),\n",
+    "                 go.Scatter(x=df.index, y=df['payload.wifi-rssi']['mean'], name=\"WiFI RSSI\"),\n",
+    "                 go.Scatter(x=df.index, y=df['payload.wifi-rssi']['min'], showlegend=False),\n",
+    "                 go.Scatter(x=df.index, y=df['payload.hall']['max'], showlegend=False),\n",
+    "                 go.Scatter(x=df.index, y=df['payload.hall']['mean'], name=\"Hall effect\"),\n",
+    "                 go.Scatter(x=df.index, y=df['payload.hall']['min'], showlegend=False)])\n",
+    "fig.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "interested-silence",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.2"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {
+     "012db04e6f6143409f22b63f147e4e2c": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "From Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_0e7d139b30d840439d6e7b5e1b424a37",
+       "style": "IPY_MODEL_73e5b69afa3a4bfda9ebba619d36f926",
+       "value": {
+        "date": 24,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "05f9de03e2354005acb4b505e0806e8d": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "07cc459eceac47dcb8b6199479039f69": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "0e7d139b30d840439d6e7b5e1b424a37": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "10de1b7c78e54dd98c5a6a43c75e44e8": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "To Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_f45d49360f3b4026a5b14087ea1f9469",
+       "style": "IPY_MODEL_ccca0316a69845e3830548986969b296",
+       "value": {
+        "date": 25,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "14937ad3e04a456f838521bba3bfd72e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "19a47aab423c45c994100a24e7f55b73": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "TextModel",
+      "state": {
+       "description": "Serial:",
+       "layout": "IPY_MODEL_23ae3f01b8e54360a2c515e52013500f",
+       "placeholder": "Type something",
+       "style": "IPY_MODEL_14937ad3e04a456f838521bba3bfd72e",
+       "value": "esp32_1"
+      }
+     },
+     "1ca6e711a3954797bd8c0fa76255509c": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "1edc09258c11486ab4a2069aa7258f45": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "2231042a87594bf28113cd8072e1f220": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "23ae3f01b8e54360a2c515e52013500f": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "27ddcc76d0954ca8bb786e18f6add137": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "2830c0db32394a31900947bab8b2dd35": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "291f93e47e9b49ef8f8eb420f3f4a47c": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "From Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_fdcc9a1e68c34653abff829b95bdf6ff",
+       "style": "IPY_MODEL_d89b526fd3124032b7dabbc10ee599f0",
+       "value": {
+        "date": 24,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "340e5e36dbba42d5b60d13e85bedf3a4": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "35ffe994f0de4f948e5d9f9c16c68ada": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "TextModel",
+      "state": {
+       "description": "Serial:",
+       "layout": "IPY_MODEL_c2defc40ac02405393c4db522a3ff8a3",
+       "placeholder": "Type something",
+       "style": "IPY_MODEL_bd2bdbb620984299acfbe96a4e6f9a6e",
+       "value": "esp32_1"
+      }
+     },
+     "3f6b191dc1d647ebb4d5ba83ee07548c": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "41974684122d4b40be5bc0a8db64ff9e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "From Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_fc6a1763ccf040b9b12e3e6384d9a5d2",
+       "style": "IPY_MODEL_2830c0db32394a31900947bab8b2dd35",
+       "value": {
+        "date": 24,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "48166986d3f94f78b1b41bb8e45f905f": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "547f5bcad26f43afa93b827e89803237": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "57c8140445e94f0686197d04f5b37a05": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "To Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_a5cadb7e8bb747b5879b6fda525c0211",
+       "style": "IPY_MODEL_07cc459eceac47dcb8b6199479039f69",
+       "value": {
+        "date": 25,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "5bd7fed916b946b1a5da6d2d7cd3229e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "TextModel",
+      "state": {
+       "description": "Serial:",
+       "layout": "IPY_MODEL_d872e99186584085b5a052778c14eabf",
+       "placeholder": "Type something",
+       "style": "IPY_MODEL_340e5e36dbba42d5b60d13e85bedf3a4",
+       "value": "esp32_1"
+      }
+     },
+     "5fe18cd07fb244b7a18f9c6a6e9cbd47": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "From Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_be1a12eef0d64b37b9612320694d96f0",
+       "style": "IPY_MODEL_48166986d3f94f78b1b41bb8e45f905f",
+       "value": {
+        "date": 24,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "6ccb0bd041e047d59b1ca9a26b867d53": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "TextModel",
+      "state": {
+       "description": "Serial:",
+       "layout": "IPY_MODEL_b4d7d72ddeb2440a98898417e787c1b4",
+       "placeholder": "Type something",
+       "style": "IPY_MODEL_818fd7e0fd414036ad7025a79d0481fa",
+       "value": "esp32_1"
+      }
+     },
+     "6f1673f654b14011aa8b7aaf315b77e8": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "73e5b69afa3a4bfda9ebba619d36f926": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "796774d19b9843178e87ea97c1efd69f": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "798e515187274617bc65df9fc4905065": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "7f0bed81c5914082886e91ee4d1cb19e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "810137e6614b4e94a008e270fbac3df3": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "To Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_be930797fbab4501865a8d753fc82881",
+       "style": "IPY_MODEL_9f84286df9b54fd98a4ad9d1c373b1b2",
+       "value": {
+        "date": 25,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "8125ad074c344e2a858ffb86f08e7078": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "818fd7e0fd414036ad7025a79d0481fa": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "957186e22f244091a89a12f460a1bfb6": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "To Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_27ddcc76d0954ca8bb786e18f6add137",
+       "style": "IPY_MODEL_ea3710cdbac84d95a5698eed13342149",
+       "value": {
+        "date": 25,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "97e421f821cc4432a90040b07a2e3695": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "To Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_1edc09258c11486ab4a2069aa7258f45",
+       "style": "IPY_MODEL_6f1673f654b14011aa8b7aaf315b77e8",
+       "value": {
+        "date": 25,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "9f84286df9b54fd98a4ad9d1c373b1b2": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "a5cadb7e8bb747b5879b6fda525c0211": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "af53446e2d6c40799a006b23836160f0": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "From Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_2231042a87594bf28113cd8072e1f220",
+       "style": "IPY_MODEL_1ca6e711a3954797bd8c0fa76255509c",
+       "value": {
+        "date": 24,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "b4d7d72ddeb2440a98898417e787c1b4": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "b81224cb976f4b20a769c9bdd191db75": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "From Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_05f9de03e2354005acb4b505e0806e8d",
+       "style": "IPY_MODEL_3f6b191dc1d647ebb4d5ba83ee07548c",
+       "value": {
+        "date": 24,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "bd2bdbb620984299acfbe96a4e6f9a6e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "be1a12eef0d64b37b9612320694d96f0": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "be930797fbab4501865a8d753fc82881": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "c2defc40ac02405393c4db522a3ff8a3": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "c33fb64e62804915a92531a42e02ab6b": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "TextModel",
+      "state": {
+       "description": "Serial:",
+       "layout": "IPY_MODEL_f0c4e8f4dd704cc1915c148293953e2e",
+       "placeholder": "Type something",
+       "style": "IPY_MODEL_c3fb12046c5748a1a4416805e2a2e316",
+       "value": "esp32_1"
+      }
+     },
+     "c3fb12046c5748a1a4416805e2a2e316": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "ca386f6b0c3b4599bf7a3258a3dd1aff": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "ccca0316a69845e3830548986969b296": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "d1d6e852a7dd4fb3b12c266b755bf111": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "d32b6b4bb576460484d567b610f76c44": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "To Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_d1d6e852a7dd4fb3b12c266b755bf111",
+       "style": "IPY_MODEL_7f0bed81c5914082886e91ee4d1cb19e",
+       "value": {
+        "date": 25,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "d872e99186584085b5a052778c14eabf": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "d89b526fd3124032b7dabbc10ee599f0": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "dc4dffcc7ea94b85b77f140461468811": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "dc520429d99d49c18fa060ce1e95263c": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "dfce0d9a690546e19c7b354d4e13f4f1": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "ea3710cdbac84d95a5698eed13342149": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DescriptionStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "f0c4e8f4dd704cc1915c148293953e2e": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "f45d49360f3b4026a5b14087ea1f9469": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "fbbf425e2b204570a64e24eb47f7055f": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "TextModel",
+      "state": {
+       "description": "Serial:",
+       "layout": "IPY_MODEL_547f5bcad26f43afa93b827e89803237",
+       "placeholder": "Type something",
+       "style": "IPY_MODEL_dfce0d9a690546e19c7b354d4e13f4f1",
+       "value": "esp32_1"
+      }
+     },
+     "fc6a1763ccf040b9b12e3e6384d9a5d2": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "fc9c348988ec4ce0874decc937da8078": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "TextModel",
+      "state": {
+       "description": "Serial:",
+       "layout": "IPY_MODEL_8125ad074c344e2a858ffb86f08e7078",
+       "placeholder": "Type something",
+       "style": "IPY_MODEL_796774d19b9843178e87ea97c1efd69f",
+       "value": "esp32_1"
+      }
+     },
+     "fcc822c898014962804e2fa52d397b5e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "From Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_dc520429d99d49c18fa060ce1e95263c",
+       "style": "IPY_MODEL_dc4dffcc7ea94b85b77f140461468811",
+       "value": {
+        "date": 24,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     },
+     "fdcc9a1e68c34653abff829b95bdf6ff": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "1.2.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "fe9cfdeb23104476a3bbc9afaeaf9838": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "1.5.0",
+      "model_name": "DatePickerModel",
+      "state": {
+       "description": "To Date",
+       "disabled": false,
+       "layout": "IPY_MODEL_798e515187274617bc65df9fc4905065",
+       "style": "IPY_MODEL_ca386f6b0c3b4599bf7a3258a3dd1aff",
+       "value": {
+        "date": 25,
+        "month": 2,
+        "year": 2021
+       }
+      }
+     }
+    },
+    "version_major": 2,
+    "version_minor": 0
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

esp32/notebook/requirements.txt

@@ -0,0 +1,4 @@
+requests
+jupyter
+pandas
+ipywidgets

esp32/nvs_prog/nvs_prog.ino

@@ -0,0 +1,77 @@
+/* -*- coding: utf-8 -*-
+* vim: tabstop=2 shiftwidth=2 softtabstop=2
+*
+* BITE - A Basic/IoT/Example
+* Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
+*
+* BITE is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* BITE is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <Preferences.h>
+#include <Ethernet.h>
+#include "settings.h"
+
+#define ERASE_FIRST 1
+
+Preferences preferences;
+const char* serial = SERIAL;
+
+const char* ssid = SECRET_SSID;
+const char* password = SECRET_PASSWORD;
+
+struct netConfig {
+  IPAddress iot_address = IOT_IP;
+  unsigned int iot_port = IOT_PORT;
+  IPAddress ntp_address = NTP_IP;
+  unsigned int ntp_port = NTP_PORT;
+} config;
+
+void setup() {
+
+  Serial.begin(115200);
+  while (!Serial) {
+    ; // wait for serial port to connect. Needed for native USB port only
+  }
+
+  preferences.begin("iot", false);
+#if ERASE_FIRST
+  Serial.println("Erasing IoT data");
+  preferences.clear();
+#endif
+  Serial.print("Writing IoT data");
+  preferences.putString("serial", serial);
+  Serial.print(".");
+  Serial.println(".");
+
+  preferences.putBytes("config", &config, sizeof(config));
+  Serial.println("Committing...");
+  preferences.end();
+  Serial.println("IoT data written!");
+
+  preferences.begin("wifi", false);
+#if ERASE_FIRST
+  Serial.println("Erasing WiFI data");
+  preferences.clear();
+#endif
+  Serial.println("Writing WiFi data");
+  preferences.putString("ssid", ssid);
+  preferences.putString("password", password);
+  Serial.println("Committing...");
+  preferences.end();
+  Serial.println("WiFi data written!");
+}
+
+void loop() {
+  /* Empty loop */
+}

esp32/nvs_prog/settings.h.tmpl

@@ -0,0 +1,27 @@
+/* -*- coding: utf-8 -*-
+* vim: tabstop=2 shiftwidth=2 softtabstop=2 syntax=c
+*
+* BITE - A Basic/IoT/Example
+* Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
+*
+* BITE is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* BITE is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define SERIAL "<fillme>"
+#define IOT_IP {192, 168, 10, 1}
+#define IOT_PORT 80
+#define NTP_IP {192, 168, 10, 1}
+#define NTP_PORT 123
+#define SECRET_SSID "<fillme>"
+#define SECRET_PASSWORD "<fillme>"

esp32/rssiHall/rssiHall.ino

@@ -0,0 +1,201 @@
+/* -*- coding: utf-8 -*-
+* vim: tabstop=2 shiftwidth=2 softtabstop=2
+*
+* BITE - A Basic/IoT/Example
+* Copyright (C) 2020-2021 Daniele Viganò <daniele@vigano.me>
+*
+* BITE is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* BITE is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <Preferences.h>
+#include <WiFi.h>
+#include <WiFiUdp.h>
+#include <PubSubClient.h>
+#include <NTPClient.h>
+#include <ArduinoJson.h>
+
+#define DEBUG_TO_SERIAL   1   // debug on serial port
+#define USE_MQTT          1   // use mqtt protocol instead of http post
+#define USE_INTERNAL_NTP  1   // use default ntp server or the internal one
+#define TELEMETRY_DELAY  10   // second between telemetry samples
+
+// const String serverName = "sensor.server.domain";
+const size_t capacity = 2 * JSON_OBJECT_SIZE(3) + JSON_OBJECT_SIZE(2) + 20;
+
+Preferences preferences;
+
+DynamicJsonDocument telemetry(capacity);
+JsonObject payload = telemetry.createNestedObject("payload");
+
+unsigned int counter = 0;
+
+WiFiUDP ntpUDP;
+NTPClient timeClient(ntpUDP);
+bool NTPValid = false;
+
+WiFiClient ethClient;
+PubSubClient clientMQTT(ethClient);
+
+struct netConfig {
+  IPAddress iot_address;
+  unsigned int iot_port;
+  IPAddress ntp_address;
+  unsigned int ntp_port;  // not used
+} config;
+
+char* serial;
+const String dpsURL = "/dps/device/subscribe/";
+const String telemetryURL = "/telemetry/";
+
+void setup(void) {
+  Serial.begin(115200);
+
+  StaticJsonDocument<64> dps;
+
+  preferences.begin("iot");
+  // Get the serial number from flash
+  serial = strdup(preferences.getString("serial").c_str());
+
+  // Get network configuration
+  size_t _len = preferences.getBytesLength("config");
+  char _buffer[_len];
+  preferences.getBytes("config", &_buffer, _len);
+  memcpy(&config, _buffer, _len);
+  preferences.end();
+
+  // Get WiFi parameters
+  preferences.begin("wifi");
+  const char* _ssid = strdup(preferences.getString("ssid").c_str());
+  const char* _password = strdup(preferences.getString("password").c_str());
+  preferences.end();
+
+  Serial.print("Starting connecting WiFi to ");
+  Serial.print(_ssid);
+  delay(10);
+  WiFi.begin(_ssid, _password);
+  while (WiFi.status() != WL_CONNECTED) {
+    delay(500);
+    Serial.print(".");
+  }
+  Serial.println("\nWiFi connected");
+
+  Serial.print("IoT #");
+  Serial.print(serial);
+  Serial.print(" at address: ");
+  Serial.println(WiFi.localIP());
+  Serial.println();
+  Serial.print("Connecting to: ");
+  Serial.print(config.iot_address.toString());
+  Serial.print(":");
+  Serial.print(config.iot_port);
+  Serial.print(" every ");
+  Serial.print(TELEMETRY_DELAY);
+  Serial.println("s");
+
+#if USE_INTERNAL_NTP
+  Serial.print("Using NTP: ");
+  Serial.println(config.ntp_address.toString());
+  timeClient.setPoolServerIP(config.ntp_address);
+#endif
+  timeClient.begin();
+  if (timeClient.update()) {
+    NTPValid = true;
+  }
+
+#if DEBUG_TO_SERIAL
+  Serial.println("DEBUG: clock updated via NTP.");
+#endif
+
+  dps["serial"] = serial;
+  postData(config, dpsURL, dps);
+
+  telemetry["device"] = serial;
+  // payload["id"] = serverName;
+
+#if USE_MQTT
+  clientMQTT.setServer(config.iot_address, 1883);
+#endif
+}
+
+void loop(void) {
+  const int postDelay = TELEMETRY_DELAY * 1000;
+
+  if (NTPValid) {
+    telemetry["clock"] = timeClient.getEpochTime();
+  } else {
+    telemetry["clock"] = NULL; // converted into 0
+  }
+
+  payload["hall"] = hallRead();
+  payload["wifi-rssi"] = WiFi.RSSI();
+
+#if USE_MQTT
+  publishData(config, telemetry);
+#else
+  postData(config, telemetryURL, telemetry);
+#endif
+
+  if (counter == 6 * 120) { // Update clock every 6 times * 10 sec * 120 minutes = 2 hrs
+    timeClient.update();
+    counter = 0;
+#if DEBUG_TO_SERIAL
+    Serial.println("DEBUG: clock updated via NTP.");
+#endif
+  } else {
+    counter++;
+  }
+
+  delay(postDelay);
+}
+
+#if USE_MQTT
+void publishData(const netConfig &mqtt, const DynamicJsonDocument &json) {
+  if (clientMQTT.connect(serial)) {
+    char buffer[256];
+    serializeJson(json, buffer);
+    clientMQTT.publish(serial, buffer);
+
+#if DEBUG_TO_SERIAL
+    Serial.println("DEBUG: MQTT PUBLISH>>>");
+    serializeJsonPretty(json, Serial);
+    Serial.println("\n<<<");
+#endif
+  }
+}
+#endif
+
+void postData(const netConfig &postAPI, const String &URL, const DynamicJsonDocument &json) {
+  if (ethClient.connect(postAPI.iot_address, postAPI.iot_port)) {
+    ethClient.print("POST ");
+    ethClient.print(URL);
+    ethClient.println(" HTTP/1.1");
+    ethClient.print("Host: ");
+    ethClient.print(postAPI.iot_address.toString());
+    ethClient.print(":");
+    ethClient.println(postAPI.iot_port);
+    ethClient.println("Content-Type: application/json");
+    ethClient.print("Content-Length: ");
+    ethClient.println(measureJson(json));
+    ethClient.println("Connection: close");
+    ethClient.println();
+    serializeJson(json, ethClient);
+    ethClient.stop();
+
+#if DEBUG_TO_SERIAL
+    Serial.println("DEBUG: HTTP POST>>>");
+    serializeJsonPretty(json, Serial);
+    Serial.println("\n<<<");
+#endif
+  }
+}

requirements-dev.txt

@@ -4,3 +4,4 @@ ipython
 flake8
 pyinstrument
 django-debug-toolbar
+urllib3

requirements.txt

@@ -1,9 +1,11 @@
+pytz
 Django
 djangorestframework
 django-health-check
 psycopg2-binary
-paho-mqtt==1.5.0
-asyncio-mqtt==0.5.0
+paho-mqtt
+kafka-python
+aiomqtt
 PyYAML
 uritemplate
 pygments