Initial IoT Config Service (#287)

* IoT Config service initial implementation of organization and route management services and backend data management, as well as the HPR route update stream

.dockerignore

@@ -0,0 +1,2 @@
+target
+tests

Cargo.toml

@@ -16,6 +16,7 @@ members = [
     "reward_index",
     "metrics",
     "denylist",
+    "iot_config"
 ]
 
 [workspace.package]
@@ -49,8 +50,8 @@ sqlx = {version = "0", features = [
   "macros",
   "runtime-tokio-rustls"
 ]}
+helium-crypto = {git = "https://github.com/helium/helium-crypto-rs", branch = "main", features=["sqlx-postgres", "multisig"]}
 helium-proto = {git = "https://github.com/helium/proto", branch = "master", features = ["services"]}
-helium-crypto = {version = "0.6", features=["sqlx-postgres", "multisig"]}
 reqwest = {version = "0", default-features=false, features = ["gzip", "json", "rustls-tls"]}
 humantime = "2"
 metrics = "0"

docker-compose.yml

@@ -0,0 +1,126 @@
+version: "2.4"
+services:
+
+  iot-config:
+    image: iot-config:latest
+    build:
+      context: .
+      dockerfile: iot_config.Dockerfile
+    depends_on:
+      - postgres
+    ports:
+      - "8080:8080"
+    environment:
+      CFG_DATABASE_URL: postgres://postgres:postgres@postgres:5432/config_db
+      CFG_LISTEN: 0.0.0.0:8080
+      CFG_METRICS_ENDPOINT: 0.0.0.0:19000
+      CFG_NETWORK: mainnet
+      CFG_LOG: info
+
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_DBS: >
+        config_db
+        helius_db
+        mobile_index_db
+        iot_index_db
+        mobile_verified_db
+        iot_verified_db
+      PGDATA: /data
+    ports:
+      - "5432:5432"
+    entrypoint:
+      - /bin/bash
+      - -c
+      - |
+        for db in $${POSTGRES_DBS[@]}
+        do
+            cat > /docker-entrypoint-initdb.d/$${db}-setup.sql <<EOF
+            create database $$db
+                with
+                owner = $$POSTGRES_USER
+                encoding = 'UTF8'
+                lc_collate = 'en_US.utf8'
+                lc_ctype = 'en_US.utf8'
+                tablespace = pg_default
+                connection limit = -1;
+        EOF
+        done
+        docker-entrypoint.sh postgres -c log_statement=all
+    volumes:
+      - db-data:/data
+  minio:
+    image: minio/minio:latest
+    environment:
+      MINIO_ROOT_USER: oracleadmin
+      MINIO_ROOT_PASSWORD: oracleadmin
+    ports:
+      - "9000:9000"
+      - "9090:9090"
+    volumes:
+      - bucket-data:/data
+    command: server /data --console-address ":9090"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+  minio-setup:
+    image: minio/mc:latest
+    depends_on:
+      - minio
+    environment:
+      MINIO_ROOT_USER: oracleadmin
+      MINIO_ROOT_PASSWORD: oracleadmin
+      MINIO_BUCKETS: >
+        mobile-ingest
+        iot-ingest
+        iot-entropy
+        mobile-verified
+        iot-verified
+        mobile-index
+        iot-index
+      ORACLE_ID: oraclesecretid
+      ORACLE_KEY: oraclesecretkey
+    entrypoint:
+      - /bin/bash
+      - -c
+      - |
+        cat > /bucket-policy.json <<EOF
+        {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Sid": "ListObjectsInBucket",
+                    "Effect": "Allow",
+                    "Action": ["s3:ListBucket"],
+                    "Resource": ["arn:aws:s3:::*"]
+                },
+                {
+                    "Sid": "AllObjectActions",
+                    "Effect": "Allow",
+                    "Action": "s3:*Object",
+                    "Resource": ["arn:aws:s3:::*"]
+                }
+            ]
+        }
+        EOF
+        sleep 2
+        /usr/bin/mc alias set localminio http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD}
+        for bucket in $${MINIO_BUCKETS[@]}
+        do
+            if ! /usr/bin/mc ls localminio/$${bucket} > /dev/null 2>&1 ; then
+                echo "creating bucket $${bucket}"
+                /usr/bin/mc mb localminio/$${bucket}
+            fi
+        done
+        /usr/bin/mc admin policy add localminio fullaccess /bucket-policy.json
+        /usr/bin/mc admin user add localminio $${ORACLE_ID} $${ORACLE_KEY}
+        /usr/bin/mc admin policy set localminio fullaccess user=$${ORACLE_ID}
+
+volumes:
+  bucket-data:
+  db-data:

file_store/src/traits/msg_verify.rs

@@ -1,6 +1,9 @@
 use crate::{Error, Result};
 use helium_crypto::{PublicKey, Verify};
-use helium_proto::services::poc_iot::{IotBeaconReportReqV1, IotWitnessReportReqV1};
+use helium_proto::services::{
+    iot_config::{OrgCreateHeliumReqV1, OrgCreateRoamerReqV1, OrgDisableReqV1},
+    poc_iot::{IotBeaconReportReqV1, IotWitnessReportReqV1},
+};
 use helium_proto::{
     services::poc_mobile::{CellHeartbeatReqV1, DataTransferSessionReqV1, SpeedtestReqV1},
     Message,
@@ -29,6 +32,9 @@ impl_msg_verify!(SpeedtestReqV1, signature);
 impl_msg_verify!(IotBeaconReportReqV1, signature);
 impl_msg_verify!(IotWitnessReportReqV1, signature);
 impl_msg_verify!(DataTransferSessionReqV1, signature);
+impl_msg_verify!(OrgCreateHeliumReqV1, signature);
+impl_msg_verify!(OrgCreateRoamerReqV1, signature);
+impl_msg_verify!(OrgDisableReqV1, signature);
 
 #[cfg(test)]
 mod test {

iot_config.Dockerfile

@@ -0,0 +1,34 @@
+FROM rust:1.65 AS builder
+
+RUN rustup toolchain install nightly
+
+# Copy cargo file and workspace dependency crates to cache build
+COPY Cargo.toml Cargo.lock ./
+COPY db_store ./db_store/
+COPY density_scaler ./density_scaler/
+COPY file_store ./file_store/
+COPY metrics ./metrics/
+COPY node_follower /node_follower/
+COPY iot_config/Cargo.toml ./iot_config/Cargo.toml
+
+RUN mkdir ./iot_config/src \
+ # Create a dummy project file to build deps around
+ && echo "fn main() {}" > ./iot_config/src/main.rs \
+ # Remove unused members of the workspace to avoid compile error on missing members
+ && sed -i -e '/ingest/d'    -e '/mobile_rewards/d' -e '/mobile_verifier/d' \
+           -e '/poc_entropy/d'   -e '/iot_verifier/d'   -e '/poc_iot_injector/d' \
+           -e '/reward_index/d'   -e '/denylist/d' \
+           Cargo.toml \
+ # Build on nightly cargo to use sparse-registry to avoid crates indexing infinite loop
+ && cargo +nightly build --package iot-config --release -Z sparse-registry
+
+COPY iot_config ./iot_config/
+RUN cargo +nightly build --package iot-config --release -Z sparse-registry
+
+FROM debian:bullseye-slim
+
+COPY --from=builder ./target/release/iot-config /opt/iot_config/bin/iot-config
+
+EXPOSE 8080
+
+CMD ["/opt/iot_config/bin/iot-config", "server"]

iot_config/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "iot-config"
+version = "0.1.0"
+description = "Configuration APIs for the IoT subnetwork"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow = {workspace = true}
+async-trait = {workspace = true}
+base64 = {workspace = true}
+clap = {workspace = true}
+config = {workspace = true}
+db-store = {path = "../db_store"}
+file-store = {path = "../file_store"}
+futures = {workspace = true}
+futures-util = {workspace = true}
+helium-crypto = {workspace = true}
+helium-proto = {workspace = true}
+metrics = {workspace = true}
+metrics-exporter-prometheus = {workspace = true}
+poc-metrics = {path = "../metrics"}
+prost = {workspace = true}
+serde = {workspace = true}
+serde_json = {workspace = true}
+sqlx = {workspace = true}
+thiserror = {workspace = true}
+tokio = {workspace = true}
+tokio-stream = "0"
+tonic = {workspace = true}
+tracing = {workspace = true}
+tracing-subscriber = {workspace = true}
+triggered = {workspace = true}

iot_config/README.md

@@ -0,0 +1,32 @@
+# IoT Config Service
+
+The IoT Config Service provides configuration settings and values for the
+LoRaWAN IoT Helium Subnetwork. Actors on the IoT subnetwork can interact
+with the gRPC APIs provided by the Config service to perform various
+operations of the network according to role, including but not limited to:
+
+- Community Management (the Foundation) can issue network configuration variables
+  (formerly known as Chain Variables) to adjust PoC settings
+- Users of the network can manage their organization's routes
+- Gateways can request their region and associated region parameters
+
+The IoT Config service provides 4 major gRPC services:
+
+## `route`
+
+provides routing information for devices on the LoRaWAN network to correctly
+route packets and the management of those routes by their controlling organizations
+
+## `org`
+
+management of organizations using the Helium LoRaWAN network
+
+## `session key filter`
+
+management of session key filters by organizations to decrypt device and other
+data associated with usage of the network
+
+## `gateway`
+
+configuration data provided to LoRaWAN gateways serving the network, including
+the current region parameters for the region in which the gateway is asserted

iot_config/migrations/1_setup.sql

@@ -0,0 +1,27 @@
+-- This extension gives us `uuid_generate_v1mc()` which generates UUIDs that cluster better than `gen_random_uuid()`
+-- while still being difficult to predict and enumerate.
+-- Also, while unlikely, `gen_random_uuid()` can in theory produce collisions which can trigger spurious errors on
+-- insertion, whereas it's much less likely with `uuid_generate_v1mc()`.
+create extension if not exists "uuid-ossp";
+
+create or replace function set_updated_at()
+    returns trigger as
+$$
+begin
+    NEW.updated_at = now();
+    return NEW;
+end;
+$$ language plpgsql;
+
+create or replace function trigger_updated_at(tablename regclass)
+    returns void as
+$$
+begin
+    execute format('CREATE TRIGGER set_updated_at
+        BEFORE UPDATE
+        ON %s
+        FOR EACH ROW
+        WHEN (OLD is distinct from NEW)
+    EXECUTE FUNCTION set_updated_at();', tablename);
+end;
+$$ language plpgsql;